v)E>F6+2#X=FMnQ=BCTJ
z=i#eNSh{C3SUxDl0Ox|G1V3N~<#pD>DW~F-whn{P=^tkXm`R(`7&YP@*&vh
zV+5l@ALMIfJPuuP5Y;`;+so+A;d6K6w}PEy31b}_G>e@xQ5YQojV8Vanx}!XEND^G
z(cWhF*NNl1a^=00&@*U_D{$#i6s{I{^YX9PMUv?ZX*Mu_cBa!^5{zRc(R*j6N|=^U
zHMURWDSHAto;j>Y`cW-DzjZt=pPQi0qM{w<`IIfTM&8c;mgh8el^x7V)-zPdPADO6
zb;WkycG!Ts(dOltlf1l3d#8=NqisiENWIO==5Ca12qxR$eHRZyHN)cN8EIxxYs$1^
zT0Y_SCmg}+yH-X!aN@MSd#3$)%RQ03LX>rd
zbj$Zn?@}&wmQoqQDD%<6i8*%w*q%qsRfewWtg7+HWDF1>R7
zgh>#Uws>41#zk-G-GBh~E9S>t89NrsMpd@s9vOiVC|H@hwy?D5_o3n1xafrIyxsF}
zSG?20y`LBKI*B5>cNL8r^R)17)1lNwHG+Q@X8T6bLiS5|*BSJ>@=2`K2NUv73HD9b
zRDBXCqc7}g53-&`k~MdU=30Pgfr3UCvRfbPI1t!iUr`-RW4vl@{>)2Q#aB`gWI3O6t4Xqi{fh1WW6%)
zLGWEX&|jW?=6d)l^8wsZ8I|7bmpjY1NZ5Dt`~Dd!*(o;WZ0YiTV#t*9h^E0Hoc48f
zXX)33r!vJ8Kbx%jk#+YcI8%jeD`Le?B(tE`aOl`E=AKD!Eq(0rDWml7rcSvij$pPc
zQoPy-5vd+CcSY{Kc-{myCd3pSXrp%$iiqYI5!WE19CMm_Z6%?zjH3^4z54R)1>D_d
zvW`B2{mWRO7N;~t7jvc1^YxSX(aO$3oG9$^r?Out>(`D$a*Eu)dqUmH67G%u=IEX{
zF@Ay=5m3A@wk{Z~yFNXu$eN=wxKBTWtNbySVc(Ut56kXpT90_boaH$|Z-k;qO8a>|
zaFVbU)rR-4c}_iUQS7tu41Zp@9Q+fMk|Rl)Yl#l+I$Izak7q#=00IfPL2_C{1iqHFAAVJE-RYi-ELZWJMG9q-@e!h!GH5}=nF{`S)%Ou~4$
zE?hS9N;S)mu#E`Bsy
z4h&(1o>mQTK%p}t<-2AIQKVeU>Zn3G?&`-p=3N8G|`
zrx$pih3I5j+1weA$iAazUg1bIao1j@+%t9TaB*+Tt{2^~Z1JG*y%O`+SRL9!h|{IV
zct5g4m{1wbBO+w{?$YfKhpgXwMididsv>QZj`jNW2m6SNK}{hbc!lmK?5G$dW`e7=
z)nfgLv}M}E9yDik6YR67pz@d9*`+s&=j+jv(J02_7YrRs>P)lB3w!V|IIZx5ic_0pR4lwLd?~1
z|L$RM`vtcAhVYjzveRGuY;Nh=bfsu!_meRBsrmy`NsWzVK1=Z4tZ&uQ+{lXDGb^v1x3Tn7iK;thW
zLlNP+7BR9N+XCa*B4SDgj%GN0e>s(mPkD*56u7xR=L)r?X*__eSpUL(<$lUdhS2)6
zt%ooNq8`y$&4y69bU
zFk};ldm3j4NEkg%N#3vsu%lI4w27m_Db1xbG#3OlJ`l)zhE8~Q!Z~810w0+c0Z^^_
zeGWROTv6W!qH)p{SYT=$#C4vp6c6#m
zT^s3eejStP(0Zjl7;Jk{oe?o&-Vfr9QfYmeqqi95h9MkC^UZC;0a3z1eTx)l^YEi>
z%i%CsoVQN#kW8%?Pv-8pxiw;UoI6c_xeSd)?Ct;s=s&qSB4!uS0dwF!{nDeWd>dgh
z+T$bBy4(4rVGDj&e;}?P`Qy&L?ZXVchMI+N1I=vEdI=Gz={=IR?8|b8DwZ
z;FUOv>B?aTc8(zlrsnl06vujuJ-LzA)fXEGH0GD_F
z8Sq_$sQtg_ydM=B2m-c7WZ$3P(22*!h0vO9;2O>UdY#K;_lvNQmN!7?@;Mk~PWue~
z7=H?&|5p7kt8>l27+avtd0@6x&|;QcbM9$)y5)@j7X7cwxgzo7smlEA1K^)fA~0ET
z&Uer@3;@dXGW$irQN(ljlG80GR*VrvejT!(l$3Xf15uwNZE5
Dk3PG3
literal 0
HcmV?d00001
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore
new file mode 100644
index 000000000..50af03172
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml
new file mode 100644
index 000000000..55508024f
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A legacy Helm chart for Kubernetes
+name: helmchartwithdeps-v1
+version: 0.3.0
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml
new file mode 100644
index 000000000..d6c815e6f
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml
@@ -0,0 +1,4 @@
+dependencies:
+- name: helmchart-v1
+ version: "0.2.0"
+ repository: "file://../helmchart-v1"
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt
new file mode 100644
index 000000000..c9a8aa76a
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt
@@ -0,0 +1,21 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+ {{- range .paths }}
+ http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+ {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+ export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helmchart-v1.fullname" . }})
+ export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+ echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+ NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+ You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helmchart-v1.fullname" . }}'
+ export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helmchart-v1.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+ echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+ export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helmchart-v1.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+ echo "Visit http://127.0.0.1:8080 to use your application"
+ kubectl port-forward $POD_NAME 8080:80
+{{- end }}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl
new file mode 100644
index 000000000..ecb988262
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl
@@ -0,0 +1,56 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "helmchart-v1.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "helmchart-v1.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "helmchart-v1.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "helmchart-v1.labels" -}}
+app.kubernetes.io/name: {{ include "helmchart-v1.name" . }}
+helm.sh/chart: {{ include "helmchart-v1.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "helmchart-v1.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+ {{ default (include "helmchart-v1.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+ {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml
new file mode 100644
index 000000000..8a435b3a1
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml
@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "helmchart-v1.fullname" . }}
+ labels:
+{{ include "helmchart-v1.labels" . | indent 4 }}
+spec:
+ replicas: {{ .Values.replicaCount }}
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ include "helmchart-v1.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: {{ include "helmchart-v1.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ spec:
+ {{- with .Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ serviceAccountName: {{ template "helmchart-v1.serviceAccountName" . }}
+ securityContext:
+ {{- toYaml .Values.podSecurityContext | nindent 8 }}
+ containers:
+ - name: {{ .Chart.Name }}
+ securityContext:
+ {{- toYaml .Values.securityContext | nindent 12 }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ ports:
+ - name: http
+ containerPort: 80
+ protocol: TCP
+ livenessProbe:
+ httpGet:
+ path: /
+ port: http
+ readinessProbe:
+ httpGet:
+ path: /
+ port: http
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml
new file mode 100644
index 000000000..7db207166
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml
@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "helmchart-v1.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+ name: {{ $fullName }}
+ labels:
+{{ include "helmchart-v1.labels" . | indent 4 }}
+ {{- with .Values.ingress.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+{{- if .Values.ingress.tls }}
+ tls:
+ {{- range .Values.ingress.tls }}
+ - hosts:
+ {{- range .hosts }}
+ - {{ . | quote }}
+ {{- end }}
+ secretName: {{ .secretName }}
+ {{- end }}
+{{- end }}
+ rules:
+ {{- range .Values.ingress.hosts }}
+ - host: {{ .host | quote }}
+ http:
+ paths:
+ {{- range .paths }}
+ - path: {{ . }}
+ backend:
+ serviceName: {{ $fullName }}
+ servicePort: {{ $svcPort }}
+ {{- end }}
+ {{- end }}
+{{- end }}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml
new file mode 100644
index 000000000..81a8cb688
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "helmchart-v1.fullname" . }}
+ labels:
+{{ include "helmchart-v1.labels" . | indent 4 }}
+spec:
+ type: {{ .Values.service.type }}
+ ports:
+ - port: {{ .Values.service.port }}
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ app.kubernetes.io/name: {{ include "helmchart-v1.name" . }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml
new file mode 100644
index 000000000..2f9b53dcb
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml
@@ -0,0 +1,8 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ template "helmchart-v1.serviceAccountName" . }}
+ labels:
+{{ include "helmchart-v1.labels" . | indent 4 }}
+{{- end -}}
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml
new file mode 100644
index 000000000..da5b5c324
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: "{{ include "helmchart-v1.fullname" . }}-test-connection"
+ labels:
+{{ include "helmchart-v1.labels" . | indent 4 }}
+ annotations:
+ "helm.sh/hook": test-success
+spec:
+ containers:
+ - name: wget
+ image: busybox
+ command: ['wget']
+ args: ['{{ include "helmchart-v1.fullname" . }}:{{ .Values.service.port }}']
+ restartPolicy: Never
diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml
new file mode 100644
index 000000000..3c03b2cd9
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml
@@ -0,0 +1,68 @@
+# Default values for helmchart-v1.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+ repository: nginx
+ tag: stable
+ pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+ # Specifies whether a service account should be created
+ create: true
+ # The name of the service account to use.
+ # If not set and create is true, a name is generated using the fullname template
+ name: ""
+
+podSecurityContext: {}
+ # fsGroup: 2000
+
+securityContext: {}
+ # capabilities:
+ # drop:
+ # - ALL
+ # readOnlyRootFilesystem: true
+ # runAsNonRoot: true
+ # runAsUser: 1000
+
+service:
+ type: ClusterIP
+ port: 80
+
+ingress:
+ enabled: false
+ annotations: {}
+ # kubernetes.io/ingress.class: nginx
+ # kubernetes.io/tls-acme: "true"
+ hosts:
+ - host: chart-example.local
+ paths: []
+
+ tls: []
+ # - secretName: chart-example-tls
+ # hosts:
+ # - chart-example.local
+
+resources: {}
+ # We usually recommend not to specify default resources and to leave this as a conscious
+ # choice for the user. This also increases chances charts run on environments with little
+ # resources, such as Minikube. If you do want to specify resources, uncomment the following
+ # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
From 44c18633349f867f71ba3f20d0e57a4f734bfe36 Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Sat, 30 Oct 2021 01:27:04 +0200
Subject: [PATCH 0222/1397] internal/helm: add repository cache helpers
This commits adds simple caching capabilities to the
`ChartRepository`, which makes it possible to load the `Index` from a
defined `CachePath` using `LoadFromCache()`, and to download the index
to a new `CachePath` using `CacheIndex()`.
In addition, the repository tests have been updated to make use of
Gomega, and some missing ones have been added.
Signed-off-by: Hidde Beydals
---
internal/helm/repository.go | 156 +++++++++--
internal/helm/repository_test.go | 443 ++++++++++++++++++++-----------
internal/helm/utils_test.go | 60 +++++
3 files changed, 474 insertions(+), 185 deletions(-)
create mode 100644 internal/helm/utils_test.go
diff --git a/internal/helm/repository.go b/internal/helm/repository.go
index 49728452d..c57df111f 100644
--- a/internal/helm/repository.go
+++ b/internal/helm/repository.go
@@ -18,12 +18,17 @@ package helm
import (
"bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
"fmt"
"io"
"net/url"
+ "os"
"path"
"sort"
"strings"
+ "sync"
"github.com/Masterminds/semver/v3"
"helm.sh/helm/v3/pkg/getter"
@@ -33,20 +38,37 @@ import (
"github.com/fluxcd/pkg/version"
)
+var ErrNoChartIndex = errors.New("no chart index")
+
// ChartRepository represents a Helm chart repository, and the configuration
-// required to download the chart index, and charts from the repository.
+// required to download the chart index and charts from the repository.
+// All methods are thread safe unless defined otherwise.
type ChartRepository struct {
- URL string
- Index *repo.IndexFile
- Client getter.Getter
+ // URL the ChartRepository's index.yaml can be found at,
+ // without the index.yaml suffix.
+ URL string
+ // Client to use while downloading the Index or a chart from the URL.
+ Client getter.Getter
+ // Options to configure the Client with while downloading the Index
+ // or a chart from the URL.
Options []getter.Option
+ // CachePath is the path of a cached index.yaml for read-only operations.
+ CachePath string
+ // Index contains a loaded chart repository index if not nil.
+ Index *repo.IndexFile
+ // Checksum contains the SHA256 checksum of the loaded chart repository
+ // index bytes.
+ Checksum string
+
+ *sync.RWMutex
}
// NewChartRepository constructs and returns a new ChartRepository with
// the ChartRepository.Client configured to the getter.Getter for the
// repository URL scheme. It returns an error on URL parsing failures,
// or if there is no getter available for the scheme.
-func NewChartRepository(repositoryURL string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) {
+func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) {
+ r := newChartRepository()
u, err := url.Parse(repositoryURL)
if err != nil {
return nil, err
@@ -55,17 +77,29 @@ func NewChartRepository(repositoryURL string, providers getter.Providers, opts [
if err != nil {
return nil, err
}
+ r.URL = repositoryURL
+ r.CachePath = cachePath
+ r.Client = c
+ r.Options = opts
+ return r, nil
+}
+
+func newChartRepository() *ChartRepository {
return &ChartRepository{
- URL: repositoryURL,
- Client: c,
- Options: opts,
- }, nil
+ RWMutex: &sync.RWMutex{},
+ }
}
// Get returns the repo.ChartVersion for the given name, the version is expected
// to be a semver.Constraints compatible string. If version is empty, the latest
// stable version will be returned and prerelease versions will be ignored.
func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) {
+ r.RLock()
+ defer r.RUnlock()
+
+ if r.Index == nil {
+ return nil, ErrNoChartIndex
+ }
cvs, ok := r.Index.Entries[name]
if !ok {
return nil, repo.ErrNoChartName
@@ -114,7 +148,7 @@ func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) {
lookup[v] = cv
}
if len(matchedVersions) == 0 {
- return nil, fmt.Errorf("no chart version found for %s-%s", name, ver)
+ return nil, fmt.Errorf("no '%s' chart with version matching '%s' found", name, ver)
}
// Sort versions
@@ -145,7 +179,7 @@ func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) {
// ChartRepository. It returns a bytes.Buffer containing the chart data.
func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer, error) {
if len(chart.URLs) == 0 {
- return nil, fmt.Errorf("chart %q has no downloadable URLs", chart.Name)
+ return nil, fmt.Errorf("chart '%s' has no downloadable URLs", chart.Name)
}
// TODO(hidde): according to the Helm source the first item is not
@@ -175,13 +209,9 @@ func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer
return r.Client.Get(u.String(), r.Options...)
}
-// LoadIndex loads the given bytes into the Index while performing
-// minimal validity checks. It fails if the API version is not set
-// (repo.ErrNoAPIVersion), or if the unmarshal fails.
-//
-// The logic is derived from and on par with:
-// https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index.go#L301
-func (r *ChartRepository) LoadIndex(b []byte) error {
+// LoadIndexFromBytes loads Index from the given bytes.
+// It returns a repo.ErrNoAPIVersion error if the API version is not set
+func (r *ChartRepository) LoadIndexFromBytes(b []byte) error {
i := &repo.IndexFile{}
if err := yaml.UnmarshalStrict(b, i); err != nil {
return err
@@ -190,14 +220,68 @@ func (r *ChartRepository) LoadIndex(b []byte) error {
return repo.ErrNoAPIVersion
}
i.SortEntries()
+
+ r.Lock()
r.Index = i
+ r.Checksum = fmt.Sprintf("%x", sha256.Sum256(b))
+ r.Unlock()
return nil
}
+// LoadFromFile reads the file at the given path and loads it into Index.
+func (r *ChartRepository) LoadFromFile(path string) error {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ return r.LoadIndexFromBytes(b)
+}
+
+// CacheIndex attempts to write the index from the remote into a new temporary file
+// using DownloadIndex, and sets CachePath.
+// It returns the SHA256 checksum of the downloaded index bytes, or an error.
+// The caller is expected to handle the garbage collection of CachePath, and to
+// load the Index separately using LoadFromCache if required.
+func (r *ChartRepository) CacheIndex() (string, error) {
+ f, err := os.CreateTemp("", "chart-index-*.yaml")
+ if err != nil {
+ return "", fmt.Errorf("failed to create temp file to cache index to: %w", err)
+ }
+
+ h := sha256.New()
+ mw := io.MultiWriter(f, h)
+ if err = r.DownloadIndex(mw); err != nil {
+ f.Close()
+ os.RemoveAll(f.Name())
+ return "", fmt.Errorf("failed to cache index to '%s': %w", f.Name(), err)
+ }
+ if err = f.Close(); err != nil {
+ os.RemoveAll(f.Name())
+ return "", fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err)
+ }
+
+ r.Lock()
+ r.CachePath = f.Name()
+ r.Unlock()
+ return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// LoadFromCache attempts to load the Index from the configured CachePath.
+// It returns an error if no CachePath is set, or if the load failed.
+func (r *ChartRepository) LoadFromCache() error {
+ r.RLock()
+ if cachePath := r.CachePath; cachePath != "" {
+ r.RUnlock()
+ return r.LoadFromFile(cachePath)
+ }
+ r.RUnlock()
+ return fmt.Errorf("no cache path set")
+}
+
// DownloadIndex attempts to download the chart repository index using
-// the Client and set Options, and loads the index file into the Index.
-// It returns an error on URL parsing and Client failures.
-func (r *ChartRepository) DownloadIndex() error {
+// the Client and set Options, and writes the index to the given io.Writer.
+// It returns an url.Error if the URL failed to parse.
+func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) {
u, err := url.Parse(r.URL)
if err != nil {
return err
@@ -205,14 +289,36 @@ func (r *ChartRepository) DownloadIndex() error {
u.RawPath = path.Join(u.RawPath, "index.yaml")
u.Path = path.Join(u.Path, "index.yaml")
- res, err := r.Client.Get(u.String(), r.Options...)
+ var res *bytes.Buffer
+ res, err = r.Client.Get(u.String(), r.Options...)
if err != nil {
return err
}
- b, err := io.ReadAll(res)
- if err != nil {
+ if _, err = io.Copy(w, res); err != nil {
return err
}
+ return nil
+}
+
+// HasIndex returns true if the Index is not nil.
+func (r *ChartRepository) HasIndex() bool {
+ r.RLock()
+ defer r.RUnlock()
+ return r.Index != nil
+}
+
+// HasCacheFile returns true if CachePath is not empty.
+func (r *ChartRepository) HasCacheFile() bool {
+ r.RLock()
+ defer r.RUnlock()
+ return r.CachePath != ""
+}
- return r.LoadIndex(b)
+// UnloadIndex sets the Index to nil.
+func (r *ChartRepository) UnloadIndex() {
+ if r != nil {
+ r.Lock()
+ r.Index = nil
+ r.Unlock()
+ }
}
diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go
index c51a19d40..95ccc7b80 100644
--- a/internal/helm/repository_test.go
+++ b/internal/helm/repository_test.go
@@ -18,45 +18,38 @@ package helm
import (
"bytes"
+ "crypto/sha256"
+ "fmt"
"net/url"
"os"
- "reflect"
- "strings"
"testing"
"time"
+ . "github.com/onsi/gomega"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/repo"
)
+var now = time.Now()
+
const (
- testfile = "testdata/local-index.yaml"
- chartmuseumtestfile = "testdata/chartmuseum-index.yaml"
- unorderedtestfile = "testdata/local-index-unordered.yaml"
- indexWithDuplicates = `
-apiVersion: v1
-entries:
- nginx:
- - urls:
- - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz
- name: nginx
- description: string
- version: 0.2.0
- home: https://github.com/something/else
- digest: "sha256:1234567890abcdef"
- nginx:
- - urls:
- - https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz
- - http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz
- name: alpine
- description: string
- version: 1.0.0
- home: https://github.com/something
- digest: "sha256:1234567890abcdef"
-`
+ testFile = "testdata/local-index.yaml"
+ chartmuseumTestFile = "testdata/chartmuseum-index.yaml"
+ unorderedTestFile = "testdata/local-index-unordered.yaml"
)
+// mockGetter can be used as a simple mocking getter.Getter implementation.
+type mockGetter struct {
+ requestedURL string
+ response []byte
+}
+
+func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) {
+ g.requestedURL = url
+ return bytes.NewBuffer(g.response), nil
+}
+
func TestNewChartRepository(t *testing.T) {
repositoryURL := "https://example.com"
providers := getter.Providers{
@@ -68,60 +61,74 @@ func TestNewChartRepository(t *testing.T) {
options := []getter.Option{getter.WithBasicAuth("username", "password")}
t.Run("should construct chart repository", func(t *testing.T) {
- r, err := NewChartRepository(repositoryURL, providers, options)
- if err != nil {
- t.Error(err)
- }
- if got := r.URL; got != repositoryURL {
- t.Fatalf("Expecting %q repository URL, got: %q", repositoryURL, got)
- }
- if r.Client == nil {
- t.Fatalf("Expecting client, got nil")
- }
- if !reflect.DeepEqual(r.Options, options) {
- t.Fatalf("Client options mismatth")
- }
+ g := NewWithT(t)
+
+ r, err := NewChartRepository(repositoryURL, "", providers, options)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(r).ToNot(BeNil())
+ g.Expect(r.URL).To(Equal(repositoryURL))
+ g.Expect(r.Client).ToNot(BeNil())
+ g.Expect(r.Options).To(Equal(options))
})
t.Run("should error on URL parsing failure", func(t *testing.T) {
- _, err := NewChartRepository("https://ex ample.com", nil, nil)
- switch err.(type) {
- case *url.Error:
- default:
- t.Fatalf("Expecting URL error, got: %v", err)
- }
+ g := NewWithT(t)
+ r, err := NewChartRepository("https://ex ample.com", "", nil, nil)
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err).To(BeAssignableToTypeOf(&url.Error{}))
+ g.Expect(r).To(BeNil())
+
})
t.Run("should error on unsupported scheme", func(t *testing.T) {
- _, err := NewChartRepository("http://example.com", providers, nil)
- if err == nil {
- t.Fatalf("Expecting unsupported scheme error")
- }
+ g := NewWithT(t)
+
+ r, err := NewChartRepository("http://example.com", "", providers, nil)
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(Equal("scheme \"http\" not supported"))
+ g.Expect(r).To(BeNil())
})
}
func TestChartRepository_Get(t *testing.T) {
- i := repo.NewIndexFile()
- i.Add(&chart.Metadata{Name: "chart", Version: "0.0.1"}, "chart-0.0.1.tgz", "http://example.com/charts", "sha256:1234567890")
- i.Add(&chart.Metadata{Name: "chart", Version: "0.1.0"}, "chart-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Add(&chart.Metadata{Name: "chart", Version: "0.1.1"}, "chart-0.1.1.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+b.min.minute"}, "chart-0.1.5+b.min.minute.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Minute)
- i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+a.min.hour"}, "chart-0.1.5+a.min.hour.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Hour)
- i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+c.now"}, "chart-0.1.5+c.now.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Add(&chart.Metadata{Name: "chart", Version: "0.2.0"}, "chart-0.2.0.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Add(&chart.Metadata{Name: "chart", Version: "1.0.0"}, "chart-1.0.0.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.Add(&chart.Metadata{Name: "chart", Version: "1.1.0-rc.1"}, "chart-1.1.0-rc.1.tgz", "http://example.com/charts", "sha256:1234567890abc")
- i.SortEntries()
- r := &ChartRepository{Index: i}
+ g := NewWithT(t)
+
+ r := newChartRepository()
+ r.Index = repo.NewIndexFile()
+ charts := []struct {
+ name string
+ version string
+ url string
+ digest string
+ created time.Time
+ }{
+ {name: "chart", version: "0.0.1", url: "http://example.com/charts", digest: "sha256:1234567890"},
+ {name: "chart", version: "0.1.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"},
+ {name: "chart", version: "0.1.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"},
+ {name: "chart", version: "0.1.5+b.min.minute", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Minute)},
+ {name: "chart", version: "0.1.5+a.min.hour", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Hour)},
+ {name: "chart", version: "0.1.5+c.now", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now},
+ {name: "chart", version: "0.2.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"},
+ {name: "chart", version: "1.0.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"},
+ {name: "chart", version: "1.1.0-rc.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"},
+ }
+ for _, c := range charts {
+ g.Expect(r.Index.MustAdd(
+ &chart.Metadata{Name: c.name, Version: c.version},
+ fmt.Sprintf("%s-%s.tgz", c.name, c.version), c.url, c.digest),
+ ).To(Succeed())
+ if !c.created.IsZero() {
+ r.Index.Entries["chart"][len(r.Index.Entries["chart"])-1].Created = c.created
+ }
+ }
+ r.Index.SortEntries()
tests := []struct {
name string
chartName string
chartVersion string
wantVersion string
- wantErr bool
+ wantErr string
}{
{
name: "exact match",
@@ -151,12 +158,12 @@ func TestChartRepository_Get(t *testing.T) {
name: "unfulfilled range",
chartName: "chart",
chartVersion: ">2.0.0",
- wantErr: true,
+ wantErr: "no 'chart' chart with version matching '>2.0.0' found",
},
{
name: "invalid chart",
chartName: "non-existing",
- wantErr: true,
+ wantErr: repo.ErrNoChartName.Error(),
},
{
name: "match newest if ambiguous",
@@ -168,14 +175,19 @@ func TestChartRepository_Get(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
cv, err := r.Get(tt.chartName, tt.chartVersion)
- if (err != nil) != tt.wantErr {
- t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(cv).To(BeNil())
return
}
- if err == nil && !strings.Contains(cv.Metadata.Version, tt.wantVersion) {
- t.Errorf("Get() unexpected version = %s, want = %s", cv.Metadata.Version, tt.wantVersion)
- }
+ g.Expect(cv).ToNot(BeNil())
+ g.Expect(cv.Metadata.Name).To(Equal(tt.chartName))
+ g.Expect(cv.Metadata.Version).To(Equal(tt.wantVersion))
+ g.Expect(err).ToNot(HaveOccurred())
})
}
}
@@ -212,117 +224,257 @@ func TestChartRepository_DownloadChart(t *testing.T) {
},
}
for _, tt := range tests {
+ tt := tt
t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+ t.Parallel()
+
mg := mockGetter{}
r := &ChartRepository{
URL: tt.url,
Client: &mg,
}
- _, err := r.DownloadChart(tt.chartVersion)
- if (err != nil) != tt.wantErr {
- t.Errorf("DownloadChart() error = %v, wantErr %v", err, tt.wantErr)
+ res, err := r.DownloadChart(tt.chartVersion)
+ if tt.wantErr {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(res).To(BeNil())
return
}
- if err == nil && mg.requestedURL != tt.wantURL {
- t.Errorf("DownloadChart() requested URL = %s, wantURL %s", mg.requestedURL, tt.wantURL)
- }
+ g.Expect(mg.requestedURL).To(Equal(tt.wantURL))
+ g.Expect(res).ToNot(BeNil())
+ g.Expect(err).ToNot(HaveOccurred())
})
}
}
func TestChartRepository_DownloadIndex(t *testing.T) {
- b, err := os.ReadFile(chartmuseumtestfile)
- if err != nil {
- t.Fatal(err)
- }
+ g := NewWithT(t)
+
+ b, err := os.ReadFile(chartmuseumTestFile)
+ g.Expect(err).ToNot(HaveOccurred())
+
mg := mockGetter{response: b}
r := &ChartRepository{
URL: "https://example.com",
Client: &mg,
}
- if err := r.DownloadIndex(); err != nil {
+
+ buf := bytes.NewBuffer([]byte{})
+ g.Expect(r.DownloadIndex(buf)).To(Succeed())
+ g.Expect(buf.Bytes()).To(Equal(b))
+ g.Expect(mg.requestedURL).To(Equal(r.URL + "/index.yaml"))
+ g.Expect(err).To(BeNil())
+}
+
+func TestChartRepository_LoadIndexFromBytes(t *testing.T) {
+ tests := []struct {
+ name string
+ b []byte
+ wantName string
+ wantVersion string
+ wantDigest string
+ wantErr string
+ }{
+ {
+ name: "index",
+ b: []byte(`
+apiVersion: v1
+entries:
+ nginx:
+ - urls:
+ - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz
+ name: nginx
+ description: string
+ version: 0.2.0
+ home: https://github.com/something/else
+ digest: "sha256:1234567890abcdef"
+`),
+ wantName: "nginx",
+ wantVersion: "0.2.0",
+ wantDigest: "sha256:1234567890abcdef",
+ },
+ {
+ name: "index without API version",
+ b: []byte(`entries:
+ nginx:
+ - name: nginx`),
+ wantErr: "no API version specified",
+ },
+ {
+ name: "index with duplicate entry",
+ b: []byte(`apiVersion: v1
+entries:
+ nginx:
+ - name: nginx"
+ nginx:
+ - name: nginx`),
+ wantErr: "key \"nginx\" already set in map",
+ },
+ }
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+ t.Parallel()
+
+ r := newChartRepository()
+ err := r.LoadIndexFromBytes(tt.b)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(r.Index).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(r.Index).ToNot(BeNil())
+ got, err := r.Index.Get(tt.wantName, tt.wantVersion)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got.Digest).To(Equal(tt.wantDigest))
+ })
+ }
+}
+
+func TestChartRepository_LoadIndexFromBytes_Unordered(t *testing.T) {
+ b, err := os.ReadFile(unorderedTestFile)
+ if err != nil {
t.Fatal(err)
}
- if expected := r.URL + "/index.yaml"; mg.requestedURL != expected {
- t.Errorf("DownloadIndex() requested URL = %s, wantURL %s", mg.requestedURL, expected)
+ r := newChartRepository()
+ err = r.LoadIndexFromBytes(b)
+ if err != nil {
+ t.Fatal(err)
}
verifyLocalIndex(t, r.Index)
}
// Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108
// to ensure parity with Helm behaviour.
-func TestChartRepository_LoadIndex(t *testing.T) {
+func TestChartRepository_LoadIndexFromFile(t *testing.T) {
tests := []struct {
name string
filename string
}{
{
name: "regular index file",
- filename: testfile,
+ filename: testFile,
},
{
name: "chartmuseum index file",
- filename: chartmuseumtestfile,
+ filename: chartmuseumTestFile,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
t.Parallel()
- b, err := os.ReadFile(tt.filename)
- if err != nil {
- t.Fatal(err)
- }
- r := &ChartRepository{}
- err = r.LoadIndex(b)
- if err != nil {
- t.Fatal(err)
- }
+
+ r := newChartRepository()
+ err := r.LoadFromFile(testFile)
+ g.Expect(err).ToNot(HaveOccurred())
+
verifyLocalIndex(t, r.Index)
})
}
}
-func TestChartRepository_LoadIndex_Duplicates(t *testing.T) {
- r := &ChartRepository{}
- if err := r.LoadIndex([]byte(indexWithDuplicates)); err == nil {
- t.Errorf("Expected an error when duplicate entries are present")
- }
+func TestChartRepository_CacheIndex(t *testing.T) {
+ g := NewWithT(t)
+
+ mg := mockGetter{response: []byte("foo")}
+ expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.response))
+
+ r := newChartRepository()
+ r.URL = "https://example.com"
+ r.Client = &mg
+
+ sum, err := r.CacheIndex()
+ g.Expect(err).To(Not(HaveOccurred()))
+
+ g.Expect(r.CachePath).ToNot(BeEmpty())
+ defer os.RemoveAll(r.CachePath)
+ g.Expect(r.CachePath).To(BeARegularFile())
+ b, _ := os.ReadFile(r.CachePath)
+
+ g.Expect(b).To(Equal(mg.response))
+ g.Expect(sum).To(BeEquivalentTo(expectSum))
}
-func TestChartRepository_LoadIndex_Unordered(t *testing.T) {
- b, err := os.ReadFile(unorderedtestfile)
- if err != nil {
- t.Fatal(err)
+func TestChartRepository_LoadIndexFromCache(t *testing.T) {
+ tests := []struct {
+ name string
+ cachePath string
+ wantErr string
+ }{
+ {
+ name: "cache path",
+ cachePath: chartmuseumTestFile,
+ },
+ {
+ name: "invalid cache path",
+ cachePath: "invalid",
+ wantErr: "open invalid: no such file",
+ },
+ {
+ name: "no cache path",
+ cachePath: "",
+ wantErr: "no cache path set",
+ },
}
- r := &ChartRepository{}
- err = r.LoadIndex(b)
- if err != nil {
- t.Fatal(err)
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ r := newChartRepository()
+ r.CachePath = tt.cachePath
+ err := r.LoadFromCache()
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(r.Index).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ verifyLocalIndex(t, r.Index)
+ })
}
- verifyLocalIndex(t, r.Index)
+}
+
+func TestChartRepository_HasIndex(t *testing.T) {
+ g := NewWithT(t)
+
+ r := newChartRepository()
+ g.Expect(r.HasIndex()).To(BeFalse())
+ r.Index = repo.NewIndexFile()
+ g.Expect(r.HasIndex()).To(BeTrue())
+}
+
+func TestChartRepository_UnloadIndex(t *testing.T) {
+ g := NewWithT(t)
+
+ r := newChartRepository()
+ g.Expect(r.HasIndex()).To(BeFalse())
+ r.Index = repo.NewIndexFile()
+ r.UnloadIndex()
+ g.Expect(r.Index).To(BeNil())
}
func verifyLocalIndex(t *testing.T, i *repo.IndexFile) {
- numEntries := len(i.Entries)
- if numEntries != 3 {
- t.Errorf("Expected 3 entries in index file but got %d", numEntries)
- }
+ g := NewWithT(t)
- alpine, ok := i.Entries["alpine"]
- if !ok {
- t.Fatalf("'alpine' section not found.")
- }
+ g.Expect(i.Entries).ToNot(BeNil())
+ g.Expect(i.Entries).To(HaveLen(3), "expected 3 entries in index file")
- if l := len(alpine); l != 1 {
- t.Fatalf("'alpine' should have 1 chart, got %d", l)
- }
+ alpine, ok := i.Entries["alpine"]
+ g.Expect(ok).To(BeTrue(), "expected 'alpine' entry to exist")
+ g.Expect(alpine).To(HaveLen(1), "'alpine' should have 1 entry")
nginx, ok := i.Entries["nginx"]
- if !ok || len(nginx) != 2 {
- t.Fatalf("Expected 2 nginx entries")
- }
+ g.Expect(ok).To(BeTrue(), "expected 'nginx' entry to exist")
+ g.Expect(nginx).To(HaveLen(2), "'nginx' should have 2 entries")
expects := []*repo.ChartVersion{
{
@@ -370,41 +522,12 @@ func verifyLocalIndex(t *testing.T, i *repo.IndexFile) {
for i, tt := range tests {
expect := expects[i]
- if tt.Name != expect.Name {
- t.Errorf("Expected name %q, got %q", expect.Name, tt.Name)
- }
- if tt.Description != expect.Description {
- t.Errorf("Expected description %q, got %q", expect.Description, tt.Description)
- }
- if tt.Version != expect.Version {
- t.Errorf("Expected version %q, got %q", expect.Version, tt.Version)
- }
- if tt.Digest != expect.Digest {
- t.Errorf("Expected digest %q, got %q", expect.Digest, tt.Digest)
- }
- if tt.Home != expect.Home {
- t.Errorf("Expected home %q, got %q", expect.Home, tt.Home)
- }
-
- for i, url := range tt.URLs {
- if url != expect.URLs[i] {
- t.Errorf("Expected URL %q, got %q", expect.URLs[i], url)
- }
- }
- for i, kw := range tt.Keywords {
- if kw != expect.Keywords[i] {
- t.Errorf("Expected keywords %q, got %q", expect.Keywords[i], kw)
- }
- }
+ g.Expect(tt.Name).To(Equal(expect.Name))
+ g.Expect(tt.Description).To(Equal(expect.Description))
+ g.Expect(tt.Version).To(Equal(expect.Version))
+ g.Expect(tt.Digest).To(Equal(expect.Digest))
+ g.Expect(tt.Home).To(Equal(expect.Home))
+ g.Expect(tt.URLs).To(ContainElements(expect.URLs))
+ g.Expect(tt.Keywords).To(ContainElements(expect.Keywords))
}
}
-
-type mockGetter struct {
- requestedURL string
- response []byte
-}
-
-func (g *mockGetter) Get(url string, options ...getter.Option) (*bytes.Buffer, error) {
- g.requestedURL = url
- return bytes.NewBuffer(g.response), nil
-}
diff --git a/internal/helm/utils_test.go b/internal/helm/utils_test.go
new file mode 100644
index 000000000..62a9e92c2
--- /dev/null
+++ b/internal/helm/utils_test.go
@@ -0,0 +1,60 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+)
+
+func TestNormalizeChartRepositoryURL(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want string
+ }{
+ {
+ name: "with slash",
+ url: "http://example.com/",
+ want: "http://example.com/",
+ },
+ {
+ name: "without slash",
+ url: "http://example.com",
+ want: "http://example.com/",
+ },
+ {
+ name: "double slash",
+ url: "http://example.com//",
+ want: "http://example.com/",
+ },
+ {
+ name: "empty",
+ url: "",
+ want: "",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ got := NormalizeChartRepositoryURL(tt.url)
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
From d60131d16b279e398257df27112b9a6b351fcc01 Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Mon, 1 Nov 2021 09:20:48 +0100
Subject: [PATCH 0223/1397] internal/helm: optimize dependency manager
This commit starts with the optimization of the `DepenendencyManager`,
ensuring the chart indexes are lazy loaded, and replacing the
(limitless) concurrency with a configurable number of workers with a
default of 1.
Signed-off-by: Hidde Beydals
---
internal/helm/dependency_manager.go | 68 ++++++++++++++++--------
internal/helm/dependency_manager_test.go | 11 ++--
2 files changed, 51 insertions(+), 28 deletions(-)
diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go
index 83b42d4d7..19d56c884 100644
--- a/internal/helm/dependency_manager.go
+++ b/internal/helm/dependency_manager.go
@@ -28,6 +28,7 @@ import (
"github.com/Masterminds/semver/v3"
securejoin "github.com/cyphar/filepath-securejoin"
"golang.org/x/sync/errgroup"
+ "golang.org/x/sync/semaphore"
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
)
@@ -58,38 +59,51 @@ type DependencyManager struct {
// Dependencies contains a list of dependencies, and the respective
// repository the dependency can be found at.
Dependencies []*DependencyWithRepository
+ // Workers is the number of concurrent chart-add operations during
+ // Build. Defaults to 1 (non-concurrent).
+ Workers int64
mu sync.Mutex
}
-// Build compiles and builds the dependencies of the Chart.
+// Build compiles and builds the dependencies of the Chart with the
+// configured number of Workers.
func (dm *DependencyManager) Build(ctx context.Context) error {
if len(dm.Dependencies) == 0 {
return nil
}
- errs, ctx := errgroup.WithContext(ctx)
- for _, i := range dm.Dependencies {
- item := i
- errs.Go(func() error {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- }
+ workers := dm.Workers
+ if workers <= 0 {
+ workers = 1
+ }
- var err error
- switch item.Repository {
- case nil:
- err = dm.addLocalDependency(item)
- default:
- err = dm.addRemoteDependency(item)
+ defer func() {
+ for _, dep := range dm.Dependencies {
+ dep.Repository.UnloadIndex()
+ }
+ }()
+
+ group, groupCtx := errgroup.WithContext(ctx)
+ group.Go(func() error {
+ sem := semaphore.NewWeighted(workers)
+ for _, dep := range dm.Dependencies {
+ dep := dep
+ if err := sem.Acquire(groupCtx, 1); err != nil {
+ return err
}
- return err
- })
- }
+ group.Go(func() error {
+ defer sem.Release(1)
+ if dep.Repository == nil {
+ return dm.addLocalDependency(dep)
+ }
+ return dm.addRemoteDependency(dep)
+ })
+ }
+ return nil
+ })
- return errs.Wait()
+ return group.Wait()
}
func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error {
@@ -136,7 +150,18 @@ func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) e
func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error {
if dpr.Repository == nil {
- return fmt.Errorf("no ChartRepository given for '%s' dependency", dpr.Dependency.Name)
+ return fmt.Errorf("no HelmRepository for '%s' dependency", dpr.Dependency.Name)
+ }
+
+ if !dpr.Repository.HasIndex() {
+ if !dpr.Repository.HasCacheFile() {
+ if _, err := dpr.Repository.CacheIndex(); err != nil {
+ return err
+ }
+ }
+ if err := dpr.Repository.LoadFromCache(); err != nil {
+ return err
+ }
}
chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version)
@@ -157,7 +182,6 @@ func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository)
dm.mu.Lock()
dm.Chart.AddDependency(ch)
dm.mu.Unlock()
-
return nil
}
diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go
index 6a38997b2..a8e6a0480 100644
--- a/internal/helm/dependency_manager_test.go
+++ b/internal/helm/dependency_manager_test.go
@@ -182,13 +182,12 @@ func TestBuild_WithRemoteChart(t *testing.T) {
t.Fatal(err)
}
i := repo.NewIndexFile()
- i.Add(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890")
+ i.MustAdd(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890")
mg := mockGetter{response: b}
- cr := &ChartRepository{
- URL: remoteDepFixture.Repository,
- Index: i,
- Client: &mg,
- }
+ cr := newChartRepository()
+ cr.URL = remoteDepFixture.Repository
+ cr.Index = i
+ cr.Client = &mg
dm := DependencyManager{
Chart: &chart,
Dependencies: []*DependencyWithRepository{
From f5f212ff430391c579b78f1ee56db2ec1be54166 Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Thu, 4 Nov 2021 17:31:22 +0100
Subject: [PATCH 0224/1397] internal/helm: introduce ChartBuilder
This commit starts with the creation of a `ChartBuilder` to facilitate
the (conditional) build of a chart outside of the reconciler logic.
The builder can be configured with a set of (modifying) options, which
define together with the type of chart source what steps are taken
during the build.
To better facilitate the builder's needs and attempt to be more
efficient, changes have been made to the `DependencyBuilder` and
`ChartRepository` around (order of) operations and/or lazy-load
capabilities.
Signed-off-by: Hidde Beydals
---
internal/helm/chart.go | 12 +-
internal/helm/chart_builder.go | 384 +++++++++++
internal/helm/chart_builder_test.go | 598 +++++++++++++++++
internal/helm/chart_test.go | 35 +-
internal/helm/dependency_manager.go | 265 ++++++--
internal/helm/dependency_manager_test.go | 634 +++++++++++++++---
internal/helm/repository.go | 70 +-
internal/helm/repository_test.go | 7 +-
.../charts/helmchart/values-prod.yaml | 1 +
.../charts/helmchartwithdeps/Chart.lock | 12 +
10 files changed, 1796 insertions(+), 222 deletions(-)
create mode 100644 internal/helm/chart_builder.go
create mode 100644 internal/helm/chart_builder_test.go
create mode 100644 internal/helm/testdata/charts/helmchart/values-prod.yaml
create mode 100644 internal/helm/testdata/charts/helmchartwithdeps/Chart.lock
diff --git a/internal/helm/chart.go b/internal/helm/chart.go
index accbc69a9..dcc868c1d 100644
--- a/internal/helm/chart.go
+++ b/internal/helm/chart.go
@@ -70,17 +70,17 @@ func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, err
// LoadChartMetadata attempts to load the chart.Metadata from the "Chart.yaml" file in the directory or archive at the
// given chartPath. It takes "requirements.yaml" files into account, and is therefore compatible with the
// chart.APIVersionV1 format.
-func LoadChartMetadata(chartPath string) (*helmchart.Metadata, error) {
+func LoadChartMetadata(chartPath string) (meta *helmchart.Metadata, err error) {
i, err := os.Stat(chartPath)
if err != nil {
return nil, err
}
- switch {
- case i.IsDir():
- return LoadChartMetadataFromDir(chartPath)
- default:
- return LoadChartMetadataFromArchive(chartPath)
+ if i.IsDir() {
+ meta, err = LoadChartMetadataFromDir(chartPath)
+ return
}
+ meta, err = LoadChartMetadataFromArchive(chartPath)
+ return
}
// LoadChartMetadataFromDir loads the chart.Metadata from the "Chart.yaml" file in the directory at the given path.
diff --git a/internal/helm/chart_builder.go b/internal/helm/chart_builder.go
new file mode 100644
index 000000000..7b90cba81
--- /dev/null
+++ b/internal/helm/chart_builder.go
@@ -0,0 +1,384 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/fluxcd/source-controller/internal/fs"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "helm.sh/helm/v3/pkg/chartutil"
+ "sigs.k8s.io/yaml"
+
+ "github.com/fluxcd/pkg/runtime/transform"
+)
+
+// ChartBuilder aims to efficiently build a Helm chart from a directory or packaged chart.
+// It avoids or delays loading the chart into memory in full, working with chart.Metadata
+// as much as it can, and returns early (by copying over the already packaged source chart)
+// if no modifications were made during the build process.
+type ChartBuilder struct {
+ // baseDir is the chroot for the chart builder when path isDir.
+ // It must be (a higher) relative to path. File references (during e.g.
+ // value file merge operations) are not allowed to traverse out of it.
+ baseDir string
+
+ // path is the file or directory path to a chart source.
+ path string
+
+ // chart holds a (partly) loaded chart.Chart, it contains at least the
+ // chart.Metadata, which may expand to the full chart.Chart if required
+ // for Build operations.
+ chart *helmchart.Chart
+
+ // valueFiles holds a list of path references of valueFiles that should be
+ // merged and packaged as a single "values.yaml" during Build.
+ valueFiles []string
+
+ // repositories holds an index of repository URLs and their ChartRepository.
+ // They are used to configure a DependencyManager for missing chart dependencies
+ // if isDir is true.
+ repositories map[string]*ChartRepository
+
+ // getChartRepositoryCallback is used to configure a DependencyManager for
+ // missing chart dependencies if isDir is true.
+ getChartRepositoryCallback GetChartRepositoryCallback
+
+ mu sync.Mutex
+}
+
+// NewChartBuilder constructs a new ChartBuilder for the given chart path.
+// It returns an error if no chart.Metadata can be loaded from the path.
+func NewChartBuilder(path string) (*ChartBuilder, error) {
+ metadata, err := LoadChartMetadata(path)
+ if err != nil {
+ return nil, fmt.Errorf("could not create new chart builder: %w", err)
+ }
+ return &ChartBuilder{
+ path: path,
+ chart: &helmchart.Chart{
+ Metadata: metadata,
+ },
+ }, nil
+}
+
+// WithBaseDir configures the base dir on the ChartBuilder.
+func (b *ChartBuilder) WithBaseDir(p string) *ChartBuilder {
+ b.mu.Lock()
+ b.baseDir = p
+ b.mu.Unlock()
+ return b
+}
+
+// WithValueFiles appends the given paths to the ChartBuilder's valueFiles.
+func (b *ChartBuilder) WithValueFiles(path ...string) *ChartBuilder {
+ b.mu.Lock()
+ b.valueFiles = append(b.valueFiles, path...)
+ b.mu.Unlock()
+ return b
+}
+
+// WithChartRepository indexes the given ChartRepository by the NormalizeChartRepositoryURL,
+// used to configure the DependencyManager if the chart is not packaged.
+func (b *ChartBuilder) WithChartRepository(url string, index *ChartRepository) *ChartBuilder {
+ b.mu.Lock()
+ b.repositories[NormalizeChartRepositoryURL(url)] = index
+ b.mu.Unlock()
+ return b
+}
+
+// WithChartRepositoryCallback configures the GetChartRepositoryCallback used by the
+// DependencyManager if the chart is not packaged.
+func (b *ChartBuilder) WithChartRepositoryCallback(c GetChartRepositoryCallback) *ChartBuilder {
+ b.mu.Lock()
+ b.getChartRepositoryCallback = c
+ b.mu.Unlock()
+ return b
+}
+
+// ChartBuildResult contains the ChartBuilder result, including build specific
+// information about the chart.
+type ChartBuildResult struct {
+ // SourceIsDir indicates if the chart was build from a directory.
+ SourceIsDir bool
+ // Path contains the absolute path to the packaged chart.
+ Path string
+ // ValuesOverwrite holds a structured map with the merged values used
+ // to overwrite chart default "values.yaml".
+ ValuesOverwrite map[string]interface{}
+ // CollectedDependencies contains the number of missing local and remote
+ // dependencies that were collected by the DependencyManager before building
+ // the chart.
+ CollectedDependencies int
+ // Packaged indicates if the ChartBuilder has packaged the chart.
+ // This can for example be false if SourceIsDir is false and ValuesOverwrite
+ // is nil, which makes the ChartBuilder copy the chart source to Path without
+ // making any modifications.
+ Packaged bool
+}
+
+// String returns the Path of the ChartBuildResult.
+func (b *ChartBuildResult) String() string {
+ if b != nil {
+ return b.Path
+ }
+ return ""
+}
+
+// Build attempts to build a new chart using ChartBuilder configuration,
+// writing it to the provided path.
+// It returns a ChartBuildResult containing all information about the resulting chart,
+// or an error.
+func (b *ChartBuilder) Build(ctx context.Context, p string) (_ *ChartBuildResult, err error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ if b.chart == nil {
+ err = fmt.Errorf("chart build failed: no initial chart (metadata) loaded")
+ return
+ }
+ if b.path == "" {
+ err = fmt.Errorf("chart build failed: no path set")
+ return
+ }
+
+ result := &ChartBuildResult{}
+ result.SourceIsDir = pathIsDir(b.path)
+ result.Path = p
+
+ // Merge chart values
+ if err = b.mergeValues(result); err != nil {
+ err = fmt.Errorf("chart build failed: %w", err)
+ return
+ }
+
+ // Ensure chart has all dependencies
+ if err = b.buildDependencies(ctx, result); err != nil {
+ err = fmt.Errorf("chart build failed: %w", err)
+ return
+ }
+
+ // Package (or copy) chart
+ if err = b.packageChart(result); err != nil {
+ err = fmt.Errorf("chart package failed: %w", err)
+ return
+ }
+ return result, nil
+}
+
+// load lazy-loads chart.Chart into chart from the set path, it replaces any previously set
+// chart.Metadata shim.
+func (b *ChartBuilder) load() (err error) {
+ if b.chart == nil || len(b.chart.Files) <= 0 {
+ if b.path == "" {
+ return fmt.Errorf("failed to load chart: path not set")
+ }
+ chart, err := loader.Load(b.path)
+ if err != nil {
+ return fmt.Errorf("failed to load chart: %w", err)
+ }
+ b.chart = chart
+ }
+ return
+}
+
+// buildDependencies builds the missing dependencies for a chart from a directory.
+// Using the chart using a NewDependencyManager and the configured repositories
+// and getChartRepositoryCallback
+// It returns the number of dependencies it collected, or an error.
+func (b *ChartBuilder) buildDependencies(ctx context.Context, result *ChartBuildResult) (err error) {
+ if !result.SourceIsDir {
+ return
+ }
+
+ if err = b.load(); err != nil {
+ err = fmt.Errorf("failed to ensure chart has no missing dependencies: %w", err)
+ return
+ }
+
+ dm := NewDependencyManager(b.chart, b.baseDir, strings.TrimLeft(b.path, b.baseDir)).
+ WithRepositories(b.repositories).
+ WithChartRepositoryCallback(b.getChartRepositoryCallback)
+
+ result.CollectedDependencies, err = dm.Build(ctx)
+ return
+}
+
+// mergeValues strategically merges the valueFiles, it merges using mergeFileValues
+// or mergeChartValues depending on if the chart is sourced from a package or directory.
+// Ir only calls load to propagate the chart if required by the strategy.
+// It returns the merged values, or an error.
+func (b *ChartBuilder) mergeValues(result *ChartBuildResult) (err error) {
+ if len(b.valueFiles) == 0 {
+ return
+ }
+
+ if result.SourceIsDir {
+ result.ValuesOverwrite, err = mergeFileValues(b.baseDir, b.valueFiles)
+ if err != nil {
+ err = fmt.Errorf("failed to merge value files: %w", err)
+ }
+ return
+ }
+
+ // Values equal to default
+ if len(b.valueFiles) == 1 && b.valueFiles[0] == chartutil.ValuesfileName {
+ return
+ }
+
+ if err = b.load(); err != nil {
+ err = fmt.Errorf("failed to merge chart values: %w", err)
+ return
+ }
+
+ if result.ValuesOverwrite, err = mergeChartValues(b.chart, b.valueFiles); err != nil {
+ err = fmt.Errorf("failed to merge chart values: %w", err)
+ return
+ }
+ return nil
+}
+
+// packageChart determines if it should copyFileToPath or packageToPath
+// based on the provided result. It sets Packaged on ChartBuildResult to
+// true if packageToPath is successful.
+func (b *ChartBuilder) packageChart(result *ChartBuildResult) error {
+ // If we are not building from a directory, and we do not have any
+ // replacement values, we can copy over the already packaged source
+ // chart without making any modifications
+ if !result.SourceIsDir && len(result.ValuesOverwrite) == 0 {
+ if err := copyFileToPath(b.path, result.Path); err != nil {
+ return fmt.Errorf("chart build failed: %w", err)
+ }
+ return nil
+ }
+
+ // Package chart to a new temporary directory
+ if err := packageToPath(b.chart, result.Path); err != nil {
+ return fmt.Errorf("chart build failed: %w", err)
+ }
+ result.Packaged = true
+ return nil
+}
+
+// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map.
+// It returns the merge result, or an error.
+func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range paths {
+ cfn := filepath.Clean(p)
+ if cfn == chartutil.ValuesfileName {
+ mergedValues = transform.MergeMaps(mergedValues, chart.Values)
+ continue
+ }
+ var b []byte
+ for _, f := range chart.Files {
+ if f.Name == cfn {
+ b = f.Data
+ break
+ }
+ }
+ if b == nil {
+ return nil, fmt.Errorf("no values file found at path '%s'", p)
+ }
+ values := make(map[string]interface{})
+ if err := yaml.Unmarshal(b, &values); err != nil {
+ return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
+
+// mergeFileValues merges the given value file paths into a single "values.yaml" map.
+// The provided (relative) paths may not traverse outside baseDir. It returns the merge
+// result, or an error.
+func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range paths {
+ secureP, err := securejoin.SecureJoin(baseDir, p)
+ if err != nil {
+ return nil, err
+ }
+ if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() {
+ return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')",
+ strings.TrimPrefix(secureP, baseDir), p)
+ }
+ b, err := os.ReadFile(secureP)
+ if err != nil {
+ return nil, fmt.Errorf("could not read values from file '%s': %w", p, err)
+ }
+ values := make(map[string]interface{})
+ err = yaml.Unmarshal(b, &values)
+ if err != nil {
+ return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
+
+// copyFileToPath attempts to copy in to out. It returns an error if out already exists.
+func copyFileToPath(in, out string) error {
+ o, err := os.Create(out)
+ if err != nil {
+ return fmt.Errorf("failed to create copy target: %w", err)
+ }
+ defer o.Close()
+ i, err := os.Open(in)
+ if err != nil {
+ return fmt.Errorf("failed to open file to copy from: %w", err)
+ }
+ defer i.Close()
+ if _, err := o.ReadFrom(i); err != nil {
+ return fmt.Errorf("failed to read from source during copy: %w", err)
+ }
+ return nil
+}
+
+// packageToPath attempts to package the given chart.Chart to the out filepath.
+func packageToPath(chart *helmchart.Chart, out string) error {
+ o, err := os.MkdirTemp("", "chart-build-*")
+ if err != nil {
+ return fmt.Errorf("failed to create temporary directory for chart: %w", err)
+ }
+ defer os.RemoveAll(o)
+
+ p, err := chartutil.Save(chart, o)
+ if err != nil {
+ return fmt.Errorf("failed to package chart: %w", err)
+ }
+ return fs.RenameWithFallback(p, out)
+}
+
+// pathIsDir returns a boolean indicating if the given path points to a directory.
+// In case os.Stat on the given path returns an error it returns false as well.
+func pathIsDir(p string) bool {
+ if p == "" {
+ return false
+ }
+ if i, err := os.Stat(p); err != nil || !i.IsDir() {
+ return false
+ }
+ return true
+}
diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart_builder_test.go
new file mode 100644
index 000000000..afc0107ce
--- /dev/null
+++ b/internal/helm/chart_builder_test.go
@@ -0,0 +1,598 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "context"
+ "encoding/hex"
+ "fmt"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+
+ . "github.com/onsi/gomega"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "helm.sh/helm/v3/pkg/chartutil"
+ "helm.sh/helm/v3/pkg/repo"
+)
+
+func TestChartBuildResult_String(t *testing.T) {
+ g := NewWithT(t)
+
+ var result *ChartBuildResult
+ g.Expect(result.String()).To(Equal(""))
+ result = &ChartBuildResult{}
+ g.Expect(result.String()).To(Equal(""))
+ result = &ChartBuildResult{Path: "/foo/"}
+ g.Expect(result.String()).To(Equal("/foo/"))
+}
+
+func TestChartBuilder_Build(t *testing.T) {
+ tests := []struct {
+ name string
+ baseDir string
+ path string
+ valueFiles []string
+ repositories map[string]*ChartRepository
+ getChartRepositoryCallback GetChartRepositoryCallback
+ wantErr string
+ }{
+ {
+ name: "builds chart from directory",
+ path: "testdata/charts/helmchart",
+ },
+ {
+ name: "builds chart from package",
+ path: "testdata/charts/helmchart-0.1.0.tgz",
+ },
+ {
+ // TODO(hidde): add more diverse tests
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ b, err := NewChartBuilder(tt.path)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(b).ToNot(BeNil())
+
+ b.WithBaseDir(tt.baseDir)
+ b.WithValueFiles(tt.valueFiles...)
+ b.WithChartRepositoryCallback(b.getChartRepositoryCallback)
+ for k, v := range tt.repositories {
+ b.WithChartRepository(k, v)
+ }
+
+ out := tmpFile("build-0.1.0", ".tgz")
+ defer os.RemoveAll(out)
+ got, err := b.Build(context.TODO(), out)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).ToNot(BeNil())
+
+ g.Expect(got.Path).ToNot(BeEmpty())
+ g.Expect(got.Path).To(Equal(out))
+ g.Expect(got.Path).To(BeARegularFile())
+ _, err = loader.Load(got.Path)
+ g.Expect(err).ToNot(HaveOccurred())
+ })
+ }
+}
+
+func TestChartBuilder_load(t *testing.T) {
+ tests := []struct {
+ name string
+ path string
+ chart *helmchart.Chart
+ wantFunc func(g *WithT, c *helmchart.Chart)
+ wantErr string
+ }{
+ {
+ name: "loads chart",
+ chart: nil,
+ path: "testdata/charts/helmchart-0.1.0.tgz",
+ wantFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Metadata.Name).To(Equal("helmchart"))
+ g.Expect(c.Files).ToNot(BeZero())
+ },
+ },
+ {
+ name: "overwrites chart without any files (metadata shim)",
+ chart: &helmchart.Chart{
+ Metadata: &helmchart.Metadata{Name: "dummy"},
+ },
+ path: "testdata/charts/helmchart-0.1.0.tgz",
+ wantFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Metadata.Name).To(Equal("helmchart"))
+ g.Expect(c.Files).ToNot(BeZero())
+ },
+ },
+ {
+ name: "does not overwrite loaded chart",
+ chart: &helmchart.Chart{
+ Metadata: &helmchart.Metadata{Name: "dummy"},
+ Files: []*helmchart.File{
+ {Name: "mock.yaml", Data: []byte("loaded chart")},
+ },
+ },
+ path: "testdata/charts/helmchart-0.1.0.tgz",
+ wantFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Metadata.Name).To(Equal("dummy"))
+ g.Expect(c.Files).To(HaveLen(1))
+ },
+ },
+ {
+ name: "no path",
+ wantErr: "failed to load chart: path not set",
+ },
+ {
+ name: "invalid chart",
+ path: "testdata/charts/empty.tgz",
+ wantErr: "failed to load chart: no files in chart archive",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ b := &ChartBuilder{
+ path: tt.path,
+ chart: tt.chart,
+ }
+ err := b.load()
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ if tt.wantFunc != nil {
+ tt.wantFunc(g, b.chart)
+ }
+ })
+ }
+}
+
+func TestChartBuilder_buildDependencies(t *testing.T) {
+ g := NewWithT(t)
+
+ chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chartB).ToNot(BeEmpty())
+
+ mockRepo := func() *ChartRepository {
+ return &ChartRepository{
+ Client: &mockGetter{
+ response: chartB,
+ },
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ "grafana": {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: "grafana",
+ Version: "6.17.4",
+ },
+ URLs: []string{"https://example.com/chart.tgz"},
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ }
+ }
+
+ var mockCallback GetChartRepositoryCallback = func(url string) (*ChartRepository, error) {
+ if url == "https://grafana.github.io/helm-charts/" {
+ return mockRepo(), nil
+ }
+ return nil, fmt.Errorf("no repository for URL")
+ }
+
+ tests := []struct {
+ name string
+ baseDir string
+ path string
+ chart *helmchart.Chart
+ fromDir bool
+ repositories map[string]*ChartRepository
+ getChartRepositoryCallback GetChartRepositoryCallback
+ wantCollectedDependencies int
+ wantErr string
+ }{
+ {
+ name: "builds dependencies using callback",
+ fromDir: true,
+ baseDir: "testdata/charts",
+ path: "testdata/charts/helmchartwithdeps",
+ getChartRepositoryCallback: mockCallback,
+ wantCollectedDependencies: 2,
+ },
+ {
+ name: "builds dependencies using repositories",
+ fromDir: true,
+ baseDir: "testdata/charts",
+ path: "testdata/charts/helmchartwithdeps",
+ repositories: map[string]*ChartRepository{
+ "https://grafana.github.io/helm-charts/": mockRepo(),
+ },
+ wantCollectedDependencies: 2,
+ },
+ {
+ name: "skips dependency build for packaged chart",
+ path: "testdata/charts/helmchart-0.1.0.tgz",
+ },
+ {
+ name: "attempts to load chart",
+ fromDir: true,
+ path: "testdata",
+ wantErr: "failed to ensure chart has no missing dependencies",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ b := &ChartBuilder{
+ baseDir: tt.baseDir,
+ path: tt.path,
+ chart: tt.chart,
+ repositories: tt.repositories,
+ getChartRepositoryCallback: tt.getChartRepositoryCallback,
+ }
+
+ result := &ChartBuildResult{SourceIsDir: tt.fromDir}
+ err := b.buildDependencies(context.TODO(), result)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(result.CollectedDependencies).To(BeZero())
+ g.Expect(b.chart).To(Equal(tt.chart))
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(result).ToNot(BeNil())
+ g.Expect(result.CollectedDependencies).To(Equal(tt.wantCollectedDependencies))
+ if tt.wantCollectedDependencies > 0 {
+ g.Expect(b.chart).ToNot(Equal(tt.chart))
+ }
+ })
+ }
+}
+
+func TestChartBuilder_mergeValues(t *testing.T) {
+ tests := []struct {
+ name string
+ baseDir string
+ path string
+ isDir bool
+ chart *helmchart.Chart
+ valueFiles []string
+ want map[string]interface{}
+ wantErr string
+ }{
+ {
+ name: "merges chart values",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "a.yaml", Data: []byte("a: b")},
+ {Name: "b.yaml", Data: []byte("a: c")},
+ },
+ },
+ valueFiles: []string{"a.yaml", "b.yaml"},
+ want: map[string]interface{}{
+ "a": "c",
+ },
+ },
+ {
+ name: "chart values merge error",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "b.yaml", Data: []byte("a: c")},
+ },
+ },
+ valueFiles: []string{"a.yaml"},
+ wantErr: "failed to merge chart values",
+ },
+ {
+ name: "merges file values",
+ isDir: true,
+ baseDir: "testdata/charts",
+ path: "helmchart",
+ valueFiles: []string{"helmchart/values-prod.yaml"},
+ want: map[string]interface{}{
+ "replicaCount": float64(2),
+ },
+ },
+ {
+ name: "file values merge error",
+ isDir: true,
+ baseDir: "testdata/charts",
+ path: "helmchart",
+ valueFiles: []string{"invalid.yaml"},
+ wantErr: "failed to merge value files",
+ },
+ {
+ name: "error on chart load failure",
+ baseDir: "testdata/charts",
+ path: "invalid",
+ wantErr: "failed to load chart",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ b := &ChartBuilder{
+ baseDir: tt.baseDir,
+ path: tt.path,
+ chart: tt.chart,
+ valueFiles: tt.valueFiles,
+ }
+
+ result := &ChartBuildResult{SourceIsDir: tt.isDir}
+ err := b.mergeValues(result)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(result.ValuesOverwrite).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(result.ValuesOverwrite).To(Equal(tt.want))
+ })
+ }
+}
+
+func Test_mergeChartValues(t *testing.T) {
+ tests := []struct {
+ name string
+ chart *helmchart.Chart
+ paths []string
+ want map[string]interface{}
+ wantErr string
+ }{
+ {
+ name: "merges values",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "a.yaml", Data: []byte("a: b")},
+ {Name: "b.yaml", Data: []byte("b: c")},
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ },
+ paths: []string{"a.yaml", "b.yaml", "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "uses chart values",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ Values: map[string]interface{}{
+ "a": "b",
+ },
+ },
+ paths: []string{chartutil.ValuesfileName, "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "unmarshal error",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "invalid", Data: []byte("abcd")},
+ },
+ },
+ paths: []string{"invalid"},
+ wantErr: "unmarshaling values from 'invalid' failed",
+ },
+ {
+ name: "error on invalid path",
+ chart: &helmchart.Chart{},
+ paths: []string{"a.yaml"},
+ wantErr: "no values file found at path 'a.yaml'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ got, err := mergeChartValues(tt.chart, tt.paths)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
+
+func Test_mergeFileValues(t *testing.T) {
+ tests := []struct {
+ name string
+ files []*helmchart.File
+ paths []string
+ want map[string]interface{}
+ wantErr string
+ }{
+ {
+ name: "merges values from files",
+ files: []*helmchart.File{
+ {Name: "a.yaml", Data: []byte("a: b")},
+ {Name: "b.yaml", Data: []byte("b: c")},
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ paths: []string{"a.yaml", "b.yaml", "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "illegal traverse",
+ paths: []string{"../../../traversing/illegally/a/p/a/b"},
+ wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'",
+ },
+ {
+ name: "unmarshal error",
+ files: []*helmchart.File{
+ {Name: "invalid", Data: []byte("abcd")},
+ },
+ paths: []string{"invalid"},
+ wantErr: "unmarshaling values from 'invalid' failed",
+ },
+ {
+ name: "error on invalid path",
+ paths: []string{"a.yaml"},
+ wantErr: "no values file found at path '/a.yaml'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ baseDir, err := os.MkdirTemp("", "merge-file-values-*")
+ g.Expect(err).ToNot(HaveOccurred())
+ defer os.RemoveAll(baseDir)
+
+ for _, f := range tt.files {
+ g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed())
+ }
+
+ got, err := mergeFileValues(baseDir, tt.paths)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
+
+func Test_copyFileToPath(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ wantErr string
+ }{
+ {
+ name: "copies input file",
+ in: "testdata/local-index.yaml",
+ },
+ {
+ name: "invalid input file",
+ in: "testdata/invalid.tgz",
+ wantErr: "failed to open file to copy from",
+ },
+ {
+ name: "invalid input directory",
+ in: "testdata/charts",
+ wantErr: "failed to read from source during copy",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ out := tmpFile("copy-0.1.0", ".tgz")
+ defer os.RemoveAll(out)
+ err := copyFileToPath(tt.in, out)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(out).To(BeARegularFile())
+ f1, err := os.ReadFile(tt.in)
+ g.Expect(err).ToNot(HaveOccurred())
+ f2, err := os.ReadFile(out)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(f2).To(Equal(f1))
+ })
+ }
+}
+
+func Test_packageToPath(t *testing.T) {
+ g := NewWithT(t)
+
+ chart, err := loader.Load("testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chart).ToNot(BeNil())
+
+ out := tmpFile("chart-0.1.0", ".tgz")
+ defer os.RemoveAll(out)
+ err = packageToPath(chart, out)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(out).To(BeARegularFile())
+ _, err = loader.Load(out)
+ g.Expect(err).ToNot(HaveOccurred())
+}
+
+func Test_pathIsDir(t *testing.T) {
+ tests := []struct {
+ name string
+ p string
+ want bool
+ }{
+ {name: "directory", p: "testdata/", want: true},
+ {name: "file", p: "testdata/local-index.yaml", want: false},
+ {name: "not found error", p: "testdata/does-not-exist.yaml", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ g.Expect(pathIsDir(tt.p)).To(Equal(tt.want))
+ })
+ }
+}
+
+func tmpFile(prefix, suffix string) string {
+ randBytes := make([]byte, 16)
+ rand.Read(randBytes)
+ return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix)
+}
diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go
index 7afa2a3f6..23d50b96b 100644
--- a/internal/helm/chart_test.go
+++ b/internal/helm/chart_test.go
@@ -17,7 +17,6 @@ limitations under the License.
package helm
import (
- "reflect"
"testing"
. "github.com/onsi/gomega"
@@ -87,33 +86,35 @@ func TestOverwriteChartDefaultValues(t *testing.T) {
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
+ g := NewWithT(t)
+
fixture := tt.chart
ok, err := OverwriteChartDefaultValues(&fixture, tt.data)
- if ok != tt.ok {
- t.Fatalf("should return %v, returned %v", tt.ok, ok)
- }
- if err != nil && !tt.expectErr {
- t.Fatalf("returned unexpected error: %v", err)
- }
- if err == nil && tt.expectErr {
- t.Fatal("expected error")
+ g.Expect(ok).To(Equal(tt.ok))
+
+ if tt.expectErr {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(ok).To(Equal(tt.ok))
+ return
}
- for _, f := range fixture.Raw {
- if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok {
- t.Error("should override values.yaml in Raw field")
+ if tt.ok {
+ for _, f := range fixture.Raw {
+ if f.Name == chartutil.ValuesfileName {
+ g.Expect(f.Data).To(Equal(tt.data))
+ }
}
- }
- for _, f := range fixture.Files {
- if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok {
- t.Error("should override values.yaml in Files field")
+ for _, f := range fixture.Files {
+ if f.Name == chartutil.ValuesfileName {
+ g.Expect(f.Data).To(Equal(tt.data))
+ }
}
}
})
}
}
-func Test_LoadChartMetadataFromDir(t *testing.T) {
+func TestLoadChartMetadataFromDir(t *testing.T) {
tests := []struct {
name string
dir string
diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go
index 19d56c884..043b0e7e3 100644
--- a/internal/helm/dependency_manager.go
+++ b/internal/helm/dependency_manager.go
@@ -33,165 +33,282 @@ import (
"helm.sh/helm/v3/pkg/chart/loader"
)
-// DependencyWithRepository is a container for a Helm chart dependency
-// and its respective repository.
-type DependencyWithRepository struct {
- // Dependency holds the reference to a chart.Chart dependency.
- Dependency *helmchart.Dependency
- // Repository is the ChartRepository the dependency should be
- // available at and can be downloaded from. If there is none,
- // a local ('file://') dependency is assumed.
- Repository *ChartRepository
-}
+// GetChartRepositoryCallback must return a ChartRepository for the URL,
+// or an error describing why it could not be returned.
+type GetChartRepositoryCallback func(url string) (*ChartRepository, error)
-// DependencyManager manages dependencies for a Helm chart.
+// DependencyManager manages dependencies for a Helm chart, downloading
+// only those that are missing from the chart it holds.
type DependencyManager struct {
- // WorkingDir is the chroot path for dependency manager operations,
+ // chart contains the chart.Chart from the path.
+ chart *helmchart.Chart
+
+ // baseDir is the chroot path for dependency manager operations,
// Dependencies that hold a local (relative) path reference are not
// allowed to traverse outside this directory.
- WorkingDir string
- // ChartPath is the path of the Chart relative to the WorkingDir,
- // the combination of the WorkingDir and ChartPath is used to
+ baseDir string
+
+ // path is the path of the chart relative to the baseDir,
+ // the combination of the baseDir and path is used to
// determine the absolute path of a local dependency.
- ChartPath string
- // Chart holds the loaded chart.Chart from the ChartPath.
- Chart *helmchart.Chart
- // Dependencies contains a list of dependencies, and the respective
- // repository the dependency can be found at.
- Dependencies []*DependencyWithRepository
- // Workers is the number of concurrent chart-add operations during
+ path string
+
+ // repositories contains a map of ChartRepository indexed by their
+ // normalized URL. It is used as a lookup table for missing
+ // dependencies.
+ repositories map[string]*ChartRepository
+
+ // getChartRepositoryCallback can be set to an on-demand get
+ // callback which returned result is cached to repositories.
+ getChartRepositoryCallback GetChartRepositoryCallback
+
+ // workers is the number of concurrent chart-add operations during
// Build. Defaults to 1 (non-concurrent).
- Workers int64
+ workers int64
+ // mu contains the lock for chart writes.
mu sync.Mutex
}
-// Build compiles and builds the dependencies of the Chart with the
-// configured number of Workers.
-func (dm *DependencyManager) Build(ctx context.Context) error {
- if len(dm.Dependencies) == 0 {
- return nil
+func NewDependencyManager(chart *helmchart.Chart, baseDir, path string) *DependencyManager {
+ return &DependencyManager{
+ chart: chart,
+ baseDir: baseDir,
+ path: path,
+ }
+}
+
+func (dm *DependencyManager) WithRepositories(r map[string]*ChartRepository) *DependencyManager {
+ dm.repositories = r
+ return dm
+}
+
+func (dm *DependencyManager) WithChartRepositoryCallback(c GetChartRepositoryCallback) *DependencyManager {
+ dm.getChartRepositoryCallback = c
+ return dm
+}
+
+func (dm *DependencyManager) WithWorkers(w int64) *DependencyManager {
+ dm.workers = w
+ return dm
+}
+
+// Build compiles and builds the dependencies of the chart with the
+// configured number of workers.
+func (dm *DependencyManager) Build(ctx context.Context) (int, error) {
+ // Collect dependency metadata
+ var (
+ deps = dm.chart.Dependencies()
+ reqs = dm.chart.Metadata.Dependencies
+ )
+ // Lock file takes precedence
+ if lock := dm.chart.Lock; lock != nil {
+ reqs = lock.Dependencies
+ }
+
+ // Collect missing dependencies
+ missing := collectMissing(deps, reqs)
+ if len(missing) == 0 {
+ return 0, nil
+ }
+
+ // Run the build for the missing dependencies
+ if err := dm.build(ctx, missing); err != nil {
+ return 0, err
}
+ return len(missing), nil
+}
- workers := dm.Workers
+// build (concurrently) adds the given list of deps to the chart with the configured
+// number of workers. It returns the first error, cancelling all other workers.
+func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmchart.Dependency) error {
+ workers := dm.workers
if workers <= 0 {
workers = 1
}
+ // Garbage collect temporary cached ChartRepository indexes
defer func() {
- for _, dep := range dm.Dependencies {
- dep.Repository.UnloadIndex()
+ for _, v := range dm.repositories {
+ v.Unload()
+ _ = v.RemoveCache()
}
}()
group, groupCtx := errgroup.WithContext(ctx)
group.Go(func() error {
sem := semaphore.NewWeighted(workers)
- for _, dep := range dm.Dependencies {
- dep := dep
+ for name, dep := range deps {
+ name, dep := name, dep
if err := sem.Acquire(groupCtx, 1); err != nil {
return err
}
- group.Go(func() error {
+ group.Go(func() (err error) {
defer sem.Release(1)
- if dep.Repository == nil {
- return dm.addLocalDependency(dep)
+ if isLocalDep(dep) {
+ if err = dm.addLocalDependency(dep); err != nil {
+ err = fmt.Errorf("failed to add local dependency '%s': %w", name, err)
+ }
+ return
}
- return dm.addRemoteDependency(dep)
+ if err = dm.addRemoteDependency(dep); err != nil {
+ err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err)
+ }
+ return
})
}
return nil
})
-
return group.Wait()
}
-func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error {
- sLocalChartPath, err := dm.secureLocalChartPath(dpr)
+// addLocalDependency attempts to resolve and add the given local chart.Dependency to the chart.
+func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error {
+ sLocalChartPath, err := dm.secureLocalChartPath(dep)
if err != nil {
return err
}
if _, err := os.Stat(sLocalChartPath); err != nil {
if os.IsNotExist(err) {
- return fmt.Errorf("no chart found at '%s' (reference '%s') for dependency '%s'",
- strings.TrimPrefix(sLocalChartPath, dm.WorkingDir), dpr.Dependency.Repository, dpr.Dependency.Name)
+ return fmt.Errorf("no chart found at '%s' (reference '%s')",
+ strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository)
}
return err
}
- ch, err := loader.Load(sLocalChartPath)
+ constraint, err := semver.NewConstraint(dep.Version)
if err != nil {
+ err = fmt.Errorf("invalid version/constraint format '%s': %w", dep.Version, err)
return err
}
- constraint, err := semver.NewConstraint(dpr.Dependency.Version)
+ ch, err := loader.Load(sLocalChartPath)
if err != nil {
- err := fmt.Errorf("dependency '%s' has an invalid version/constraint format: %w", dpr.Dependency.Name, err)
- return err
+ return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w",
+ strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository, err)
}
- v, err := semver.NewVersion(ch.Metadata.Version)
+ ver, err := semver.NewVersion(ch.Metadata.Version)
if err != nil {
return err
}
- if !constraint.Check(v) {
- err = fmt.Errorf("can't get a valid version for dependency '%s'", dpr.Dependency.Name)
+ if !constraint.Check(ver) {
+ err = fmt.Errorf("can't get a valid version for constraint '%s'", dep.Version)
return err
}
dm.mu.Lock()
- dm.Chart.AddDependency(ch)
+ dm.chart.AddDependency(ch)
dm.mu.Unlock()
-
return nil
}
-func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error {
- if dpr.Repository == nil {
- return fmt.Errorf("no HelmRepository for '%s' dependency", dpr.Dependency.Name)
+// addRemoteDependency attempts to resolve and add the given remote chart.Dependency to the chart.
+func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) error {
+ repo, err := dm.resolveRepository(dep.Repository)
+ if err != nil {
+ return err
}
- if !dpr.Repository.HasIndex() {
- if !dpr.Repository.HasCacheFile() {
- if _, err := dpr.Repository.CacheIndex(); err != nil {
- return err
- }
- }
- if err := dpr.Repository.LoadFromCache(); err != nil {
- return err
- }
+ if err = repo.StrategicallyLoadIndex(); err != nil {
+ return fmt.Errorf("failed to load index for '%s': %w", dep.Name, err)
}
- chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version)
+
+ ver, err := repo.Get(dep.Name, dep.Version)
if err != nil {
return err
}
-
- res, err := dpr.Repository.DownloadChart(chartVer)
+ res, err := repo.DownloadChart(ver)
if err != nil {
- return err
+ return fmt.Errorf("chart download of version '%s' failed: %w", ver.Version, err)
}
-
ch, err := loader.LoadArchive(res)
if err != nil {
- return err
+ return fmt.Errorf("failed to load downloaded archive of version '%s': %w", ver.Version, err)
}
dm.mu.Lock()
- dm.Chart.AddDependency(ch)
+ dm.chart.AddDependency(ch)
dm.mu.Unlock()
return nil
}
-func (dm *DependencyManager) secureLocalChartPath(dep *DependencyWithRepository) (string, error) {
- localUrl, err := url.Parse(dep.Dependency.Repository)
+// resolveRepository first attempts to resolve the url from the repositories, falling back
+// to getChartRepositoryCallback if set. It returns the resolved ChartRepository, or an error.
+func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) {
+ dm.mu.Lock()
+ defer dm.mu.Unlock()
+
+ nUrl := NormalizeChartRepositoryURL(url)
+ if _, ok := dm.repositories[nUrl]; !ok {
+ if dm.getChartRepositoryCallback == nil {
+ err = fmt.Errorf("no chart repository for URL '%s'", nUrl)
+ return
+ }
+ if dm.repositories == nil {
+ dm.repositories = map[string]*ChartRepository{}
+ }
+ if dm.repositories[nUrl], err = dm.getChartRepositoryCallback(nUrl); err != nil {
+ err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err)
+ return
+ }
+ }
+ return dm.repositories[nUrl], nil
+}
+
+// secureLocalChartPath returns the secure absolute path of a local dependency.
+// It does not allow the dependency's path to be outside the scope of baseDir.
+func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (string, error) {
+ localUrl, err := url.Parse(dep.Repository)
if err != nil {
return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err)
}
if localUrl.Scheme != "" && localUrl.Scheme != "file" {
- return "", fmt.Errorf("'%s' is not a local chart reference", dep.Dependency.Repository)
+ return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository)
+ }
+ return securejoin.SecureJoin(dm.baseDir, filepath.Join(dm.path, localUrl.Host, localUrl.Path))
+}
+
+// collectMissing returns a map with reqs that are missing from current,
+// indexed by their alias or name. All dependencies of a chart are present
+// if len of returned value == 0.
+func collectMissing(current []*helmchart.Chart, reqs []*helmchart.Dependency) map[string]*helmchart.Dependency {
+ // If the number of dependencies equals the number of requested
+ // dependencies, there are no missing dependencies
+ if len(current) == len(reqs) {
+ return nil
+ }
+
+ // Build up a map of reqs that are not in current, indexed by their
+ // alias or name
+ var missing map[string]*helmchart.Dependency
+ for _, dep := range reqs {
+ name := dep.Name
+ if dep.Alias != "" {
+ name = dep.Alias
+ }
+ // Exclude existing dependencies
+ found := false
+ for _, existing := range current {
+ if existing.Name() == name {
+ found = true
+ }
+ }
+ if found {
+ continue
+ }
+ if missing == nil {
+ missing = map[string]*helmchart.Dependency{}
+ }
+ missing[name] = dep
}
- return securejoin.SecureJoin(dm.WorkingDir, filepath.Join(dm.ChartPath, localUrl.Host, localUrl.Path))
+ return missing
+}
+
+// isLocalDep returns true if the given chart.Dependency contains a local (file) path reference.
+func isLocalDep(dep *helmchart.Dependency) bool {
+ return dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://")
}
diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go
index a8e6a0480..e51e6b768 100644
--- a/internal/helm/dependency_manager_test.go
+++ b/internal/helm/dependency_manager_test.go
@@ -18,12 +18,16 @@ package helm
import (
"context"
+ "errors"
"fmt"
"os"
- "strings"
+ "path/filepath"
+ "sync"
"testing"
+ . "github.com/onsi/gomega"
helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/repo"
)
@@ -47,177 +51,585 @@ var (
chartVersionV1 = "0.3.0"
)
-func TestBuild_WithEmptyDependencies(t *testing.T) {
- dm := DependencyManager{
- Dependencies: nil,
+func TestDependencyManager_Build(t *testing.T) {
+ tests := []struct {
+ name string
+ baseDir string
+ path string
+ repositories map[string]*ChartRepository
+ getChartRepositoryCallback GetChartRepositoryCallback
+ want int
+ wantChartFunc func(g *WithT, c *helmchart.Chart)
+ wantErr string
+ }{
+ //{
+ // // TODO(hidde): add various happy paths
+ //},
+ //{
+ // // TODO(hidde): test Chart.lock
+ //},
+ {
+ name: "build failure returns error",
+ baseDir: "testdata/charts",
+ path: "helmchartwithdeps",
+ wantErr: "failed to add remote dependency 'grafana': no chart repository for URL",
+ },
+ {
+ name: "no dependencies returns zero",
+ baseDir: "testdata/charts",
+ path: "helmchart",
+ want: 0,
+ },
}
- if err := dm.Build(context.TODO()); err != nil {
- t.Errorf("Build() should return nil")
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path))
+ g.Expect(err).ToNot(HaveOccurred())
+
+ got, err := NewDependencyManager(chart, tt.baseDir, tt.path).
+ WithRepositories(tt.repositories).
+ WithChartRepositoryCallback(tt.getChartRepositoryCallback).
+ Build(context.TODO())
+
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeZero())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ if tt.wantChartFunc != nil {
+ tt.wantChartFunc(g, chart)
+ }
+ })
}
}
-func TestBuild_WithLocalChart(t *testing.T) {
+func TestDependencyManager_build(t *testing.T) {
tests := []struct {
name string
- dep helmchart.Dependency
- wantErr bool
- errMsg string
+ deps map[string]*helmchart.Dependency
+ wantErr string
+ }{
+ {
+ name: "error remote dependency",
+ deps: map[string]*helmchart.Dependency{
+ "example": {Repository: "https://example.com"},
+ },
+ wantErr: "failed to add remote dependency",
+ },
+ {
+ name: "error local dependency",
+ deps: map[string]*helmchart.Dependency{
+ "example": {Repository: "file:///invalid"},
+ },
+ wantErr: "failed to add remote dependency",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ dm := &DependencyManager{
+ baseDir: "testdata/charts",
+ }
+ err := dm.build(context.TODO(), tt.deps)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ })
+ }
+}
+
+func TestDependencyManager_addLocalDependency(t *testing.T) {
+ tests := []struct {
+ name string
+ dep *helmchart.Dependency
+ wantErr string
+ wantFunc func(g *WithT, c *helmchart.Chart)
}{
{
- name: "valid path",
- dep: helmchart.Dependency{
+ name: "local dependency",
+ dep: &helmchart.Dependency{
Name: chartName,
Version: chartVersion,
- Repository: chartLocalRepository,
+ Repository: "file://../helmchart",
+ },
+ wantFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(1))
},
},
{
- name: "valid path",
- dep: helmchart.Dependency{
+ name: "version not matching constraint",
+ dep: &helmchart.Dependency{
Name: chartName,
- Alias: "aliased",
- Version: chartVersion,
- Repository: chartLocalRepository,
+ Version: "0.2.0",
+ Repository: "file://../helmchart",
},
+ wantErr: "can't get a valid version for constraint '0.2.0'",
},
{
- name: "allowed traversing path",
- dep: helmchart.Dependency{
+ name: "invalid local reference",
+ dep: &helmchart.Dependency{
Name: chartName,
- Alias: "aliased",
Version: chartVersion,
- Repository: "file://../../../testdata/charts/helmchartwithdeps/../helmchart",
+ Repository: "file://../../../absolutely/invalid",
},
+ wantErr: "no chart found at 'absolutely/invalid'",
},
{
- name: "invalid path",
- dep: helmchart.Dependency{
+ name: "invalid chart archive",
+ dep: &helmchart.Dependency{
Name: chartName,
Version: chartVersion,
- Repository: "file://../invalid",
+ Repository: "file://../empty.tgz",
},
- wantErr: true,
- errMsg: "no chart found at",
+ wantErr: "failed to load chart from 'empty.tgz'",
},
{
- name: "illegal traversing path",
- dep: helmchart.Dependency{
+ name: "invalid constraint",
+ dep: &helmchart.Dependency{
Name: chartName,
- Version: chartVersion,
- Repository: "file://../../../../../controllers/testdata/charts/helmchart",
+ Version: "invalid",
+ Repository: "file://../helmchart",
+ },
+ wantErr: "invalid version/constraint format 'invalid'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ dm := &DependencyManager{
+ chart: &helmchart.Chart{},
+ baseDir: "testdata/charts/",
+ path: "helmchartwithdeps",
+ }
+
+ err := dm.addLocalDependency(tt.dep)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ return
+ }
+ g.Expect(err).ToNot(HaveOccurred())
+ })
+ }
+}
+
+func TestDependencyManager_addRemoteDependency(t *testing.T) {
+ g := NewWithT(t)
+
+ chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chartB).ToNot(BeEmpty())
+
+ tests := []struct {
+ name string
+ repositories map[string]*ChartRepository
+ dep *helmchart.Dependency
+ wantFunc func(g *WithT, c *helmchart.Chart)
+ wantErr string
+ }{
+ {
+ name: "adds remote dependency",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ Client: &mockGetter{
+ response: chartB,
+ },
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ chartName: {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ Version: chartVersion,
+ },
+ URLs: []string{"https://example.com/foo.tgz"},
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
+ Name: chartName,
+ Repository: "https://example.com",
+ },
+ wantFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(1))
+ },
+ },
+ {
+ name: "resolve repository error",
+ repositories: map[string]*ChartRepository{},
+ dep: &helmchart.Dependency{
+ Repository: "https://example.com",
+ },
+ wantErr: "no chart repository for URL",
+ },
+ {
+ name: "strategic load error",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ CachePath: "/invalid/cache/path/foo",
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
+ Repository: "https://example.com",
+ },
+ wantErr: "failed to strategically load index",
+ },
+ {
+ name: "repository get error",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ Index: &repo.IndexFile{},
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
+ Repository: "https://example.com",
},
- wantErr: true,
- errMsg: "no chart found at",
+ wantErr: "no chart name found",
},
{
- name: "invalid version constraint format",
- dep: helmchart.Dependency{
+ name: "repository version constraint error",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ chartName: {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ Version: "0.1.0",
+ },
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
Name: chartName,
- Version: "!2.0",
- Repository: chartLocalRepository,
+ Version: "0.2.0",
+ Repository: "https://example.com",
},
- wantErr: true,
- errMsg: "has an invalid version/constraint format",
+ wantErr: fmt.Sprintf("no '%s' chart with version matching '0.2.0' found", chartName),
},
{
- name: "invalid version",
- dep: helmchart.Dependency{
+ name: "repository chart download error",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ chartName: {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ Version: chartVersion,
+ },
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
Name: chartName,
Version: chartVersion,
- Repository: chartLocalRepository,
+ Repository: "https://example.com",
},
- wantErr: true,
- errMsg: "can't get a valid version for dependency",
+ wantErr: "chart download of version '0.1.0' failed",
+ },
+ {
+ name: "chart load error",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {
+ Client: &mockGetter{},
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ chartName: {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ Version: chartVersion,
+ },
+ URLs: []string{"https://example.com/foo.tgz"},
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ },
+ },
+ dep: &helmchart.Dependency{
+ Name: chartName,
+ Version: chartVersion,
+ Repository: "https://example.com",
+ },
+ wantErr: "failed to load downloaded archive of version '0.1.0'",
},
}
-
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- c := chartFixture
- dm := DependencyManager{
- WorkingDir: "./",
- ChartPath: "testdata/charts/helmchart",
- Chart: &c,
- Dependencies: []*DependencyWithRepository{
- {
- Dependency: &tt.dep,
- Repository: nil,
- },
- },
- }
+ g := NewWithT(t)
- err := dm.Build(context.TODO())
- deps := dm.Chart.Dependencies()
-
- if (err != nil) && tt.wantErr {
- if !strings.Contains(err.Error(), tt.errMsg) {
- t.Errorf("Build() expected to return error: %s, got: %s", tt.errMsg, err)
- }
- if len(deps) > 0 {
- t.Fatalf("chart expected to have no dependencies registered")
- }
- return
- } else if err != nil {
- t.Errorf("Build() not expected to return an error: %s", err)
+ dm := &DependencyManager{
+ chart: &helmchart.Chart{},
+ repositories: tt.repositories,
+ }
+ err := dm.addRemoteDependency(tt.dep)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
return
}
+ g.Expect(err).ToNot(HaveOccurred())
+ if tt.wantFunc != nil {
+ tt.wantFunc(g, dm.chart)
+ }
+ })
+ }
+}
+
+func TestDependencyManager_resolveRepository(t *testing.T) {
+ tests := []struct {
+ name string
+ repositories map[string]*ChartRepository
+ getChartRepositoryCallback GetChartRepositoryCallback
+ url string
+ want *ChartRepository
+ wantRepositories map[string]*ChartRepository
+ wantErr string
+ }{
+ {
+ name: "resolves from repositories index",
+ url: "https://example.com",
+ repositories: map[string]*ChartRepository{
+ "https://example.com/": {URL: "https://example.com"},
+ },
+ want: &ChartRepository{URL: "https://example.com"},
+ },
+ {
+ name: "resolves from callback",
+ url: "https://example.com",
+ getChartRepositoryCallback: func(url string) (*ChartRepository, error) {
+ return &ChartRepository{URL: "https://example.com"}, nil
+ },
+ want: &ChartRepository{URL: "https://example.com"},
+ wantRepositories: map[string]*ChartRepository{
+ "https://example.com/": {URL: "https://example.com"},
+ },
+ },
+ {
+ name: "error from callback",
+ url: "https://example.com",
+ getChartRepositoryCallback: func(url string) (*ChartRepository, error) {
+ return nil, errors.New("a very unique error")
+ },
+ wantErr: "a very unique error",
+ wantRepositories: map[string]*ChartRepository{},
+ },
+ {
+ name: "error on not found",
+ url: "https://example.com",
+ wantErr: "no chart repository for URL",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
- if len(deps) == 0 {
- t.Fatalf("chart expected to have at least one dependency registered")
+ dm := &DependencyManager{
+ repositories: tt.repositories,
+ getChartRepositoryCallback: tt.getChartRepositoryCallback,
}
- if deps[0].Metadata.Name != chartName {
- t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name)
+
+ got, err := dm.resolveRepository(tt.url)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
}
- if deps[0].Metadata.Version != chartVersion {
- t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version)
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ if tt.wantRepositories != nil {
+ g.Expect(dm.repositories).To(Equal(tt.wantRepositories))
}
})
}
}
-func TestBuild_WithRemoteChart(t *testing.T) {
- chart := chartFixture
- b, err := os.ReadFile(helmPackageFile)
- if err != nil {
- t.Fatal(err)
- }
- i := repo.NewIndexFile()
- i.MustAdd(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890")
- mg := mockGetter{response: b}
- cr := newChartRepository()
- cr.URL = remoteDepFixture.Repository
- cr.Index = i
- cr.Client = &mg
- dm := DependencyManager{
- Chart: &chart,
- Dependencies: []*DependencyWithRepository{
- {
- Dependency: &remoteDepFixture,
- Repository: cr,
+func TestDependencyManager_secureLocalChartPath(t *testing.T) {
+ tests := []struct {
+ name string
+ baseDir string
+ path string
+ dep *helmchart.Dependency
+ want string
+ wantErr string
+ }{
+ {
+ name: "secure local file path",
+ baseDir: "/tmp/workdir",
+ path: "/chart",
+ dep: &helmchart.Dependency{
+ Repository: "../dep",
+ },
+ want: "/tmp/workdir/dep",
+ },
+ {
+ name: "insecure local file path",
+ baseDir: "/tmp/workdir",
+ path: "/",
+ dep: &helmchart.Dependency{
+ Repository: "/../../dep",
},
+ want: "/tmp/workdir/dep",
+ },
+ {
+ name: "URL parse error",
+ dep: &helmchart.Dependency{
+ Repository: ": //example.com",
+ },
+ wantErr: "missing protocol scheme",
+ },
+ {
+ name: "error on URL scheme other than file",
+ dep: &helmchart.Dependency{
+ Repository: "https://example.com",
+ },
+ wantErr: "not a local chart reference",
},
}
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
- if err := dm.Build(context.TODO()); err != nil {
- t.Errorf("Build() expected to not return error: %s", err)
+ dm := &DependencyManager{
+ baseDir: tt.baseDir,
+ path: tt.path,
+ }
+ got, err := dm.secureLocalChartPath(tt.dep)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ return
+ }
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).ToNot(BeEmpty())
+ g.Expect(got).To(Equal(tt.want))
+ })
}
+}
- deps := dm.Chart.Dependencies()
- if len(deps) != 1 {
- t.Fatalf("chart expected to have one dependency registered")
- }
- if deps[0].Metadata.Name != chartName {
- t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name)
+func Test_collectMissing(t *testing.T) {
+ tests := []struct {
+ name string
+ current []*helmchart.Chart
+ reqs []*helmchart.Dependency
+ want map[string]*helmchart.Dependency
+ }{
+ {
+ name: "one missing",
+ current: []*helmchart.Chart{},
+ reqs: []*helmchart.Dependency{
+ {Name: chartName},
+ },
+ want: map[string]*helmchart.Dependency{
+ chartName: {Name: chartName},
+ },
+ },
+ {
+ name: "alias missing",
+ current: []*helmchart.Chart{
+ {
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ },
+ },
+ },
+ reqs: []*helmchart.Dependency{
+ {Name: chartName},
+ {Name: chartName, Alias: chartName + "-alias"},
+ },
+ want: map[string]*helmchart.Dependency{
+ chartName + "-alias": {Name: chartName, Alias: chartName + "-alias"},
+ },
+ },
+ {
+ name: "all current",
+ current: []*helmchart.Chart{
+ {
+ Metadata: &helmchart.Metadata{
+ Name: chartName,
+ },
+ },
+ },
+ reqs: []*helmchart.Dependency{
+ {Name: chartName},
+ },
+ want: nil,
+ },
+ {
+ name: "nil",
+ current: nil,
+ reqs: nil,
+ want: nil,
+ },
}
- if deps[0].Metadata.Version != chartVersion {
- t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version)
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+ g.Expect(collectMissing(tt.current, tt.reqs)).To(Equal(tt.want))
+ })
+ })
}
+}
- // When repo is not set
- dm.Dependencies[0].Repository = nil
- if err := dm.Build(context.TODO()); err == nil {
- t.Errorf("Build() expected to return error")
- } else if !strings.Contains(err.Error(), "is not a local chart reference") {
- t.Errorf("Build() expected to return different error, got: %s", err)
+func Test_isLocalDep(t *testing.T) {
+ tests := []struct {
+ name string
+ dep *helmchart.Dependency
+ want bool
+ }{
+ {
+ name: "file protocol",
+ dep: &helmchart.Dependency{Repository: "file:///some/path"},
+ want: true,
+ },
+ {
+ name: "empty",
+ dep: &helmchart.Dependency{Repository: ""},
+ want: true,
+ },
+ {
+ name: "https url",
+ dep: &helmchart.Dependency{Repository: "https://example.com"},
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+ g.Expect(isLocalDep(tt.dep)).To(Equal(tt.want))
+ })
}
}
diff --git a/internal/helm/repository.go b/internal/helm/repository.go
index c57df111f..e2446f944 100644
--- a/internal/helm/repository.go
+++ b/internal/helm/repository.go
@@ -54,6 +54,9 @@ type ChartRepository struct {
Options []getter.Option
// CachePath is the path of a cached index.yaml for read-only operations.
CachePath string
+ // Cached indicates if the ChartRepository index.yaml has been cached
+ // to CachePath.
+ Cached bool
// Index contains a loaded chart repository index if not nil.
Index *repo.IndexFile
// Checksum contains the SHA256 checksum of the loaded chart repository
@@ -68,7 +71,6 @@ type ChartRepository struct {
// repository URL scheme. It returns an error on URL parsing failures,
// or if there is no getter available for the scheme.
func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) {
- r := newChartRepository()
u, err := url.Parse(repositoryURL)
if err != nil {
return nil, err
@@ -77,6 +79,8 @@ func NewChartRepository(repositoryURL, cachePath string, providers getter.Provid
if err != nil {
return nil, err
}
+
+ r := newChartRepository()
r.URL = repositoryURL
r.CachePath = cachePath
r.Client = c
@@ -238,7 +242,7 @@ func (r *ChartRepository) LoadFromFile(path string) error {
}
// CacheIndex attempts to write the index from the remote into a new temporary file
-// using DownloadIndex, and sets CachePath.
+// using DownloadIndex, and sets CachePath and Cached.
// It returns the SHA256 checksum of the downloaded index bytes, or an error.
// The caller is expected to handle the garbage collection of CachePath, and to
// load the Index separately using LoadFromCache if required.
@@ -262,19 +266,40 @@ func (r *ChartRepository) CacheIndex() (string, error) {
r.Lock()
r.CachePath = f.Name()
+ r.Cached = true
r.Unlock()
return hex.EncodeToString(h.Sum(nil)), nil
}
+// StrategicallyLoadIndex lazy-loads the Index from CachePath using
+// LoadFromCache if it does not HasIndex.
+// If it not HasCacheFile, a cache attempt is made using CacheIndex
+// before continuing to load.
+// It returns a boolean indicating if it cached the index before
+// loading, or an error.
+func (r *ChartRepository) StrategicallyLoadIndex() (err error) {
+ if r.HasIndex() {
+ return
+ }
+ if !r.HasCacheFile() {
+ if _, err = r.CacheIndex(); err != nil {
+ err = fmt.Errorf("failed to strategically load index: %w", err)
+ return
+ }
+ }
+ if err = r.LoadFromCache(); err != nil {
+ err = fmt.Errorf("failed to strategically load index: %w", err)
+ return
+ }
+ return
+}
+
// LoadFromCache attempts to load the Index from the configured CachePath.
// It returns an error if no CachePath is set, or if the load failed.
func (r *ChartRepository) LoadFromCache() error {
- r.RLock()
if cachePath := r.CachePath; cachePath != "" {
- r.RUnlock()
return r.LoadFromFile(cachePath)
}
- r.RUnlock()
return fmt.Errorf("no cache path set")
}
@@ -314,11 +339,34 @@ func (r *ChartRepository) HasCacheFile() bool {
return r.CachePath != ""
}
-// UnloadIndex sets the Index to nil.
-func (r *ChartRepository) UnloadIndex() {
- if r != nil {
- r.Lock()
- r.Index = nil
- r.Unlock()
+// Unload can be used to signal the Go garbage collector the Index can
+// be freed from memory if the ChartRepository object is expected to
+// continue to exist in the stack for some time.
+func (r *ChartRepository) Unload() {
+ if r == nil {
+ return
}
+
+ r.Lock()
+ defer r.Unlock()
+ r.Index = nil
+}
+
+// RemoveCache removes the CachePath if Cached.
+func (r *ChartRepository) RemoveCache() error {
+ if r == nil {
+ return nil
+ }
+
+ r.Lock()
+ defer r.Unlock()
+
+ if r.Cached {
+ if err := os.Remove(r.CachePath); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ r.CachePath = ""
+ r.Cached = false
+ }
+ return nil
}
diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go
index 95ccc7b80..0d2077dfd 100644
--- a/internal/helm/repository_test.go
+++ b/internal/helm/repository_test.go
@@ -47,7 +47,8 @@ type mockGetter struct {
func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) {
g.requestedURL = url
- return bytes.NewBuffer(g.response), nil
+ r := g.response
+ return bytes.NewBuffer(r), nil
}
func TestNewChartRepository(t *testing.T) {
@@ -402,7 +403,7 @@ func TestChartRepository_CacheIndex(t *testing.T) {
g.Expect(sum).To(BeEquivalentTo(expectSum))
}
-func TestChartRepository_LoadIndexFromCache(t *testing.T) {
+func TestChartRepository_LoadFromCache(t *testing.T) {
tests := []struct {
name string
cachePath string
@@ -458,7 +459,7 @@ func TestChartRepository_UnloadIndex(t *testing.T) {
r := newChartRepository()
g.Expect(r.HasIndex()).To(BeFalse())
r.Index = repo.NewIndexFile()
- r.UnloadIndex()
+ r.Unload()
g.Expect(r.Index).To(BeNil())
}
diff --git a/internal/helm/testdata/charts/helmchart/values-prod.yaml b/internal/helm/testdata/charts/helmchart/values-prod.yaml
new file mode 100644
index 000000000..5ef7832ca
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchart/values-prod.yaml
@@ -0,0 +1 @@
+replicaCount: 2
diff --git a/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock
new file mode 100644
index 000000000..83401ac65
--- /dev/null
+++ b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock
@@ -0,0 +1,12 @@
+dependencies:
+- name: helmchart
+ repository: file://../helmchart
+ version: 0.1.0
+- name: helmchart
+ repository: file://../helmchart
+ version: 0.1.0
+- name: grafana
+ repository: https://grafana.github.io/helm-charts
+ version: 6.17.4
+digest: sha256:1e41c97e27347f433ff0212bf52c344bc82dd435f70129d15e96cd2c8fcc32bb
+generated: "2021-11-02T01:25:59.624290788+01:00"
From d23bcbb5db2ae441000b43b4100ef678a6068b9d Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Fri, 5 Nov 2021 13:20:25 +0100
Subject: [PATCH 0225/1397] controllers: wire ChartRepository in reconciler
This wires the `ChartRepository` changes into the reconciler to ensure
it works.
Signed-off-by: Hidde Beydals
---
controllers/helmrepository_controller.go | 57 ++++++++++++++----------
1 file changed, 33 insertions(+), 24 deletions(-)
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index b7f8cd516..d7fb57e58 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -17,12 +17,15 @@ limitations under the License.
package controllers
import (
- "bytes"
"context"
"fmt"
"net/url"
"time"
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/runtime/metrics"
+ "github.com/fluxcd/pkg/runtime/predicates"
"github.com/go-logr/logr"
"helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
@@ -37,12 +40,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/predicate"
- "sigs.k8s.io/yaml"
-
- "github.com/fluxcd/pkg/apis/meta"
- "github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
- "github.com/fluxcd/pkg/runtime/predicates"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"github.com/fluxcd/source-controller/internal/helm"
@@ -198,7 +195,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
clientOpts = append(clientOpts, opts...)
}
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
+ chartRepo, err := helm.NewChartRepository(repository.Spec.URL, "", r.Getters, clientOpts)
if err != nil {
switch err.(type) {
case *url.Error:
@@ -207,22 +204,21 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
}
}
- if err := chartRepo.DownloadIndex(); err != nil {
+ revision, err := chartRepo.CacheIndex()
+ if err != nil {
err = fmt.Errorf("failed to download repository index: %w", err)
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
}
+ defer chartRepo.RemoveCache()
- indexBytes, err := yaml.Marshal(&chartRepo.Index)
- if err != nil {
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- hash := r.Storage.Checksum(bytes.NewReader(indexBytes))
artifact := r.Storage.NewArtifactFor(repository.Kind,
repository.ObjectMeta.GetObjectMeta(),
- hash,
- fmt.Sprintf("index-%s.yaml", hash))
- // return early on unchanged index
- if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) {
+ revision,
+ fmt.Sprintf("index-%s.yaml", revision))
+
+ // Return early on unchanged index
+ if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) &&
+ repository.GetArtifact().HasRevision(artifact.Revision) {
if artifact.URL != repository.GetArtifact().URL {
r.Storage.SetArtifactURL(repository.GetArtifact())
repository.Status.URL = r.Storage.SetHostname(repository.Status.URL)
@@ -230,14 +226,20 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
return repository, nil
}
- // create artifact dir
+ // Load the cached repository index to ensure it passes validation
+ if err := chartRepo.LoadFromCache(); err != nil {
+ return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ }
+ defer chartRepo.Unload()
+
+ // Create artifact dir
err = r.Storage.MkdirAll(artifact)
if err != nil {
err = fmt.Errorf("unable to create repository index directory: %w", err)
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- // acquire lock
+ // Acquire lock
unlock, err := r.Storage.Lock(artifact)
if err != nil {
err = fmt.Errorf("unable to acquire lock: %w", err)
@@ -245,13 +247,20 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
}
defer unlock()
- // save artifact to storage
- if err := r.Storage.AtomicWriteFile(&artifact, bytes.NewReader(indexBytes), 0644); err != nil {
- err = fmt.Errorf("unable to write repository index file: %w", err)
+ // Save artifact to storage
+ storageTarget := r.Storage.LocalPath(artifact)
+ if storageTarget == "" {
+ err := fmt.Errorf("failed to calcalute local storage path to store artifact to")
+ return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ }
+ if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil {
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
+ // TODO(hidde): it would be better to make the Storage deal with this
+ artifact.Checksum = chartRepo.Checksum
+ artifact.LastUpdateTime = metav1.Now()
- // update index symlink
+ // Update index symlink
indexURL, err := r.Storage.Symlink(artifact, "index.yaml")
if err != nil {
err = fmt.Errorf("storage error: %w", err)
From 52459c899da85724ec3b2ef155ce871d0e484756 Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Sat, 13 Nov 2021 00:16:59 +0100
Subject: [PATCH 0226/1397] internal/helm: make ChartBuilder an interface
This commit refactors the `ChartBuilder` that used to be a do-it-all
struct into an interace with two implementations:
- `LocalChartBuilder`: to build charts from a source on the local
filesystem, either from a directory or from a packaged chart.
- `RemoteChartBuilder`: to build charts from a remote Helm repository
index.
The new logic within the builders validates the size of the Helm size
it works with based on the `Max*Size` global variables in the internal
`helm` package, to address the recommendation from the security audit.
In addition, changes `ClientOptionsFromSecret` takes now a directory
argument which temporary files are placed in, making it easier to
perform a garbage collection of the whole directory at the end of a
reconcile run.
Signed-off-by: Hidde Beydals
---
internal/helm/chart.go | 51 +-
internal/helm/chart_builder.go | 416 ++++------------
internal/helm/chart_builder_local.go | 190 +++++++
internal/helm/chart_builder_local_test.go | 137 ++++++
internal/helm/chart_builder_remote.go | 199 ++++++++
internal/helm/chart_builder_remote_test.go | 118 +++++
internal/helm/chart_builder_test.go | 543 +--------------------
internal/helm/chart_test.go | 20 +-
internal/helm/dependency_manager.go | 175 ++++---
internal/helm/dependency_manager_test.go | 46 +-
internal/helm/getter.go | 82 ++--
internal/helm/getter_test.go | 21 +-
internal/helm/helm.go | 29 ++
internal/helm/repository.go | 12 +-
internal/helm/repository_test.go | 2 +-
15 files changed, 1023 insertions(+), 1018 deletions(-)
create mode 100644 internal/helm/chart_builder_local.go
create mode 100644 internal/helm/chart_builder_local_test.go
create mode 100644 internal/helm/chart_builder_remote.go
create mode 100644 internal/helm/chart_builder_remote_test.go
create mode 100644 internal/helm/helm.go
diff --git a/internal/helm/chart.go b/internal/helm/chart.go
index dcc868c1d..4f89cab61 100644
--- a/internal/helm/chart.go
+++ b/internal/helm/chart.go
@@ -19,6 +19,7 @@ package helm
import (
"archive/tar"
"bufio"
+ "bytes"
"compress/gzip"
"errors"
"fmt"
@@ -35,30 +36,35 @@ import (
)
// OverwriteChartDefaultValues overwrites the chart default values file with the given data.
-func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, error) {
- // Read override values file data
- values, err := chartutil.ReadValues(data)
- if err != nil {
- return false, fmt.Errorf("failed to parse provided override values file data")
+func OverwriteChartDefaultValues(chart *helmchart.Chart, vals chartutil.Values) (bool, error) {
+ if vals == nil {
+ return false, nil
+ }
+
+ var bVals bytes.Buffer
+ if len(vals) > 0 {
+ if err := vals.Encode(&bVals); err != nil {
+ return false, err
+ }
}
// Replace current values file in Raw field
for _, f := range chart.Raw {
if f.Name == chartutil.ValuesfileName {
// Do nothing if contents are equal
- if reflect.DeepEqual(f.Data, data) {
+ if reflect.DeepEqual(f.Data, bVals.Bytes()) {
return false, nil
}
// Replace in Files field
for _, f := range chart.Files {
if f.Name == chartutil.ValuesfileName {
- f.Data = data
+ f.Data = bVals.Bytes()
}
}
- f.Data = data
- chart.Values = values
+ f.Data = bVals.Bytes()
+ chart.Values = vals.AsMap()
return true, nil
}
}
@@ -100,7 +106,21 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) {
m.APIVersion = helmchart.APIVersionV1
}
- b, err = os.ReadFile(filepath.Join(dir, "requirements.yaml"))
+ fp := filepath.Join(dir, "requirements.yaml")
+ stat, err := os.Stat(fp)
+ if (err != nil && !errors.Is(err, os.ErrNotExist)) || stat != nil {
+ if err != nil {
+ return nil, err
+ }
+ if stat.IsDir() {
+ return nil, fmt.Errorf("'%s' is a directory", stat.Name())
+ }
+ if stat.Size() > MaxChartFileSize {
+ return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), MaxChartFileSize)
+ }
+ }
+
+ b, err = os.ReadFile(fp)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
@@ -115,6 +135,17 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) {
// LoadChartMetadataFromArchive loads the chart.Metadata from the "Chart.yaml" file in the archive at the given path.
// It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format.
func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) {
+ stat, err := os.Stat(archive)
+ if err != nil || stat.IsDir() {
+ if err == nil {
+ err = fmt.Errorf("'%s' is a directory", stat.Name())
+ }
+ return nil, err
+ }
+ if stat.Size() > MaxChartSize {
+ return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), MaxChartSize)
+ }
+
f, err := os.Open(archive)
if err != nil {
return nil, err
diff --git a/internal/helm/chart_builder.go b/internal/helm/chart_builder.go
index 7b90cba81..4177983c6 100644
--- a/internal/helm/chart_builder.go
+++ b/internal/helm/chart_builder.go
@@ -22,338 +22,145 @@ import (
"os"
"path/filepath"
"strings"
- "sync"
- securejoin "github.com/cyphar/filepath-securejoin"
"github.com/fluxcd/source-controller/internal/fs"
helmchart "helm.sh/helm/v3/pkg/chart"
- "helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
- "sigs.k8s.io/yaml"
-
- "github.com/fluxcd/pkg/runtime/transform"
)
-// ChartBuilder aims to efficiently build a Helm chart from a directory or packaged chart.
-// It avoids or delays loading the chart into memory in full, working with chart.Metadata
-// as much as it can, and returns early (by copying over the already packaged source chart)
-// if no modifications were made during the build process.
-type ChartBuilder struct {
- // baseDir is the chroot for the chart builder when path isDir.
- // It must be (a higher) relative to path. File references (during e.g.
- // value file merge operations) are not allowed to traverse out of it.
- baseDir string
-
- // path is the file or directory path to a chart source.
- path string
-
- // chart holds a (partly) loaded chart.Chart, it contains at least the
- // chart.Metadata, which may expand to the full chart.Chart if required
- // for Build operations.
- chart *helmchart.Chart
-
- // valueFiles holds a list of path references of valueFiles that should be
- // merged and packaged as a single "values.yaml" during Build.
- valueFiles []string
-
- // repositories holds an index of repository URLs and their ChartRepository.
- // They are used to configure a DependencyManager for missing chart dependencies
- // if isDir is true.
- repositories map[string]*ChartRepository
-
- // getChartRepositoryCallback is used to configure a DependencyManager for
- // missing chart dependencies if isDir is true.
- getChartRepositoryCallback GetChartRepositoryCallback
-
- mu sync.Mutex
-}
-
-// NewChartBuilder constructs a new ChartBuilder for the given chart path.
-// It returns an error if no chart.Metadata can be loaded from the path.
-func NewChartBuilder(path string) (*ChartBuilder, error) {
- metadata, err := LoadChartMetadata(path)
- if err != nil {
- return nil, fmt.Errorf("could not create new chart builder: %w", err)
- }
- return &ChartBuilder{
- path: path,
- chart: &helmchart.Chart{
- Metadata: metadata,
- },
- }, nil
+// ChartReference holds information to locate a chart.
+type ChartReference interface {
+ // Validate returns an error if the ChartReference is not valid according
+ // to the spec of the interface implementation.
+ Validate() error
}
-// WithBaseDir configures the base dir on the ChartBuilder.
-func (b *ChartBuilder) WithBaseDir(p string) *ChartBuilder {
- b.mu.Lock()
- b.baseDir = p
- b.mu.Unlock()
- return b
-}
-
-// WithValueFiles appends the given paths to the ChartBuilder's valueFiles.
-func (b *ChartBuilder) WithValueFiles(path ...string) *ChartBuilder {
- b.mu.Lock()
- b.valueFiles = append(b.valueFiles, path...)
- b.mu.Unlock()
- return b
-}
-
-// WithChartRepository indexes the given ChartRepository by the NormalizeChartRepositoryURL,
-// used to configure the DependencyManager if the chart is not packaged.
-func (b *ChartBuilder) WithChartRepository(url string, index *ChartRepository) *ChartBuilder {
- b.mu.Lock()
- b.repositories[NormalizeChartRepositoryURL(url)] = index
- b.mu.Unlock()
- return b
-}
-
-// WithChartRepositoryCallback configures the GetChartRepositoryCallback used by the
-// DependencyManager if the chart is not packaged.
-func (b *ChartBuilder) WithChartRepositoryCallback(c GetChartRepositoryCallback) *ChartBuilder {
- b.mu.Lock()
- b.getChartRepositoryCallback = c
- b.mu.Unlock()
- return b
-}
-
-// ChartBuildResult contains the ChartBuilder result, including build specific
-// information about the chart.
-type ChartBuildResult struct {
- // SourceIsDir indicates if the chart was build from a directory.
- SourceIsDir bool
- // Path contains the absolute path to the packaged chart.
+// LocalChartReference contains sufficient information to locate a chart on the
+// local filesystem.
+type LocalChartReference struct {
+ // BaseDir used as chroot during build operations.
+ // File references are not allowed to traverse outside it.
+ BaseDir string
+ // Path of the chart on the local filesystem.
Path string
- // ValuesOverwrite holds a structured map with the merged values used
- // to overwrite chart default "values.yaml".
- ValuesOverwrite map[string]interface{}
- // CollectedDependencies contains the number of missing local and remote
- // dependencies that were collected by the DependencyManager before building
- // the chart.
- CollectedDependencies int
- // Packaged indicates if the ChartBuilder has packaged the chart.
- // This can for example be false if SourceIsDir is false and ValuesOverwrite
- // is nil, which makes the ChartBuilder copy the chart source to Path without
- // making any modifications.
- Packaged bool
}
-// String returns the Path of the ChartBuildResult.
-func (b *ChartBuildResult) String() string {
- if b != nil {
- return b.Path
+// Validate returns an error if the LocalChartReference does not have
+// a Path set.
+func (r LocalChartReference) Validate() error {
+ if r.Path == "" {
+ return fmt.Errorf("no path set for local chart reference")
}
- return ""
+ return nil
}
-// Build attempts to build a new chart using ChartBuilder configuration,
-// writing it to the provided path.
-// It returns a ChartBuildResult containing all information about the resulting chart,
-// or an error.
-func (b *ChartBuilder) Build(ctx context.Context, p string) (_ *ChartBuildResult, err error) {
- b.mu.Lock()
- defer b.mu.Unlock()
-
- if b.chart == nil {
- err = fmt.Errorf("chart build failed: no initial chart (metadata) loaded")
- return
- }
- if b.path == "" {
- err = fmt.Errorf("chart build failed: no path set")
- return
- }
-
- result := &ChartBuildResult{}
- result.SourceIsDir = pathIsDir(b.path)
- result.Path = p
-
- // Merge chart values
- if err = b.mergeValues(result); err != nil {
- err = fmt.Errorf("chart build failed: %w", err)
- return
- }
-
- // Ensure chart has all dependencies
- if err = b.buildDependencies(ctx, result); err != nil {
- err = fmt.Errorf("chart build failed: %w", err)
- return
- }
-
- // Package (or copy) chart
- if err = b.packageChart(result); err != nil {
- err = fmt.Errorf("chart package failed: %w", err)
- return
- }
- return result, nil
+// RemoteChartReference contains sufficient information to look up a chart in
+// a ChartRepository.
+type RemoteChartReference struct {
+ // Name of the chart.
+ Name string
+ // Version of the chart.
+ // Can be a Semver range, or empty for latest.
+ Version string
}
-// load lazy-loads chart.Chart into chart from the set path, it replaces any previously set
-// chart.Metadata shim.
-func (b *ChartBuilder) load() (err error) {
- if b.chart == nil || len(b.chart.Files) <= 0 {
- if b.path == "" {
- return fmt.Errorf("failed to load chart: path not set")
- }
- chart, err := loader.Load(b.path)
- if err != nil {
- return fmt.Errorf("failed to load chart: %w", err)
- }
- b.chart = chart
+// Validate returns an error if the RemoteChartReference does not have
+// a Name set.
+func (r RemoteChartReference) Validate() error {
+ if r.Name == "" {
+ return fmt.Errorf("no name set for remote chart reference")
}
- return
+ return nil
}
-// buildDependencies builds the missing dependencies for a chart from a directory.
-// Using the chart using a NewDependencyManager and the configured repositories
-// and getChartRepositoryCallback
-// It returns the number of dependencies it collected, or an error.
-func (b *ChartBuilder) buildDependencies(ctx context.Context, result *ChartBuildResult) (err error) {
- if !result.SourceIsDir {
- return
- }
-
- if err = b.load(); err != nil {
- err = fmt.Errorf("failed to ensure chart has no missing dependencies: %w", err)
- return
+// ChartBuilder is capable of building a (specific) ChartReference.
+type ChartBuilder interface {
+ // Build builds and packages a Helm chart with the given ChartReference
+ // and BuildOptions and writes it to p. It returns the ChartBuild result,
+ // or an error. It may return an error for unsupported ChartReference
+ // implementations.
+ Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error)
+}
+
+// BuildOptions provides a list of options for ChartBuilder.Build.
+type BuildOptions struct {
+ // VersionMetadata can be set to SemVer build metadata as defined in
+ // the spec, and is included during packaging.
+ // Ref: https://semver.org/#spec-item-10
+ VersionMetadata string
+ // ValueFiles can be set to a list of relative paths, used to compose
+ // and overwrite an alternative default "values.yaml" for the chart.
+ ValueFiles []string
+ // CachedChart can be set to the absolute path of a chart stored on
+ // the local filesystem, and is used for simple validation by metadata
+ // comparisons.
+ CachedChart string
+ // Force can be set to force the build of the chart, for example
+ // because the list of ValueFiles has changed.
+ Force bool
+}
+
+// GetValueFiles returns BuildOptions.ValueFiles, except if it equals
+// "values.yaml", which returns nil.
+func (o BuildOptions) GetValueFiles() []string {
+ if len(o.ValueFiles) == 1 && filepath.Clean(o.ValueFiles[0]) == filepath.Clean(chartutil.ValuesfileName) {
+ return nil
}
-
- dm := NewDependencyManager(b.chart, b.baseDir, strings.TrimLeft(b.path, b.baseDir)).
- WithRepositories(b.repositories).
- WithChartRepositoryCallback(b.getChartRepositoryCallback)
-
- result.CollectedDependencies, err = dm.Build(ctx)
- return
+ return o.ValueFiles
}
-// mergeValues strategically merges the valueFiles, it merges using mergeFileValues
-// or mergeChartValues depending on if the chart is sourced from a package or directory.
-// Ir only calls load to propagate the chart if required by the strategy.
-// It returns the merged values, or an error.
-func (b *ChartBuilder) mergeValues(result *ChartBuildResult) (err error) {
- if len(b.valueFiles) == 0 {
- return
- }
+// ChartBuild contains the ChartBuilder.Build result, including specific
+// information about the built chart like ResolvedDependencies.
+type ChartBuild struct {
+ // Path is the absolute path to the packaged chart.
+ Path string
+ // Name of the packaged chart.
+ Name string
+ // Version of the packaged chart.
+ Version string
+ // ValueFiles is the list of files used to compose the chart's
+ // default "values.yaml".
+ ValueFiles []string
+ // ResolvedDependencies is the number of local and remote dependencies
+ // collected by the DependencyManager before building the chart.
+ ResolvedDependencies int
+ // Packaged indicates if the ChartBuilder has packaged the chart.
+ // This can for example be false if ValueFiles is empty and the chart
+ // source was already packaged.
+ Packaged bool
+}
- if result.SourceIsDir {
- result.ValuesOverwrite, err = mergeFileValues(b.baseDir, b.valueFiles)
- if err != nil {
- err = fmt.Errorf("failed to merge value files: %w", err)
- }
- return
+// Summary returns a human-readable summary of the ChartBuild.
+func (b *ChartBuild) Summary() string {
+ if b == nil {
+ return "no chart build"
}
- // Values equal to default
- if len(b.valueFiles) == 1 && b.valueFiles[0] == chartutil.ValuesfileName {
- return
- }
+ var s strings.Builder
- if err = b.load(); err != nil {
- err = fmt.Errorf("failed to merge chart values: %w", err)
- return
+ action := "Fetched"
+ if b.Packaged {
+ action = "Packaged"
}
+ s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'.", action, b.Name, b.Version))
- if result.ValuesOverwrite, err = mergeChartValues(b.chart, b.valueFiles); err != nil {
- err = fmt.Errorf("failed to merge chart values: %w", err)
- return
+ if b.Packaged && b.ResolvedDependencies > 0 {
+ s.WriteString(fmt.Sprintf(" Resolved %d dependencies before packaging.", b.ResolvedDependencies))
}
- return nil
-}
-// packageChart determines if it should copyFileToPath or packageToPath
-// based on the provided result. It sets Packaged on ChartBuildResult to
-// true if packageToPath is successful.
-func (b *ChartBuilder) packageChart(result *ChartBuildResult) error {
- // If we are not building from a directory, and we do not have any
- // replacement values, we can copy over the already packaged source
- // chart without making any modifications
- if !result.SourceIsDir && len(result.ValuesOverwrite) == 0 {
- if err := copyFileToPath(b.path, result.Path); err != nil {
- return fmt.Errorf("chart build failed: %w", err)
- }
- return nil
+ if len(b.ValueFiles) > 0 {
+ s.WriteString(fmt.Sprintf(" Merged %v value files into default chart values.", b.ValueFiles))
}
- // Package chart to a new temporary directory
- if err := packageToPath(b.chart, result.Path); err != nil {
- return fmt.Errorf("chart build failed: %w", err)
- }
- result.Packaged = true
- return nil
+ return s.String()
}
-// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map.
-// It returns the merge result, or an error.
-func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) {
- mergedValues := make(map[string]interface{})
- for _, p := range paths {
- cfn := filepath.Clean(p)
- if cfn == chartutil.ValuesfileName {
- mergedValues = transform.MergeMaps(mergedValues, chart.Values)
- continue
- }
- var b []byte
- for _, f := range chart.Files {
- if f.Name == cfn {
- b = f.Data
- break
- }
- }
- if b == nil {
- return nil, fmt.Errorf("no values file found at path '%s'", p)
- }
- values := make(map[string]interface{})
- if err := yaml.Unmarshal(b, &values); err != nil {
- return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
- }
- mergedValues = transform.MergeMaps(mergedValues, values)
- }
- return mergedValues, nil
-}
-
-// mergeFileValues merges the given value file paths into a single "values.yaml" map.
-// The provided (relative) paths may not traverse outside baseDir. It returns the merge
-// result, or an error.
-func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) {
- mergedValues := make(map[string]interface{})
- for _, p := range paths {
- secureP, err := securejoin.SecureJoin(baseDir, p)
- if err != nil {
- return nil, err
- }
- if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() {
- return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')",
- strings.TrimPrefix(secureP, baseDir), p)
- }
- b, err := os.ReadFile(secureP)
- if err != nil {
- return nil, fmt.Errorf("could not read values from file '%s': %w", p, err)
- }
- values := make(map[string]interface{})
- err = yaml.Unmarshal(b, &values)
- if err != nil {
- return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
- }
- mergedValues = transform.MergeMaps(mergedValues, values)
- }
- return mergedValues, nil
-}
-
-// copyFileToPath attempts to copy in to out. It returns an error if out already exists.
-func copyFileToPath(in, out string) error {
- o, err := os.Create(out)
- if err != nil {
- return fmt.Errorf("failed to create copy target: %w", err)
- }
- defer o.Close()
- i, err := os.Open(in)
- if err != nil {
- return fmt.Errorf("failed to open file to copy from: %w", err)
- }
- defer i.Close()
- if _, err := o.ReadFrom(i); err != nil {
- return fmt.Errorf("failed to read from source during copy: %w", err)
+// String returns the Path of the ChartBuild.
+func (b *ChartBuild) String() string {
+ if b != nil {
+ return b.Path
}
- return nil
+ return ""
}
// packageToPath attempts to package the given chart.Chart to the out filepath.
@@ -368,17 +175,8 @@ func packageToPath(chart *helmchart.Chart, out string) error {
if err != nil {
return fmt.Errorf("failed to package chart: %w", err)
}
- return fs.RenameWithFallback(p, out)
-}
-
-// pathIsDir returns a boolean indicating if the given path points to a directory.
-// In case os.Stat on the given path returns an error it returns false as well.
-func pathIsDir(p string) bool {
- if p == "" {
- return false
+ if err = fs.RenameWithFallback(p, out); err != nil {
+ return fmt.Errorf("failed to write chart to file: %w", err)
}
- if i, err := os.Stat(p); err != nil || !i.IsDir() {
- return false
- }
- return true
+ return nil
}
diff --git a/internal/helm/chart_builder_local.go b/internal/helm/chart_builder_local.go
new file mode 100644
index 000000000..13e5dbe9c
--- /dev/null
+++ b/internal/helm/chart_builder_local.go
@@ -0,0 +1,190 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/Masterminds/semver/v3"
+ securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/fluxcd/pkg/runtime/transform"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "sigs.k8s.io/yaml"
+)
+
+type localChartBuilder struct {
+ dm *DependencyManager
+}
+
+// NewLocalChartBuilder returns a ChartBuilder capable of building a Helm
+// chart with a LocalChartReference. For chart references pointing to a
+// directory, the DependencyManager is used to resolve missing local and
+// remote dependencies.
+func NewLocalChartBuilder(dm *DependencyManager) ChartBuilder {
+ return &localChartBuilder{
+ dm: dm,
+ }
+}
+
+func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) {
+ localRef, ok := ref.(LocalChartReference)
+ if !ok {
+ return nil, fmt.Errorf("expected local chart reference")
+ }
+
+ if err := ref.Validate(); err != nil {
+ return nil, err
+ }
+
+ // Load the chart metadata from the LocalChartReference to ensure it points
+ // to a chart
+ curMeta, err := LoadChartMetadata(localRef.Path)
+ if err != nil {
+ return nil, err
+ }
+
+ result := &ChartBuild{}
+ result.Name = curMeta.Name
+
+ // Set build specific metadata if instructed
+ result.Version = curMeta.Version
+ if opts.VersionMetadata != "" {
+ ver, err := semver.NewVersion(curMeta.Version)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse chart version from metadata as SemVer: %w", err)
+ }
+ if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil {
+ return nil, fmt.Errorf("failed to set metadata on chart version: %w", err)
+ }
+ result.Version = ver.String()
+ }
+
+ // If all the following is true, we do not need to package the chart:
+ // Chart version from metadata matches chart version for ref
+ // BuildOptions.Force is False
+ if opts.CachedChart != "" && !opts.Force {
+ if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version {
+ result.Path = opts.CachedChart
+ result.ValueFiles = opts.ValueFiles
+ return result, nil
+ }
+ }
+
+ // If the chart at the path is already packaged and no custom value files
+ // options are set, we can copy the chart without making modifications
+ isChartDir := pathIsDir(localRef.Path)
+ if !isChartDir && len(opts.GetValueFiles()) == 0 {
+ if err := copyFileToPath(localRef.Path, p); err != nil {
+ return nil, err
+ }
+ result.Path = p
+ return result, nil
+ }
+
+ // Merge chart values, if instructed
+ var mergedValues map[string]interface{}
+ if len(opts.GetValueFiles()) > 0 {
+ if mergedValues, err = mergeFileValues(localRef.BaseDir, opts.ValueFiles); err != nil {
+ return nil, fmt.Errorf("failed to merge value files: %w", err)
+ }
+ }
+
+ // At this point we are certain we need to load the chart;
+ // either to package it because it originates from a directory,
+ // or because we have merged values and need to repackage
+ chart, err := loader.Load(localRef.Path)
+ if err != nil {
+ return nil, err
+ }
+ // Set earlier resolved version (with metadata)
+ chart.Metadata.Version = result.Version
+
+ // Overwrite default values with merged values, if any
+ if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil {
+ if err != nil {
+ return nil, err
+ }
+ result.ValueFiles = opts.GetValueFiles()
+ }
+
+ // Ensure dependencies are fetched if building from a directory
+ if isChartDir {
+ if b.dm == nil {
+ return nil, fmt.Errorf("local chart builder requires dependency manager for unpackaged charts")
+ }
+ if result.ResolvedDependencies, err = b.dm.Build(ctx, ref, chart); err != nil {
+ return nil, err
+ }
+ }
+
+ // Package the chart
+ if err = packageToPath(chart, p); err != nil {
+ return nil, err
+ }
+ result.Path = p
+ result.Packaged = true
+ return result, nil
+}
+
+// mergeFileValues merges the given value file paths into a single "values.yaml" map.
+// The provided (relative) paths may not traverse outside baseDir. It returns the merge
+// result, or an error.
+func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range paths {
+ secureP, err := securejoin.SecureJoin(baseDir, p)
+ if err != nil {
+ return nil, err
+ }
+ if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() {
+ return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')",
+ strings.TrimPrefix(secureP, baseDir), p)
+ }
+ b, err := os.ReadFile(secureP)
+ if err != nil {
+ return nil, fmt.Errorf("could not read values from file '%s': %w", p, err)
+ }
+ values := make(map[string]interface{})
+ err = yaml.Unmarshal(b, &values)
+ if err != nil {
+ return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
+
+// copyFileToPath attempts to copy in to out. It returns an error if out already exists.
+func copyFileToPath(in, out string) error {
+ o, err := os.Create(out)
+ if err != nil {
+ return fmt.Errorf("failed to create copy target: %w", err)
+ }
+ defer o.Close()
+ i, err := os.Open(in)
+ if err != nil {
+ return fmt.Errorf("failed to open file to copy from: %w", err)
+ }
+ defer i.Close()
+ if _, err := o.ReadFrom(i); err != nil {
+ return fmt.Errorf("failed to read from source during copy: %w", err)
+ }
+ return nil
+}
diff --git a/internal/helm/chart_builder_local_test.go b/internal/helm/chart_builder_local_test.go
new file mode 100644
index 000000000..c2f16d694
--- /dev/null
+++ b/internal/helm/chart_builder_local_test.go
@@ -0,0 +1,137 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ . "github.com/onsi/gomega"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+)
+
+func Test_mergeFileValues(t *testing.T) {
+ tests := []struct {
+ name string
+ files []*helmchart.File
+ paths []string
+ want map[string]interface{}
+ wantErr string
+ }{
+ {
+ name: "merges values from files",
+ files: []*helmchart.File{
+ {Name: "a.yaml", Data: []byte("a: b")},
+ {Name: "b.yaml", Data: []byte("b: c")},
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ paths: []string{"a.yaml", "b.yaml", "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "illegal traverse",
+ paths: []string{"../../../traversing/illegally/a/p/a/b"},
+ wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'",
+ },
+ {
+ name: "unmarshal error",
+ files: []*helmchart.File{
+ {Name: "invalid", Data: []byte("abcd")},
+ },
+ paths: []string{"invalid"},
+ wantErr: "unmarshaling values from 'invalid' failed",
+ },
+ {
+ name: "error on invalid path",
+ paths: []string{"a.yaml"},
+ wantErr: "no values file found at path '/a.yaml'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ baseDir, err := os.MkdirTemp("", "merge-file-values-*")
+ g.Expect(err).ToNot(HaveOccurred())
+ defer os.RemoveAll(baseDir)
+
+ for _, f := range tt.files {
+ g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed())
+ }
+
+ got, err := mergeFileValues(baseDir, tt.paths)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
+
+func Test_copyFileToPath(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ wantErr string
+ }{
+ {
+ name: "copies input file",
+ in: "testdata/local-index.yaml",
+ },
+ {
+ name: "invalid input file",
+ in: "testdata/invalid.tgz",
+ wantErr: "failed to open file to copy from",
+ },
+ {
+ name: "invalid input directory",
+ in: "testdata/charts",
+ wantErr: "failed to read from source during copy",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ out := tmpFile("copy-0.1.0", ".tgz")
+ defer os.RemoveAll(out)
+ err := copyFileToPath(tt.in, out)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(out).To(BeARegularFile())
+ f1, err := os.ReadFile(tt.in)
+ g.Expect(err).ToNot(HaveOccurred())
+ f2, err := os.ReadFile(out)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(f2).To(Equal(f1))
+ })
+ }
+}
diff --git a/internal/helm/chart_builder_remote.go b/internal/helm/chart_builder_remote.go
new file mode 100644
index 000000000..18ff317d8
--- /dev/null
+++ b/internal/helm/chart_builder_remote.go
@@ -0,0 +1,199 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+
+ "github.com/Masterminds/semver/v3"
+ "github.com/fluxcd/pkg/runtime/transform"
+ "github.com/fluxcd/source-controller/internal/fs"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "helm.sh/helm/v3/pkg/chartutil"
+ "sigs.k8s.io/yaml"
+)
+
+type remoteChartBuilder struct {
+ remote *ChartRepository
+}
+
+// NewRemoteChartBuilder returns a ChartBuilder capable of building a Helm
+// chart with a RemoteChartReference from the given ChartRepository.
+func NewRemoteChartBuilder(repository *ChartRepository) ChartBuilder {
+ return &remoteChartBuilder{
+ remote: repository,
+ }
+}
+
+func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) {
+ remoteRef, ok := ref.(RemoteChartReference)
+ if !ok {
+ return nil, fmt.Errorf("expected remote chart reference")
+ }
+
+ if err := ref.Validate(); err != nil {
+ return nil, err
+ }
+
+ if err := b.remote.LoadFromCache(); err != nil {
+ return nil, fmt.Errorf("could not load repository index for remote chart reference: %w", err)
+ }
+ defer b.remote.Unload()
+
+ // Get the current version for the RemoteChartReference
+ cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get chart version for remote reference: %w", err)
+ }
+
+ result := &ChartBuild{}
+ result.Name = cv.Name
+ result.Version = cv.Version
+ // Set build specific metadata if instructed
+ if opts.VersionMetadata != "" {
+ ver, err := semver.NewVersion(result.Version)
+ if err != nil {
+ return nil, err
+ }
+ if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil {
+ return nil, err
+ }
+ result.Version = ver.String()
+ }
+
+ // If all the following is true, we do not need to download and/or build the chart:
+ // Chart version from metadata matches chart version for ref
+ // BuildOptions.Force is False
+ if opts.CachedChart != "" && !opts.Force {
+ if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version {
+ result.Path = opts.CachedChart
+ result.ValueFiles = opts.GetValueFiles()
+ return result, nil
+ }
+ }
+
+ // Download the package for the resolved version
+ res, err := b.remote.DownloadChart(cv)
+ if err != nil {
+ return nil, fmt.Errorf("failed to download chart for remote reference: %w", err)
+ }
+
+ // Use literal chart copy from remote if no custom value files options are set
+ if len(opts.GetValueFiles()) == 0 {
+ if err = validatePackageAndWriteToPath(res, p); err != nil {
+ return nil, err
+ }
+ result.Path = p
+ return result, nil
+ }
+
+ // Load the chart and merge chart values
+ var chart *helmchart.Chart
+ if chart, err = loader.LoadArchive(res); err != nil {
+ return nil, fmt.Errorf("failed to load downloaded chart: %w", err)
+ }
+
+ mergedValues, err := mergeChartValues(chart, opts.ValueFiles)
+ if err != nil {
+ return nil, fmt.Errorf("failed to merge chart values: %w", err)
+ }
+ // Overwrite default values with merged values, if any
+ if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil {
+ if err != nil {
+ return nil, err
+ }
+ result.ValueFiles = opts.GetValueFiles()
+ }
+
+ // Package the chart with the custom values
+ if err = packageToPath(chart, p); err != nil {
+ return nil, err
+ }
+ result.Path = p
+ result.Packaged = true
+ return result, nil
+}
+
+// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map.
+// It returns the merge result, or an error.
+func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) {
+ mergedValues := make(map[string]interface{})
+ for _, p := range paths {
+ cfn := filepath.Clean(p)
+ if cfn == chartutil.ValuesfileName {
+ mergedValues = transform.MergeMaps(mergedValues, chart.Values)
+ continue
+ }
+ var b []byte
+ for _, f := range chart.Files {
+ if f.Name == cfn {
+ b = f.Data
+ break
+ }
+ }
+ if b == nil {
+ return nil, fmt.Errorf("no values file found at path '%s'", p)
+ }
+ values := make(map[string]interface{})
+ if err := yaml.Unmarshal(b, &values); err != nil {
+ return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err)
+ }
+ mergedValues = transform.MergeMaps(mergedValues, values)
+ }
+ return mergedValues, nil
+}
+
+// validatePackageAndWriteToPath atomically writes the packaged chart from reader
+// to out while validating it by loading the chart metadata from the archive.
+func validatePackageAndWriteToPath(reader io.Reader, out string) error {
+ tmpFile, err := os.CreateTemp("", filepath.Base(out))
+ if err != nil {
+ return fmt.Errorf("failed to create temporary file for chart: %w", err)
+ }
+ defer os.Remove(tmpFile.Name())
+ if _, err = tmpFile.ReadFrom(reader); err != nil {
+ _ = tmpFile.Close()
+ return fmt.Errorf("failed to write chart to file: %w", err)
+ }
+ if err = tmpFile.Close(); err != nil {
+ return err
+ }
+ if _, err = LoadChartMetadataFromArchive(tmpFile.Name()); err != nil {
+ return fmt.Errorf("failed to load chart metadata from written chart: %w", err)
+ }
+ if err = fs.RenameWithFallback(tmpFile.Name(), out); err != nil {
+ return fmt.Errorf("failed to write chart to file: %w", err)
+ }
+ return nil
+}
+
+// pathIsDir returns a boolean indicating if the given path points to a directory.
+// In case os.Stat on the given path returns an error it returns false as well.
+func pathIsDir(p string) bool {
+ if p == "" {
+ return false
+ }
+ if i, err := os.Stat(p); err != nil || !i.IsDir() {
+ return false
+ }
+ return true
+}
diff --git a/internal/helm/chart_builder_remote_test.go b/internal/helm/chart_builder_remote_test.go
new file mode 100644
index 000000000..260bcbce1
--- /dev/null
+++ b/internal/helm/chart_builder_remote_test.go
@@ -0,0 +1,118 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+ helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chartutil"
+)
+
+func Test_mergeChartValues(t *testing.T) {
+ tests := []struct {
+ name string
+ chart *helmchart.Chart
+ paths []string
+ want map[string]interface{}
+ wantErr string
+ }{
+ {
+ name: "merges values",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "a.yaml", Data: []byte("a: b")},
+ {Name: "b.yaml", Data: []byte("b: c")},
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ },
+ paths: []string{"a.yaml", "b.yaml", "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "uses chart values",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "c.yaml", Data: []byte("b: d")},
+ },
+ Values: map[string]interface{}{
+ "a": "b",
+ },
+ },
+ paths: []string{chartutil.ValuesfileName, "c.yaml"},
+ want: map[string]interface{}{
+ "a": "b",
+ "b": "d",
+ },
+ },
+ {
+ name: "unmarshal error",
+ chart: &helmchart.Chart{
+ Files: []*helmchart.File{
+ {Name: "invalid", Data: []byte("abcd")},
+ },
+ },
+ paths: []string{"invalid"},
+ wantErr: "unmarshaling values from 'invalid' failed",
+ },
+ {
+ name: "error on invalid path",
+ chart: &helmchart.Chart{},
+ paths: []string{"a.yaml"},
+ wantErr: "no values file found at path 'a.yaml'",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ got, err := mergeChartValues(tt.chart, tt.paths)
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(got).To(BeNil())
+ return
+ }
+
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
+
+func Test_pathIsDir(t *testing.T) {
+ tests := []struct {
+ name string
+ p string
+ want bool
+ }{
+ {name: "directory", p: "testdata/", want: true},
+ {name: "file", p: "testdata/local-index.yaml", want: false},
+ {name: "not found error", p: "testdata/does-not-exist.yaml", want: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ g.Expect(pathIsDir(tt.p)).To(Equal(tt.want))
+ })
+ }
+}
diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart_builder_test.go
index afc0107ce..a4252be8f 100644
--- a/internal/helm/chart_builder_test.go
+++ b/internal/helm/chart_builder_test.go
@@ -17,545 +17,27 @@ limitations under the License.
package helm
import (
- "context"
"encoding/hex"
- "fmt"
"math/rand"
"os"
"path/filepath"
- "sync"
"testing"
. "github.com/onsi/gomega"
- helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
- "helm.sh/helm/v3/pkg/chartutil"
- "helm.sh/helm/v3/pkg/repo"
)
func TestChartBuildResult_String(t *testing.T) {
g := NewWithT(t)
- var result *ChartBuildResult
+ var result *ChartBuild
g.Expect(result.String()).To(Equal(""))
- result = &ChartBuildResult{}
+ result = &ChartBuild{}
g.Expect(result.String()).To(Equal(""))
- result = &ChartBuildResult{Path: "/foo/"}
+ result = &ChartBuild{Path: "/foo/"}
g.Expect(result.String()).To(Equal("/foo/"))
}
-func TestChartBuilder_Build(t *testing.T) {
- tests := []struct {
- name string
- baseDir string
- path string
- valueFiles []string
- repositories map[string]*ChartRepository
- getChartRepositoryCallback GetChartRepositoryCallback
- wantErr string
- }{
- {
- name: "builds chart from directory",
- path: "testdata/charts/helmchart",
- },
- {
- name: "builds chart from package",
- path: "testdata/charts/helmchart-0.1.0.tgz",
- },
- {
- // TODO(hidde): add more diverse tests
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- b, err := NewChartBuilder(tt.path)
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(b).ToNot(BeNil())
-
- b.WithBaseDir(tt.baseDir)
- b.WithValueFiles(tt.valueFiles...)
- b.WithChartRepositoryCallback(b.getChartRepositoryCallback)
- for k, v := range tt.repositories {
- b.WithChartRepository(k, v)
- }
-
- out := tmpFile("build-0.1.0", ".tgz")
- defer os.RemoveAll(out)
- got, err := b.Build(context.TODO(), out)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- g.Expect(got).To(BeNil())
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(got).ToNot(BeNil())
-
- g.Expect(got.Path).ToNot(BeEmpty())
- g.Expect(got.Path).To(Equal(out))
- g.Expect(got.Path).To(BeARegularFile())
- _, err = loader.Load(got.Path)
- g.Expect(err).ToNot(HaveOccurred())
- })
- }
-}
-
-func TestChartBuilder_load(t *testing.T) {
- tests := []struct {
- name string
- path string
- chart *helmchart.Chart
- wantFunc func(g *WithT, c *helmchart.Chart)
- wantErr string
- }{
- {
- name: "loads chart",
- chart: nil,
- path: "testdata/charts/helmchart-0.1.0.tgz",
- wantFunc: func(g *WithT, c *helmchart.Chart) {
- g.Expect(c.Metadata.Name).To(Equal("helmchart"))
- g.Expect(c.Files).ToNot(BeZero())
- },
- },
- {
- name: "overwrites chart without any files (metadata shim)",
- chart: &helmchart.Chart{
- Metadata: &helmchart.Metadata{Name: "dummy"},
- },
- path: "testdata/charts/helmchart-0.1.0.tgz",
- wantFunc: func(g *WithT, c *helmchart.Chart) {
- g.Expect(c.Metadata.Name).To(Equal("helmchart"))
- g.Expect(c.Files).ToNot(BeZero())
- },
- },
- {
- name: "does not overwrite loaded chart",
- chart: &helmchart.Chart{
- Metadata: &helmchart.Metadata{Name: "dummy"},
- Files: []*helmchart.File{
- {Name: "mock.yaml", Data: []byte("loaded chart")},
- },
- },
- path: "testdata/charts/helmchart-0.1.0.tgz",
- wantFunc: func(g *WithT, c *helmchart.Chart) {
- g.Expect(c.Metadata.Name).To(Equal("dummy"))
- g.Expect(c.Files).To(HaveLen(1))
- },
- },
- {
- name: "no path",
- wantErr: "failed to load chart: path not set",
- },
- {
- name: "invalid chart",
- path: "testdata/charts/empty.tgz",
- wantErr: "failed to load chart: no files in chart archive",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- b := &ChartBuilder{
- path: tt.path,
- chart: tt.chart,
- }
- err := b.load()
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- if tt.wantFunc != nil {
- tt.wantFunc(g, b.chart)
- }
- })
- }
-}
-
-func TestChartBuilder_buildDependencies(t *testing.T) {
- g := NewWithT(t)
-
- chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz")
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(chartB).ToNot(BeEmpty())
-
- mockRepo := func() *ChartRepository {
- return &ChartRepository{
- Client: &mockGetter{
- response: chartB,
- },
- Index: &repo.IndexFile{
- Entries: map[string]repo.ChartVersions{
- "grafana": {
- &repo.ChartVersion{
- Metadata: &helmchart.Metadata{
- Name: "grafana",
- Version: "6.17.4",
- },
- URLs: []string{"https://example.com/chart.tgz"},
- },
- },
- },
- },
- RWMutex: &sync.RWMutex{},
- }
- }
-
- var mockCallback GetChartRepositoryCallback = func(url string) (*ChartRepository, error) {
- if url == "https://grafana.github.io/helm-charts/" {
- return mockRepo(), nil
- }
- return nil, fmt.Errorf("no repository for URL")
- }
-
- tests := []struct {
- name string
- baseDir string
- path string
- chart *helmchart.Chart
- fromDir bool
- repositories map[string]*ChartRepository
- getChartRepositoryCallback GetChartRepositoryCallback
- wantCollectedDependencies int
- wantErr string
- }{
- {
- name: "builds dependencies using callback",
- fromDir: true,
- baseDir: "testdata/charts",
- path: "testdata/charts/helmchartwithdeps",
- getChartRepositoryCallback: mockCallback,
- wantCollectedDependencies: 2,
- },
- {
- name: "builds dependencies using repositories",
- fromDir: true,
- baseDir: "testdata/charts",
- path: "testdata/charts/helmchartwithdeps",
- repositories: map[string]*ChartRepository{
- "https://grafana.github.io/helm-charts/": mockRepo(),
- },
- wantCollectedDependencies: 2,
- },
- {
- name: "skips dependency build for packaged chart",
- path: "testdata/charts/helmchart-0.1.0.tgz",
- },
- {
- name: "attempts to load chart",
- fromDir: true,
- path: "testdata",
- wantErr: "failed to ensure chart has no missing dependencies",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- b := &ChartBuilder{
- baseDir: tt.baseDir,
- path: tt.path,
- chart: tt.chart,
- repositories: tt.repositories,
- getChartRepositoryCallback: tt.getChartRepositoryCallback,
- }
-
- result := &ChartBuildResult{SourceIsDir: tt.fromDir}
- err := b.buildDependencies(context.TODO(), result)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- g.Expect(result.CollectedDependencies).To(BeZero())
- g.Expect(b.chart).To(Equal(tt.chart))
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(result).ToNot(BeNil())
- g.Expect(result.CollectedDependencies).To(Equal(tt.wantCollectedDependencies))
- if tt.wantCollectedDependencies > 0 {
- g.Expect(b.chart).ToNot(Equal(tt.chart))
- }
- })
- }
-}
-
-func TestChartBuilder_mergeValues(t *testing.T) {
- tests := []struct {
- name string
- baseDir string
- path string
- isDir bool
- chart *helmchart.Chart
- valueFiles []string
- want map[string]interface{}
- wantErr string
- }{
- {
- name: "merges chart values",
- chart: &helmchart.Chart{
- Files: []*helmchart.File{
- {Name: "a.yaml", Data: []byte("a: b")},
- {Name: "b.yaml", Data: []byte("a: c")},
- },
- },
- valueFiles: []string{"a.yaml", "b.yaml"},
- want: map[string]interface{}{
- "a": "c",
- },
- },
- {
- name: "chart values merge error",
- chart: &helmchart.Chart{
- Files: []*helmchart.File{
- {Name: "b.yaml", Data: []byte("a: c")},
- },
- },
- valueFiles: []string{"a.yaml"},
- wantErr: "failed to merge chart values",
- },
- {
- name: "merges file values",
- isDir: true,
- baseDir: "testdata/charts",
- path: "helmchart",
- valueFiles: []string{"helmchart/values-prod.yaml"},
- want: map[string]interface{}{
- "replicaCount": float64(2),
- },
- },
- {
- name: "file values merge error",
- isDir: true,
- baseDir: "testdata/charts",
- path: "helmchart",
- valueFiles: []string{"invalid.yaml"},
- wantErr: "failed to merge value files",
- },
- {
- name: "error on chart load failure",
- baseDir: "testdata/charts",
- path: "invalid",
- wantErr: "failed to load chart",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- b := &ChartBuilder{
- baseDir: tt.baseDir,
- path: tt.path,
- chart: tt.chart,
- valueFiles: tt.valueFiles,
- }
-
- result := &ChartBuildResult{SourceIsDir: tt.isDir}
- err := b.mergeValues(result)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- g.Expect(result.ValuesOverwrite).To(BeNil())
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(result.ValuesOverwrite).To(Equal(tt.want))
- })
- }
-}
-
-func Test_mergeChartValues(t *testing.T) {
- tests := []struct {
- name string
- chart *helmchart.Chart
- paths []string
- want map[string]interface{}
- wantErr string
- }{
- {
- name: "merges values",
- chart: &helmchart.Chart{
- Files: []*helmchart.File{
- {Name: "a.yaml", Data: []byte("a: b")},
- {Name: "b.yaml", Data: []byte("b: c")},
- {Name: "c.yaml", Data: []byte("b: d")},
- },
- },
- paths: []string{"a.yaml", "b.yaml", "c.yaml"},
- want: map[string]interface{}{
- "a": "b",
- "b": "d",
- },
- },
- {
- name: "uses chart values",
- chart: &helmchart.Chart{
- Files: []*helmchart.File{
- {Name: "c.yaml", Data: []byte("b: d")},
- },
- Values: map[string]interface{}{
- "a": "b",
- },
- },
- paths: []string{chartutil.ValuesfileName, "c.yaml"},
- want: map[string]interface{}{
- "a": "b",
- "b": "d",
- },
- },
- {
- name: "unmarshal error",
- chart: &helmchart.Chart{
- Files: []*helmchart.File{
- {Name: "invalid", Data: []byte("abcd")},
- },
- },
- paths: []string{"invalid"},
- wantErr: "unmarshaling values from 'invalid' failed",
- },
- {
- name: "error on invalid path",
- chart: &helmchart.Chart{},
- paths: []string{"a.yaml"},
- wantErr: "no values file found at path 'a.yaml'",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- got, err := mergeChartValues(tt.chart, tt.paths)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- g.Expect(got).To(BeNil())
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(got).To(Equal(tt.want))
- })
- }
-}
-
-func Test_mergeFileValues(t *testing.T) {
- tests := []struct {
- name string
- files []*helmchart.File
- paths []string
- want map[string]interface{}
- wantErr string
- }{
- {
- name: "merges values from files",
- files: []*helmchart.File{
- {Name: "a.yaml", Data: []byte("a: b")},
- {Name: "b.yaml", Data: []byte("b: c")},
- {Name: "c.yaml", Data: []byte("b: d")},
- },
- paths: []string{"a.yaml", "b.yaml", "c.yaml"},
- want: map[string]interface{}{
- "a": "b",
- "b": "d",
- },
- },
- {
- name: "illegal traverse",
- paths: []string{"../../../traversing/illegally/a/p/a/b"},
- wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'",
- },
- {
- name: "unmarshal error",
- files: []*helmchart.File{
- {Name: "invalid", Data: []byte("abcd")},
- },
- paths: []string{"invalid"},
- wantErr: "unmarshaling values from 'invalid' failed",
- },
- {
- name: "error on invalid path",
- paths: []string{"a.yaml"},
- wantErr: "no values file found at path '/a.yaml'",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- baseDir, err := os.MkdirTemp("", "merge-file-values-*")
- g.Expect(err).ToNot(HaveOccurred())
- defer os.RemoveAll(baseDir)
-
- for _, f := range tt.files {
- g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed())
- }
-
- got, err := mergeFileValues(baseDir, tt.paths)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- g.Expect(got).To(BeNil())
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(got).To(Equal(tt.want))
- })
- }
-}
-
-func Test_copyFileToPath(t *testing.T) {
- tests := []struct {
- name string
- in string
- wantErr string
- }{
- {
- name: "copies input file",
- in: "testdata/local-index.yaml",
- },
- {
- name: "invalid input file",
- in: "testdata/invalid.tgz",
- wantErr: "failed to open file to copy from",
- },
- {
- name: "invalid input directory",
- in: "testdata/charts",
- wantErr: "failed to read from source during copy",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- out := tmpFile("copy-0.1.0", ".tgz")
- defer os.RemoveAll(out)
- err := copyFileToPath(tt.in, out)
- if tt.wantErr != "" {
- g.Expect(err).To(HaveOccurred())
- g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
- return
- }
-
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(out).To(BeARegularFile())
- f1, err := os.ReadFile(tt.in)
- g.Expect(err).ToNot(HaveOccurred())
- f2, err := os.ReadFile(out)
- g.Expect(err).ToNot(HaveOccurred())
- g.Expect(f2).To(Equal(f1))
- })
- }
-}
-
func Test_packageToPath(t *testing.T) {
g := NewWithT(t)
@@ -572,25 +54,6 @@ func Test_packageToPath(t *testing.T) {
g.Expect(err).ToNot(HaveOccurred())
}
-func Test_pathIsDir(t *testing.T) {
- tests := []struct {
- name string
- p string
- want bool
- }{
- {name: "directory", p: "testdata/", want: true},
- {name: "file", p: "testdata/local-index.yaml", want: false},
- {name: "not found error", p: "testdata/does-not-exist.yaml", want: false},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- g.Expect(pathIsDir(tt.p)).To(Equal(tt.want))
- })
- }
-}
-
func tmpFile(prefix, suffix string) string {
randBytes := make([]byte, 16)
rand.Read(randBytes)
diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go
index 23d50b96b..ac7114e87 100644
--- a/internal/helm/chart_test.go
+++ b/internal/helm/chart_test.go
@@ -25,8 +25,9 @@ import (
)
var (
- originalValuesFixture = []byte("override: original")
- chartFilesFixture = []*helmchart.File{
+ originalValuesFixture = []byte(`override: original
+`)
+ chartFilesFixture = []*helmchart.File{
{
Name: "values.yaml",
Data: originalValuesFixture,
@@ -69,19 +70,14 @@ func TestOverwriteChartDefaultValues(t *testing.T) {
desc: "valid override",
chart: chartFixture,
ok: true,
- data: []byte("override: test"),
+ data: []byte(`override: test
+`),
},
{
desc: "empty override",
chart: chartFixture,
ok: true,
- data: []byte(""),
- },
- {
- desc: "invalid",
- chart: chartFixture,
- data: []byte("!fail:"),
- expectErr: true,
+ data: []byte(``),
},
}
for _, tt := range testCases {
@@ -89,7 +85,9 @@ func TestOverwriteChartDefaultValues(t *testing.T) {
g := NewWithT(t)
fixture := tt.chart
- ok, err := OverwriteChartDefaultValues(&fixture, tt.data)
+ vals, err := chartutil.ReadValues(tt.data)
+ g.Expect(err).ToNot(HaveOccurred())
+ ok, err := OverwriteChartDefaultValues(&fixture, vals)
g.Expect(ok).To(Equal(tt.ok))
if tt.expectErr {
diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go
index 043b0e7e3..b8cd78571 100644
--- a/internal/helm/dependency_manager.go
+++ b/internal/helm/dependency_manager.go
@@ -37,72 +37,77 @@ import (
// or an error describing why it could not be returned.
type GetChartRepositoryCallback func(url string) (*ChartRepository, error)
-// DependencyManager manages dependencies for a Helm chart, downloading
-// only those that are missing from the chart it holds.
+// DependencyManager manages dependencies for a Helm chart.
type DependencyManager struct {
- // chart contains the chart.Chart from the path.
- chart *helmchart.Chart
-
- // baseDir is the chroot path for dependency manager operations,
- // Dependencies that hold a local (relative) path reference are not
- // allowed to traverse outside this directory.
- baseDir string
-
- // path is the path of the chart relative to the baseDir,
- // the combination of the baseDir and path is used to
- // determine the absolute path of a local dependency.
- path string
-
// repositories contains a map of ChartRepository indexed by their
// normalized URL. It is used as a lookup table for missing
// dependencies.
repositories map[string]*ChartRepository
- // getChartRepositoryCallback can be set to an on-demand get
- // callback which returned result is cached to repositories.
- getChartRepositoryCallback GetChartRepositoryCallback
+ // getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback
+ // which returned result is cached to repositories.
+ getRepositoryCallback GetChartRepositoryCallback
- // workers is the number of concurrent chart-add operations during
+ // concurrent is the number of concurrent chart-add operations during
// Build. Defaults to 1 (non-concurrent).
- workers int64
+ concurrent int64
// mu contains the lock for chart writes.
mu sync.Mutex
}
-func NewDependencyManager(chart *helmchart.Chart, baseDir, path string) *DependencyManager {
- return &DependencyManager{
- chart: chart,
- baseDir: baseDir,
- path: path,
- }
+type DependencyManagerOption interface {
+ applyToDependencyManager(dm *DependencyManager)
}
-func (dm *DependencyManager) WithRepositories(r map[string]*ChartRepository) *DependencyManager {
- dm.repositories = r
- return dm
+type WithRepositories map[string]*ChartRepository
+
+func (o WithRepositories) applyToDependencyManager(dm *DependencyManager) {
+ dm.repositories = o
}
-func (dm *DependencyManager) WithChartRepositoryCallback(c GetChartRepositoryCallback) *DependencyManager {
- dm.getChartRepositoryCallback = c
- return dm
+type WithRepositoryCallback GetChartRepositoryCallback
+
+func (o WithRepositoryCallback) applyToDependencyManager(dm *DependencyManager) {
+ dm.getRepositoryCallback = GetChartRepositoryCallback(o)
+}
+
+type WithConcurrent int64
+
+func (o WithConcurrent) applyToDependencyManager(dm *DependencyManager) {
+ dm.concurrent = int64(o)
}
-func (dm *DependencyManager) WithWorkers(w int64) *DependencyManager {
- dm.workers = w
+// NewDependencyManager returns a new DependencyManager configured with the given
+// DependencyManagerOption list.
+func NewDependencyManager(opts ...DependencyManagerOption) *DependencyManager {
+ dm := &DependencyManager{}
+ for _, v := range opts {
+ v.applyToDependencyManager(dm)
+ }
return dm
}
-// Build compiles and builds the dependencies of the chart with the
-// configured number of workers.
-func (dm *DependencyManager) Build(ctx context.Context) (int, error) {
+func (dm *DependencyManager) Clear() []error {
+ var errs []error
+ for _, v := range dm.repositories {
+ v.Unload()
+ errs = append(errs, v.RemoveCache())
+ }
+ return errs
+}
+
+// Build compiles a set of missing dependencies from chart.Chart, and attempts to
+// resolve and build them using the information from ChartReference.
+// It returns the number of resolved local and remote dependencies, or an error.
+func (dm *DependencyManager) Build(ctx context.Context, ref ChartReference, chart *helmchart.Chart) (int, error) {
// Collect dependency metadata
var (
- deps = dm.chart.Dependencies()
- reqs = dm.chart.Metadata.Dependencies
+ deps = chart.Dependencies()
+ reqs = chart.Metadata.Dependencies
)
// Lock file takes precedence
- if lock := dm.chart.Lock; lock != nil {
+ if lock := chart.Lock; lock != nil {
reqs = lock.Dependencies
}
@@ -113,31 +118,32 @@ func (dm *DependencyManager) Build(ctx context.Context) (int, error) {
}
// Run the build for the missing dependencies
- if err := dm.build(ctx, missing); err != nil {
+ if err := dm.build(ctx, ref, chart, missing); err != nil {
return 0, err
}
return len(missing), nil
}
-// build (concurrently) adds the given list of deps to the chart with the configured
-// number of workers. It returns the first error, cancelling all other workers.
-func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmchart.Dependency) error {
- workers := dm.workers
- if workers <= 0 {
- workers = 1
- }
+// chartWithLock holds a chart.Chart with a sync.Mutex to lock for writes.
+type chartWithLock struct {
+ *helmchart.Chart
+ mu sync.Mutex
+}
- // Garbage collect temporary cached ChartRepository indexes
- defer func() {
- for _, v := range dm.repositories {
- v.Unload()
- _ = v.RemoveCache()
- }
- }()
+// build adds the given list of deps to the chart with the configured number of
+// concurrent workers. If the chart.Chart references a local dependency but no
+// LocalChartReference is given, or any dependency could not be added, an error
+// is returned. The first error it encounters cancels all other workers.
+func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, chart *helmchart.Chart, deps map[string]*helmchart.Dependency) error {
+ current := dm.concurrent
+ if current <= 0 {
+ current = 1
+ }
group, groupCtx := errgroup.WithContext(ctx)
group.Go(func() error {
- sem := semaphore.NewWeighted(workers)
+ sem := semaphore.NewWeighted(current)
+ chart := &chartWithLock{Chart: chart}
for name, dep := range deps {
name, dep := name, dep
if err := sem.Acquire(groupCtx, 1); err != nil {
@@ -146,12 +152,17 @@ func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmcha
group.Go(func() (err error) {
defer sem.Release(1)
if isLocalDep(dep) {
- if err = dm.addLocalDependency(dep); err != nil {
+ localRef, ok := ref.(LocalChartReference)
+ if !ok {
+ err = fmt.Errorf("failed to add local dependency '%s': no local chart reference", name)
+ return
+ }
+ if err = dm.addLocalDependency(localRef, chart, dep); err != nil {
err = fmt.Errorf("failed to add local dependency '%s': %w", name, err)
}
return
}
- if err = dm.addRemoteDependency(dep); err != nil {
+ if err = dm.addRemoteDependency(chart, dep); err != nil {
err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err)
}
return
@@ -162,17 +173,17 @@ func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmcha
return group.Wait()
}
-// addLocalDependency attempts to resolve and add the given local chart.Dependency to the chart.
-func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error {
- sLocalChartPath, err := dm.secureLocalChartPath(dep)
+// addLocalDependency attempts to resolve and add the given local chart.Dependency
+// to the chart.
+func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *chartWithLock, dep *helmchart.Dependency) error {
+ sLocalChartPath, err := dm.secureLocalChartPath(ref, dep)
if err != nil {
return err
}
if _, err := os.Stat(sLocalChartPath); err != nil {
if os.IsNotExist(err) {
- return fmt.Errorf("no chart found at '%s' (reference '%s')",
- strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository)
+ return fmt.Errorf("no chart found at '%s' (reference '%s')", sLocalChartPath, dep.Repository)
}
return err
}
@@ -186,7 +197,7 @@ func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error
ch, err := loader.Load(sLocalChartPath)
if err != nil {
return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w",
- strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository, err)
+ strings.TrimPrefix(sLocalChartPath, ref.BaseDir), dep.Repository, err)
}
ver, err := semver.NewVersion(ch.Metadata.Version)
@@ -199,14 +210,16 @@ func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error
return err
}
- dm.mu.Lock()
- dm.chart.AddDependency(ch)
- dm.mu.Unlock()
+ chart.mu.Lock()
+ chart.AddDependency(ch)
+ chart.mu.Unlock()
return nil
}
-// addRemoteDependency attempts to resolve and add the given remote chart.Dependency to the chart.
-func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) error {
+// addRemoteDependency attempts to resolve and add the given remote chart.Dependency
+// to the chart. It locks the chartWithLock before the downloaded dependency is
+// added to the chart.
+func (dm *DependencyManager) addRemoteDependency(chart *chartWithLock, dep *helmchart.Dependency) error {
repo, err := dm.resolveRepository(dep.Repository)
if err != nil {
return err
@@ -216,7 +229,6 @@ func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) erro
return fmt.Errorf("failed to load index for '%s': %w", dep.Name, err)
}
-
ver, err := repo.Get(dep.Name, dep.Version)
if err != nil {
return err
@@ -230,28 +242,28 @@ func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) erro
return fmt.Errorf("failed to load downloaded archive of version '%s': %w", ver.Version, err)
}
- dm.mu.Lock()
- dm.chart.AddDependency(ch)
- dm.mu.Unlock()
+ chart.mu.Lock()
+ chart.AddDependency(ch)
+ chart.mu.Unlock()
return nil
}
// resolveRepository first attempts to resolve the url from the repositories, falling back
-// to getChartRepositoryCallback if set. It returns the resolved ChartRepository, or an error.
+// to getRepositoryCallback if set. It returns the resolved ChartRepository, or an error.
func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) {
dm.mu.Lock()
defer dm.mu.Unlock()
nUrl := NormalizeChartRepositoryURL(url)
if _, ok := dm.repositories[nUrl]; !ok {
- if dm.getChartRepositoryCallback == nil {
+ if dm.getRepositoryCallback == nil {
err = fmt.Errorf("no chart repository for URL '%s'", nUrl)
return
}
if dm.repositories == nil {
dm.repositories = map[string]*ChartRepository{}
}
- if dm.repositories[nUrl], err = dm.getChartRepositoryCallback(nUrl); err != nil {
+ if dm.repositories[nUrl], err = dm.getRepositoryCallback(nUrl); err != nil {
err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err)
return
}
@@ -260,8 +272,9 @@ func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository,
}
// secureLocalChartPath returns the secure absolute path of a local dependency.
-// It does not allow the dependency's path to be outside the scope of baseDir.
-func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (string, error) {
+// It does not allow the dependency's path to be outside the scope of
+// LocalChartReference.BaseDir.
+func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep *helmchart.Dependency) (string, error) {
localUrl, err := url.Parse(dep.Repository)
if err != nil {
return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err)
@@ -269,7 +282,11 @@ func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (st
if localUrl.Scheme != "" && localUrl.Scheme != "file" {
return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository)
}
- return securejoin.SecureJoin(dm.baseDir, filepath.Join(dm.path, localUrl.Host, localUrl.Path))
+ relPath, err := filepath.Rel(ref.BaseDir, ref.Path)
+ if err != nil {
+ return "", err
+ }
+ return securejoin.SecureJoin(ref.BaseDir, filepath.Join(relPath, localUrl.Host, localUrl.Path))
}
// collectMissing returns a map with reqs that are missing from current,
diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go
index e51e6b768..388eff1f4 100644
--- a/internal/helm/dependency_manager_test.go
+++ b/internal/helm/dependency_manager_test.go
@@ -88,10 +88,10 @@ func TestDependencyManager_Build(t *testing.T) {
chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path))
g.Expect(err).ToNot(HaveOccurred())
- got, err := NewDependencyManager(chart, tt.baseDir, tt.path).
- WithRepositories(tt.repositories).
- WithChartRepositoryCallback(tt.getChartRepositoryCallback).
- Build(context.TODO())
+ got, err := NewDependencyManager(
+ WithRepositories(tt.repositories),
+ WithRepositoryCallback(tt.getChartRepositoryCallback),
+ ).Build(context.TODO(), LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, chart)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
@@ -134,10 +134,8 @@ func TestDependencyManager_build(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
- dm := &DependencyManager{
- baseDir: "testdata/charts",
- }
- err := dm.build(context.TODO(), tt.deps)
+ dm := NewDependencyManager()
+ err := dm.build(context.TODO(), LocalChartReference{}, &helmchart.Chart{}, tt.deps)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
return
@@ -182,7 +180,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
Version: chartVersion,
Repository: "file://../../../absolutely/invalid",
},
- wantErr: "no chart found at 'absolutely/invalid'",
+ wantErr: "no chart found at 'testdata/charts/absolutely/invalid'",
},
{
name: "invalid chart archive",
@@ -191,7 +189,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
Version: chartVersion,
Repository: "file://../empty.tgz",
},
- wantErr: "failed to load chart from 'empty.tgz'",
+ wantErr: "failed to load chart from '/empty.tgz'",
},
{
name: "invalid constraint",
@@ -207,13 +205,10 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
- dm := &DependencyManager{
- chart: &helmchart.Chart{},
- baseDir: "testdata/charts/",
- path: "helmchartwithdeps",
- }
-
- err := dm.addLocalDependency(tt.dep)
+ dm := NewDependencyManager()
+ chart := &helmchart.Chart{}
+ err := dm.addLocalDependency(LocalChartReference{BaseDir: "testdata/charts", Path: "helmchartwithdeps"},
+ &chartWithLock{Chart: chart}, tt.dep)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
@@ -389,10 +384,10 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
g := NewWithT(t)
dm := &DependencyManager{
- chart: &helmchart.Chart{},
repositories: tt.repositories,
}
- err := dm.addRemoteDependency(tt.dep)
+ chart := &helmchart.Chart{}
+ err := dm.addRemoteDependency(&chartWithLock{Chart: chart}, tt.dep)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
@@ -400,7 +395,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
}
g.Expect(err).ToNot(HaveOccurred())
if tt.wantFunc != nil {
- tt.wantFunc(g, dm.chart)
+ tt.wantFunc(g, chart)
}
})
}
@@ -455,8 +450,8 @@ func TestDependencyManager_resolveRepository(t *testing.T) {
g := NewWithT(t)
dm := &DependencyManager{
- repositories: tt.repositories,
- getChartRepositoryCallback: tt.getChartRepositoryCallback,
+ repositories: tt.repositories,
+ getRepositoryCallback: tt.getChartRepositoryCallback,
}
got, err := dm.resolveRepository(tt.url)
@@ -522,11 +517,8 @@ func TestDependencyManager_secureLocalChartPath(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
- dm := &DependencyManager{
- baseDir: tt.baseDir,
- path: tt.path,
- }
- got, err := dm.secureLocalChartPath(tt.dep)
+ dm := NewDependencyManager()
+ got, err := dm.secureLocalChartPath(LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, tt.dep)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
diff --git a/internal/helm/getter.go b/internal/helm/getter.go
index b0f07e96b..1ca8b0e9b 100644
--- a/internal/helm/getter.go
+++ b/internal/helm/getter.go
@@ -19,31 +19,30 @@ package helm
import (
"fmt"
"os"
- "path/filepath"
"helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
)
// ClientOptionsFromSecret constructs a getter.Option slice for the given secret.
-// It returns the slice, and a callback to remove temporary files.
-func ClientOptionsFromSecret(secret corev1.Secret) ([]getter.Option, func(), error) {
+// It returns the slice, or an error.
+func ClientOptionsFromSecret(dir string, secret corev1.Secret) ([]getter.Option, error) {
var opts []getter.Option
basicAuth, err := BasicAuthFromSecret(secret)
if err != nil {
- return opts, nil, err
+ return opts, err
}
if basicAuth != nil {
opts = append(opts, basicAuth)
}
- tlsClientConfig, cleanup, err := TLSClientConfigFromSecret(secret)
+ tlsClientConfig, err := TLSClientConfigFromSecret(dir, secret)
if err != nil {
- return opts, nil, err
+ return opts, err
}
if tlsClientConfig != nil {
opts = append(opts, tlsClientConfig)
}
- return opts, cleanup, nil
+ return opts, nil
}
// BasicAuthFromSecret attempts to construct a basic auth getter.Option for the
@@ -63,50 +62,65 @@ func BasicAuthFromSecret(secret corev1.Secret) (getter.Option, error) {
}
// TLSClientConfigFromSecret attempts to construct a TLS client config
-// getter.Option for the given v1.Secret. It returns the getter.Option and a
-// callback to remove the temporary TLS files.
+// getter.Option for the given v1.Secret, placing the required TLS config
+// related files in the given directory. It returns the getter.Option, or
+// an error.
//
// Secrets with no certFile, keyFile, AND caFile are ignored, if only a
// certBytes OR keyBytes is defined it returns an error.
-func TLSClientConfigFromSecret(secret corev1.Secret) (getter.Option, func(), error) {
+func TLSClientConfigFromSecret(dir string, secret corev1.Secret) (getter.Option, error) {
certBytes, keyBytes, caBytes := secret.Data["certFile"], secret.Data["keyFile"], secret.Data["caFile"]
switch {
case len(certBytes)+len(keyBytes)+len(caBytes) == 0:
- return nil, func() {}, nil
+ return nil, nil
case (len(certBytes) > 0 && len(keyBytes) == 0) || (len(keyBytes) > 0 && len(certBytes) == 0):
- return nil, nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence",
+ return nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence",
secret.Name)
}
- // create tmp dir for TLS files
- tmp, err := os.MkdirTemp("", "helm-tls-"+secret.Name)
- if err != nil {
- return nil, nil, err
- }
- cleanup := func() { os.RemoveAll(tmp) }
-
- var certFile, keyFile, caFile string
-
+ var certPath, keyPath, caPath string
if len(certBytes) > 0 && len(keyBytes) > 0 {
- certFile = filepath.Join(tmp, "cert.crt")
- if err := os.WriteFile(certFile, certBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ certFile, err := os.CreateTemp(dir, "cert-*.crt")
+ if err != nil {
+ return nil, err
+ }
+ if _, err = certFile.Write(certBytes); err != nil {
+ _ = certFile.Close()
+ return nil, err
}
- keyFile = filepath.Join(tmp, "key.crt")
- if err := os.WriteFile(keyFile, keyBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ if err = certFile.Close(); err != nil {
+ return nil, err
}
+ certPath = certFile.Name()
+
+ keyFile, err := os.CreateTemp(dir, "key-*.crt")
+ if err != nil {
+ return nil, err
+ }
+ if _, err = keyFile.Write(keyBytes); err != nil {
+ _ = keyFile.Close()
+ return nil, err
+ }
+ if err = keyFile.Close(); err != nil {
+ return nil, err
+ }
+ keyPath = keyFile.Name()
}
if len(caBytes) > 0 {
- caFile = filepath.Join(tmp, "ca.pem")
- if err := os.WriteFile(caFile, caBytes, 0644); err != nil {
- cleanup()
- return nil, nil, err
+ caFile, err := os.CreateTemp(dir, "ca-*.pem")
+ if err != nil {
+ return nil, err
+ }
+ if _, err = caFile.Write(caBytes); err != nil {
+ _ = caFile.Close()
+ return nil, err
+ }
+ if err = caFile.Close(); err != nil {
+ return nil, err
}
+ caPath = caFile.Name()
}
- return getter.WithTLSClientConfig(certFile, keyFile, caFile), cleanup, nil
+ return getter.WithTLSClientConfig(certPath, keyPath, caPath), nil
}
diff --git a/internal/helm/getter_test.go b/internal/helm/getter_test.go
index bd4e1058c..2c55e7cbb 100644
--- a/internal/helm/getter_test.go
+++ b/internal/helm/getter_test.go
@@ -17,6 +17,7 @@ limitations under the License.
package helm
import (
+ "os"
"testing"
corev1 "k8s.io/api/core/v1"
@@ -56,10 +57,14 @@ func TestClientOptionsFromSecret(t *testing.T) {
secret.Data[k] = v
}
}
- got, cleanup, err := ClientOptionsFromSecret(secret)
- if cleanup != nil {
- defer cleanup()
+
+ tmpDir, err := os.MkdirTemp("", "client-opts-secret-")
+ if err != nil {
+ t.Fatal(err)
}
+ defer os.RemoveAll(tmpDir)
+
+ got, err := ClientOptionsFromSecret(tmpDir, secret)
if err != nil {
t.Errorf("ClientOptionsFromSecret() error = %v", err)
return
@@ -123,10 +128,14 @@ func TestTLSClientConfigFromSecret(t *testing.T) {
if tt.modify != nil {
tt.modify(secret)
}
- got, cleanup, err := TLSClientConfigFromSecret(*secret)
- if cleanup != nil {
- defer cleanup()
+
+ tmpDir, err := os.MkdirTemp("", "client-opts-secret-")
+ if err != nil {
+ t.Fatal(err)
}
+ defer os.RemoveAll(tmpDir)
+
+ got, err := TLSClientConfigFromSecret(tmpDir, *secret)
if (err != nil) != tt.wantErr {
t.Errorf("TLSClientConfigFromSecret() error = %v, wantErr %v", err, tt.wantErr)
return
diff --git a/internal/helm/helm.go b/internal/helm/helm.go
new file mode 100644
index 000000000..ec9668542
--- /dev/null
+++ b/internal/helm/helm.go
@@ -0,0 +1,29 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helm
+
+// This list defines a set of global variables used to ensure Helm files loaded
+// into memory during runtime do not exceed defined upper bound limits.
+var (
+ // MaxIndexSize is the max allowed file size in bytes of a ChartRepository.
+ MaxIndexSize int64 = 50 << 20
+ // MaxChartSize is the max allowed file size in bytes of a Helm Chart.
+ MaxChartSize int64 = 2 << 20
+ // MaxChartFileSize is the max allowed file size in bytes of any arbitrary
+ // file originating from a chart.
+ MaxChartFileSize int64 = 2 << 10
+)
diff --git a/internal/helm/repository.go b/internal/helm/repository.go
index e2446f944..eb9e668a1 100644
--- a/internal/helm/repository.go
+++ b/internal/helm/repository.go
@@ -234,6 +234,16 @@ func (r *ChartRepository) LoadIndexFromBytes(b []byte) error {
// LoadFromFile reads the file at the given path and loads it into Index.
func (r *ChartRepository) LoadFromFile(path string) error {
+ stat, err := os.Stat(path)
+ if err != nil || stat.IsDir() {
+ if err == nil {
+ err = fmt.Errorf("'%s' is a directory", path)
+ }
+ return err
+ }
+ if stat.Size() > MaxIndexSize {
+ return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), MaxIndexSize)
+ }
b, err := os.ReadFile(path)
if err != nil {
return err
@@ -342,7 +352,7 @@ func (r *ChartRepository) HasCacheFile() bool {
// Unload can be used to signal the Go garbage collector the Index can
// be freed from memory if the ChartRepository object is expected to
// continue to exist in the stack for some time.
-func (r *ChartRepository) Unload() {
+func (r *ChartRepository) Unload() {
if r == nil {
return
}
diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go
index 0d2077dfd..9c124b791 100644
--- a/internal/helm/repository_test.go
+++ b/internal/helm/repository_test.go
@@ -416,7 +416,7 @@ func TestChartRepository_LoadFromCache(t *testing.T) {
{
name: "invalid cache path",
cachePath: "invalid",
- wantErr: "open invalid: no such file",
+ wantErr: "stat invalid: no such file",
},
{
name: "no cache path",
From 9abbdd80a6eda9d47a7632328237e16def550a1f Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Fri, 5 Nov 2021 15:29:40 +0100
Subject: [PATCH 0227/1397] controllers: rough wiring of Helm chart builder
This commit starts wiring the factored out Helm chart build logic into
the reconciler to ensure, validating the API capabilities.
Signed-off-by: Hidde Beydals
---
controllers/helmchart_controller.go | 509 +++++++----------------
controllers/helmchart_controller_test.go | 1 +
controllers/helmrepository_controller.go | 11 +-
3 files changed, 160 insertions(+), 361 deletions(-)
diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go
index 5d4f952cd..bcb8f8e79 100644
--- a/controllers/helmchart_controller.go
+++ b/controllers/helmchart_controller.go
@@ -19,7 +19,6 @@ package controllers
import (
"context"
"fmt"
- "io"
"net/url"
"os"
"path/filepath"
@@ -27,14 +26,11 @@ import (
"strings"
"time"
- "github.com/Masterminds/semver/v3"
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/go-logr/logr"
- helmchart "helm.sh/helm/v3/pkg/chart"
- "helm.sh/helm/v3/pkg/chart/loader"
- "helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -50,13 +46,11 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
- "sigs.k8s.io/yaml"
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/events"
"github.com/fluxcd/pkg/runtime/metrics"
"github.com/fluxcd/pkg/runtime/predicates"
- "github.com/fluxcd/pkg/runtime/transform"
"github.com/fluxcd/pkg/untar"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
@@ -202,6 +196,19 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{Requeue: true}, err
}
+ // Create working directory
+ workDir, err := os.MkdirTemp("", chart.Kind + "-" + chart.Namespace + "-" + chart.Name + "-")
+ if err != nil {
+ err = fmt.Errorf("failed to create temporary working directory: %w", err)
+ chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error())
+ if err := r.updateStatus(ctx, req, chart.Status); err != nil {
+ log.Error(err, "unable to update status")
+ }
+ r.recordReadiness(ctx, chart)
+ return ctrl.Result{Requeue: true}, err
+ }
+ defer os.RemoveAll(workDir)
+
// Perform the reconciliation for the chart source type
var reconciledChart sourcev1.HelmChart
var reconcileErr error
@@ -222,10 +229,10 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
// Do not requeue as there is no chance on recovery.
return ctrl.Result{Requeue: false}, nil
}
- reconciledChart, reconcileErr = r.reconcileFromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), changed)
+ reconciledChart, reconcileErr = r.fromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), workDir, changed)
case *sourcev1.GitRepository, *sourcev1.Bucket:
- reconciledChart, reconcileErr = r.reconcileFromTarballArtifact(ctx, *typedSource.GetArtifact(),
- *chart.DeepCopy(), changed)
+ reconciledChart, reconcileErr = r.fromTarballArtifact(ctx, *typedSource.GetArtifact(), *chart.DeepCopy(),
+ workDir, changed)
default:
err := fmt.Errorf("unable to reconcile unsupported source reference kind '%s'", chart.Spec.SourceRef.Kind)
return ctrl.Result{Requeue: false}, err
@@ -297,8 +304,8 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm
return source, nil
}
-func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
- repository sourcev1.HelmRepository, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) {
+func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repository sourcev1.HelmRepository,
+ chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) {
// Configure ChartRepository getter options
clientOpts := []getter.Option{
getter.WithURL(repository.Spec.URL),
@@ -308,17 +315,21 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil {
return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
} else if secret != nil {
- opts, cleanup, err := helm.ClientOptionsFromSecret(*secret)
+ // Create temporary working directory for credentials
+ authDir := filepath.Join(workDir, "creds")
+ if err := os.Mkdir(authDir, 0700); err != nil {
+ err = fmt.Errorf("failed to create temporary directory for repository credentials: %w", err)
+ }
+ opts, err := helm.ClientOptionsFromSecret(authDir, *secret)
if err != nil {
- err = fmt.Errorf("auth options error: %w", err)
+ err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repository.Name, err)
return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
}
- defer cleanup()
clientOpts = append(clientOpts, opts...)
}
- // Initialize the chart repository and load the index file
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
+ // Initialize the chart repository
+ chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Storage.LocalPath(*repository.GetArtifact()), r.Getters, clientOpts)
if err != nil {
switch err.(type) {
case *url.Error:
@@ -327,29 +338,33 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
}
}
- indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- b, err := io.ReadAll(indexFile)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- if err = chartRepo.LoadIndex(b); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
+
+ var cachedChart string
+ if artifact := chart.GetArtifact(); artifact != nil {
+ cachedChart = artifact.Path
}
- // Lookup the chart version in the chart repository index
- chartVer, err := chartRepo.Get(chart.Spec.Chart, chart.Spec.Version)
+ // Build the chart
+ cBuilder := helm.NewRemoteChartBuilder(chartRepo)
+ ref := helm.RemoteChartReference{Name: chart.Spec.Chart, Version: chart.Spec.Version}
+ opts := helm.BuildOptions{
+ ValueFiles: chart.GetValuesFiles(),
+ CachedChart: cachedChart,
+ Force: force,
+ }
+ build, err := cBuilder.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts)
if err != nil {
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
}
- // Return early if the revision is still the same as the current artifact
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), chartVer.Version,
- fmt.Sprintf("%s-%s.tgz", chartVer.Name, chartVer.Version))
- if !force && repository.GetArtifact().HasRevision(newArtifact.Revision) {
- if newArtifact.URL != chart.GetArtifact().URL {
+ newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version,
+ fmt.Sprintf("%s-%s.tgz", build.Name, build.Version))
+
+ // If the path of the returned build equals the cache path,
+ // there are no changes to the chart
+ if build.Path == cachedChart {
+ // Ensure hostname is updated
+ if chart.GetArtifact().URL != newArtifact.URL {
r.Storage.SetArtifactURL(chart.GetArtifact())
chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
}
@@ -371,362 +386,106 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
}
defer unlock()
- // Attempt to download the chart
- res, err := chartRepo.DownloadChart(chartVer)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- defer os.RemoveAll(tmpFile.Name())
- if _, err = io.Copy(tmpFile, res); err != nil {
- tmpFile.Close()
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- tmpFile.Close()
-
- // Check if we need to repackage the chart with the declared defaults files.
- var (
- pkgPath = tmpFile.Name()
- readyReason = sourcev1.ChartPullSucceededReason
- readyMessage = fmt.Sprintf("Fetched revision: %s", newArtifact.Revision)
- )
-
- switch {
- case len(chart.GetValuesFiles()) > 0:
- valuesMap := make(map[string]interface{})
-
- // Load the chart
- helmChart, err := loader.LoadFile(pkgPath)
- if err != nil {
- err = fmt.Errorf("load chart error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- for _, v := range chart.GetValuesFiles() {
- if v == "values.yaml" {
- valuesMap = transform.MergeMaps(valuesMap, helmChart.Values)
- continue
- }
-
- var valuesData []byte
- cfn := filepath.Clean(v)
- for _, f := range helmChart.Files {
- if f.Name == cfn {
- valuesData = f.Data
- break
- }
- }
- if valuesData == nil {
- err = fmt.Errorf("invalid values file path: %s", v)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- yamlMap := make(map[string]interface{})
- err = yaml.Unmarshal(valuesData, &yamlMap)
- if err != nil {
- err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesMap = transform.MergeMaps(valuesMap, yamlMap)
- }
-
- yamlBytes, err := yaml.Marshal(valuesMap)
- if err != nil {
- err = fmt.Errorf("marshaling values failed: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
-
- // Overwrite values file
- if changed, err := helm.OverwriteChartDefaultValues(helmChart, yamlBytes); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- } else if !changed {
- break
- }
-
- // Create temporary working directory
- tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- defer os.RemoveAll(tmpDir)
-
- // Package the chart with the new default values
- pkgPath, err = chartutil.Save(helmChart, tmpDir)
- if err != nil {
- err = fmt.Errorf("chart package error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
-
- // Copy the packaged chart to the artifact path
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
- err = fmt.Errorf("failed to write chart package to storage: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- readyMessage = fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision)
- readyReason = sourcev1.ChartPackageSucceededReason
- }
-
- // Write artifact to storage
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
- err = fmt.Errorf("unable to write chart file: %w", err)
+ // Copy the packaged chart to the artifact path
+ if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil {
+ err = fmt.Errorf("failed to write chart package to storage: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Update symlink
- chartUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chartVer.Name))
+ cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name))
if err != nil {
err = fmt.Errorf("storage error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
-
- return sourcev1.HelmChartReady(chart, newArtifact, chartUrl, readyReason, readyMessage), nil
+ return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil
}
-func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context,
- artifact sourcev1.Artifact, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) {
- // Create temporary working directory
- tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name))
- if err != nil {
- err = fmt.Errorf("tmp dir error: %w", err)
+func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact,
+ chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) {
+ // Create temporary working directory to untar into
+ sourceDir := filepath.Join(workDir, "source")
+ if err := os.Mkdir(sourceDir, 0700); err != nil {
+ err = fmt.Errorf("failed to create temporary directory to untar source into: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- defer os.RemoveAll(tmpDir)
// Open the tarball artifact file and untar files into working directory
- f, err := os.Open(r.Storage.LocalPath(artifact))
+ f, err := os.Open(r.Storage.LocalPath(source))
if err != nil {
err = fmt.Errorf("artifact open error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- if _, err = untar.Untar(f, tmpDir); err != nil {
- f.Close()
+ if _, err = untar.Untar(f, sourceDir); err != nil {
+ _ = f.Close()
err = fmt.Errorf("artifact untar error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- f.Close()
-
- // Load the chart
- chartPath, err := securejoin.SecureJoin(tmpDir, chart.Spec.Chart)
- if err != nil {
+ if err =f.Close(); err != nil {
+ err = fmt.Errorf("artifact close error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- chartFileInfo, err := os.Stat(chartPath)
+
+ chartPath, err := securejoin.SecureJoin(sourceDir, chart.Spec.Chart)
if err != nil {
- err = fmt.Errorf("chart location read error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- helmChart, err := loader.Load(chartPath)
- if err != nil {
- err = fmt.Errorf("load chart error: %w", err)
+
+ // Setup dependency manager
+ authDir := filepath.Join(workDir, "creds")
+ if err = os.Mkdir(authDir, 0700); err != nil {
+ err = fmt.Errorf("failed to create temporaRy directory for dependency credentials: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
+ dm := helm.NewDependencyManager(
+ helm.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, chart.GetNamespace())),
+ )
+ defer dm.Clear()
- v, err := semver.NewVersion(helmChart.Metadata.Version)
- if err != nil {
- err = fmt.Errorf("semver parse error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ // Get any cached chart
+ var cachedChart string
+ if artifact := chart.Status.Artifact; artifact != nil {
+ cachedChart = artifact.Path
+ }
+
+ buildsOpts := helm.BuildOptions{
+ ValueFiles: chart.GetValuesFiles(),
+ CachedChart: cachedChart,
+ Force: force,
}
- version := v.String()
+ // Add revision metadata to chart build
if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision {
// Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix.
- splitRev := strings.Split(artifact.Revision, "/")
- v, err := v.SetMetadata(splitRev[len(splitRev)-1])
- if err != nil {
- err = fmt.Errorf("semver parse error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
+ splitRev := strings.Split(source.Revision, "/")
+ buildsOpts.VersionMetadata = splitRev[len(splitRev)-1]
+ }
- version = v.String()
- helmChart.Metadata.Version = v.String()
+ // Build chart
+ chartB := helm.NewLocalChartBuilder(dm)
+ build, err := chartB.Build(ctx, helm.LocalChartReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts)
+ if err != nil {
+ return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
}
- // Return early if the revision is still the same as the current chart artifact
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), version,
- fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, version))
- if !force && apimeta.IsStatusConditionTrue(chart.Status.Conditions, meta.ReadyCondition) && chart.GetArtifact().HasRevision(newArtifact.Revision) {
- if newArtifact.URL != artifact.URL {
+ newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version,
+ fmt.Sprintf("%s-%s.tgz", build.Name, build.Version))
+
+ // If the path of the returned build equals the cache path,
+ // there are no changes to the chart
+ if build.Path == cachedChart {
+ // Ensure hostname is updated
+ if chart.GetArtifact().URL != newArtifact.URL {
r.Storage.SetArtifactURL(chart.GetArtifact())
chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
}
return chart, nil
}
- // Either (re)package the chart with the declared default values file,
- // or write the chart directly to storage.
- pkgPath := chartPath
- isValuesFileOverriden := false
- if len(chart.GetValuesFiles()) > 0 {
- valuesMap := make(map[string]interface{})
- for _, v := range chart.GetValuesFiles() {
- srcPath, err := securejoin.SecureJoin(tmpDir, v)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- if f, err := os.Stat(srcPath); os.IsNotExist(err) || !f.Mode().IsRegular() {
- err = fmt.Errorf("invalid values file path: %s", v)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesData, err := os.ReadFile(srcPath)
- if err != nil {
- err = fmt.Errorf("failed to read from values file '%s': %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- yamlMap := make(map[string]interface{})
- err = yaml.Unmarshal(valuesData, &yamlMap)
- if err != nil {
- err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
-
- valuesMap = transform.MergeMaps(valuesMap, yamlMap)
- }
-
- yamlBytes, err := yaml.Marshal(valuesMap)
- if err != nil {
- err = fmt.Errorf("marshaling values failed: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
-
- isValuesFileOverriden, err = helm.OverwriteChartDefaultValues(helmChart, yamlBytes)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
- }
-
- isDir := chartFileInfo.IsDir()
- switch {
- case isDir:
- // Determine chart dependencies
- deps := helmChart.Dependencies()
- reqs := helmChart.Metadata.Dependencies
- lock := helmChart.Lock
- if lock != nil {
- // Load from lockfile if exists
- reqs = lock.Dependencies
- }
- var dwr []*helm.DependencyWithRepository
- for _, dep := range reqs {
- // Exclude existing dependencies
- found := false
- for _, existing := range deps {
- if existing.Name() == dep.Name {
- found = true
- }
- }
- if found {
- continue
- }
-
- // Continue loop if file scheme detected
- if dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") {
- dwr = append(dwr, &helm.DependencyWithRepository{
- Dependency: dep,
- Repository: nil,
- })
- continue
- }
-
- // Discover existing HelmRepository by URL
- repository, err := r.resolveDependencyRepository(ctx, dep, chart.Namespace)
- if err != nil {
- repository = &sourcev1.HelmRepository{
- Spec: sourcev1.HelmRepositorySpec{
- URL: dep.Repository,
- Timeout: &metav1.Duration{Duration: 60 * time.Second},
- },
- }
- }
-
- // Configure ChartRepository getter options
- clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
- }
- if secret, err := r.getHelmRepositorySecret(ctx, repository); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
- } else if secret != nil {
- opts, cleanup, err := helm.ClientOptionsFromSecret(*secret)
- if err != nil {
- err = fmt.Errorf("auth options error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
- }
- defer cleanup()
- clientOpts = append(clientOpts, opts...)
- }
-
- // Initialize the chart repository and load the index file
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts)
- if err != nil {
- switch err.(type) {
- case *url.Error:
- return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err
- default:
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- }
- if repository.Status.Artifact != nil {
- indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- b, err := io.ReadAll(indexFile)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- if err = chartRepo.LoadIndex(b); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- } else {
- // Download index
- err = chartRepo.DownloadIndex()
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
- }
- }
-
- dwr = append(dwr, &helm.DependencyWithRepository{
- Dependency: dep,
- Repository: chartRepo,
- })
- }
-
- // Construct dependencies for chart if any
- if len(dwr) > 0 {
- dm := &helm.DependencyManager{
- WorkingDir: tmpDir,
- ChartPath: chart.Spec.Chart,
- Chart: helmChart,
- Dependencies: dwr,
- }
- err = dm.Build(ctx)
- if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- }
-
- fallthrough
- case isValuesFileOverriden:
- pkgPath, err = chartutil.Save(helmChart, tmpDir)
- if err != nil {
- err = fmt.Errorf("chart package error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
- }
- }
-
// Ensure artifact directory exists
err = r.Storage.MkdirAll(newArtifact)
if err != nil {
- err = fmt.Errorf("unable to create artifact directory: %w", err)
+ err = fmt.Errorf("unable to create chart directory: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
@@ -739,20 +498,59 @@ func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context,
defer unlock()
// Copy the packaged chart to the artifact path
- if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil {
+ if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil {
err = fmt.Errorf("failed to write chart package to storage: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Update symlink
- cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", helmChart.Metadata.Name))
+ cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chart.Name))
if err != nil {
err = fmt.Errorf("storage error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- message := fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision)
- return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, message), nil
+ return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil
+}
+
+// TODO(hidde): factor out to helper?
+func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) helm.GetChartRepositoryCallback {
+ return func(url string) (*helm.ChartRepository, error) {
+ repo, err := r.resolveDependencyRepository(ctx, url, namespace)
+ if err != nil {
+ if errors.ReasonForError(err) != metav1.StatusReasonUnknown {
+ return nil, err
+ }
+ repo = &sourcev1.HelmRepository{
+ Spec: sourcev1.HelmRepositorySpec{
+ URL: url,
+ Timeout: &metav1.Duration{Duration: 60 * time.Second},
+ },
+ }
+ }
+ clientOpts := []getter.Option{
+ getter.WithURL(repo.Spec.URL),
+ getter.WithTimeout(repo.Spec.Timeout.Duration),
+ getter.WithPassCredentialsAll(repo.Spec.PassCredentials),
+ }
+ if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil {
+ return nil, err
+ } else if secret != nil {
+ opts, err := helm.ClientOptionsFromSecret(dir, *secret)
+ if err != nil {
+ return nil, err
+ }
+ clientOpts = append(clientOpts, opts...)
+ }
+ chartRepo, err := helm.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts)
+ if err != nil {
+ return nil, err
+ }
+ if repo.Status.Artifact != nil {
+ chartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact())
+ }
+ return chartRepo, nil
+ }
}
func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, chart sourcev1.HelmChart) (ctrl.Result, error) {
@@ -880,15 +678,10 @@ func (r *HelmChartReconciler) indexHelmChartBySource(o client.Object) []string {
return []string{fmt.Sprintf("%s/%s", hc.Spec.SourceRef.Kind, hc.Spec.SourceRef.Name)}
}
-func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, dep *helmchart.Dependency, namespace string) (*sourcev1.HelmRepository, error) {
- u := helm.NormalizeChartRepositoryURL(dep.Repository)
- if u == "" {
- return nil, fmt.Errorf("invalid repository URL")
- }
-
+func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, url string, namespace string) (*sourcev1.HelmRepository, error) {
listOpts := []client.ListOption{
client.InNamespace(namespace),
- client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: u},
+ client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: url},
}
var list sourcev1.HelmRepositoryList
err := r.Client.List(ctx, &list, listOpts...)
@@ -898,8 +691,7 @@ func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, d
if len(list.Items) > 0 {
return &list.Items[0], nil
}
-
- return nil, fmt.Errorf("no HelmRepository found")
+ return nil, fmt.Errorf("no HelmRepository found for '%s' in '%s' namespace", url, namespace)
}
func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repository *sourcev1.HelmRepository) (*corev1.Secret, error) {
@@ -917,7 +709,6 @@ func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repos
}
return &secret, nil
}
-
return nil, nil
}
diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go
index 35462d467..ceb30842f 100644
--- a/controllers/helmchart_controller_test.go
+++ b/controllers/helmchart_controller_test.go
@@ -732,6 +732,7 @@ var _ = Describe("HelmChartReconciler", func() {
}, timeout, interval).Should(BeTrue())
helmChart, err := loader.Load(storage.LocalPath(*now.Status.Artifact))
Expect(err).NotTo(HaveOccurred())
+ Expect(helmChart.Values).ToNot(BeNil())
Expect(helmChart.Values["testDefault"]).To(BeTrue())
Expect(helmChart.Values["testOverride"]).To(BeFalse())
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index d7fb57e58..794a912e3 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"net/url"
+ "os"
"time"
"github.com/fluxcd/pkg/apis/meta"
@@ -186,12 +187,18 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}
- opts, cleanup, err := helm.ClientOptionsFromSecret(secret)
+ authDir, err := os.MkdirTemp("", "helm-repository-")
+ if err != nil {
+ err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err)
+ return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ }
+ defer os.RemoveAll(authDir)
+
+ opts, err := helm.ClientOptionsFromSecret(authDir, secret)
if err != nil {
err = fmt.Errorf("auth options error: %w", err)
return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}
- defer cleanup()
clientOpts = append(clientOpts, opts...)
}
From 7d0f79f41b84efea5d6b0fd6cab64ea58daf149d Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Mon, 15 Nov 2021 22:31:33 +0100
Subject: [PATCH 0228/1397] internal/helm: divide into subpackages
With all the logic that used to reside in the `controllers` package
factored into this package, it became cluttered. This commit tries to
bring a bit more structure in place.
Signed-off-by: Hidde Beydals
---
controllers/helmchart_controller.go | 146 +++++++++---------
controllers/helmrepository_controller.go | 78 +++++-----
.../{chart_builder.go => chart/builder.go} | 66 ++++----
.../builder_local.go} | 21 +--
.../builder_local_test.go} | 8 +-
.../builder_remote.go} | 25 +--
.../builder_remote_test.go} | 8 +-
.../builder_test.go} | 10 +-
.../helm/{ => chart}/dependency_manager.go | 61 ++++----
.../{ => chart}/dependency_manager_test.go | 85 ++++------
internal/helm/{chart.go => chart/metadata.go} | 12 +-
.../{chart_test.go => chart/metadata_test.go} | 27 +++-
internal/helm/{ => getter}/getter.go | 2 +-
internal/helm/{ => getter}/getter_test.go | 2 +-
internal/helm/getter/mock.go | 41 +++++
.../chart_repository.go} | 10 +-
.../chart_repository_test.go} | 46 +++---
internal/helm/{ => repository}/utils.go | 9 +-
internal/helm/repository/utils_test.go | 44 ++++++
internal/helm/utils_test.go | 60 -------
20 files changed, 397 insertions(+), 364 deletions(-)
rename internal/helm/{chart_builder.go => chart/builder.go} (70%)
rename internal/helm/{chart_builder_local.go => chart/builder_local.go} (90%)
rename internal/helm/{chart_builder_local_test.go => chart/builder_local_test.go} (96%)
rename internal/helm/{chart_builder_remote.go => chart/builder_remote.go} (91%)
rename internal/helm/{chart_builder_remote_test.go => chart/builder_remote_test.go} (92%)
rename internal/helm/{chart_builder_test.go => chart/builder_test.go} (89%)
rename internal/helm/{ => chart}/dependency_manager.go (81%)
rename internal/helm/{ => chart}/dependency_manager_test.go (84%)
rename internal/helm/{chart.go => chart/metadata.go} (96%)
rename internal/helm/{chart_test.go => chart/metadata_test.go} (85%)
rename internal/helm/{ => getter}/getter.go (99%)
rename internal/helm/{ => getter}/getter_test.go (99%)
create mode 100644 internal/helm/getter/mock.go
rename internal/helm/{repository.go => repository/chart_repository.go} (98%)
rename internal/helm/{repository_test.go => repository/chart_repository_test.go} (93%)
rename internal/helm/{ => repository}/utils.go (77%)
create mode 100644 internal/helm/repository/utils_test.go
delete mode 100644 internal/helm/utils_test.go
diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go
index bcb8f8e79..d31f6c2bb 100644
--- a/controllers/helmchart_controller.go
+++ b/controllers/helmchart_controller.go
@@ -28,7 +28,7 @@ import (
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/go-logr/logr"
- "helm.sh/helm/v3/pkg/getter"
+ extgetter "helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -54,7 +54,9 @@ import (
"github.com/fluxcd/pkg/untar"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
- "github.com/fluxcd/source-controller/internal/helm"
+ "github.com/fluxcd/source-controller/internal/helm/chart"
+ "github.com/fluxcd/source-controller/internal/helm/getter"
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmcharts,verbs=get;list;watch;create;update;patch;delete
@@ -67,7 +69,7 @@ type HelmChartReconciler struct {
client.Client
Scheme *runtime.Scheme
Storage *Storage
- Getters getter.Providers
+ Getters extgetter.Providers
EventRecorder kuberecorder.EventRecorder
ExternalEventRecorder *events.Recorder
MetricsRecorder *metrics.Recorder
@@ -304,218 +306,218 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm
return source, nil
}
-func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repository sourcev1.HelmRepository,
- chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) {
- // Configure ChartRepository getter options
- clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
+func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourcev1.HelmRepository, c sourcev1.HelmChart,
+ workDir string, force bool) (sourcev1.HelmChart, error) {
+ // Configure Index getter options
+ clientOpts := []extgetter.Option{
+ extgetter.WithURL(repo.Spec.URL),
+ extgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
- if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
+ if secret, err := r.getHelmRepositorySecret(ctx, &repo); err != nil {
+ return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err
} else if secret != nil {
// Create temporary working directory for credentials
authDir := filepath.Join(workDir, "creds")
if err := os.Mkdir(authDir, 0700); err != nil {
err = fmt.Errorf("failed to create temporary directory for repository credentials: %w", err)
}
- opts, err := helm.ClientOptionsFromSecret(authDir, *secret)
+ opts, err := getter.ClientOptionsFromSecret(authDir, *secret)
if err != nil {
- err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repository.Name, err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err
+ err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repo.Name, err)
+ return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err
}
clientOpts = append(clientOpts, opts...)
}
// Initialize the chart repository
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Storage.LocalPath(*repository.GetArtifact()), r.Getters, clientOpts)
+ chartRepo, err := repository.NewChartRepository(repo.Spec.URL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, clientOpts)
if err != nil {
switch err.(type) {
case *url.Error:
- return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.URLInvalidReason, err.Error()), err
default:
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err
}
}
var cachedChart string
- if artifact := chart.GetArtifact(); artifact != nil {
+ if artifact := c.GetArtifact(); artifact != nil {
cachedChart = artifact.Path
}
// Build the chart
- cBuilder := helm.NewRemoteChartBuilder(chartRepo)
- ref := helm.RemoteChartReference{Name: chart.Spec.Chart, Version: chart.Spec.Version}
- opts := helm.BuildOptions{
- ValueFiles: chart.GetValuesFiles(),
+ cBuilder := chart.NewRemoteBuilder(chartRepo)
+ ref := chart.RemoteReference{Name: c.Spec.Chart, Version: c.Spec.Version}
+ opts := chart.BuildOptions{
+ ValueFiles: c.GetValuesFiles(),
CachedChart: cachedChart,
Force: force,
}
build, err := cBuilder.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts)
if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err
}
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version,
+ newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version,
fmt.Sprintf("%s-%s.tgz", build.Name, build.Version))
// If the path of the returned build equals the cache path,
// there are no changes to the chart
if build.Path == cachedChart {
// Ensure hostname is updated
- if chart.GetArtifact().URL != newArtifact.URL {
- r.Storage.SetArtifactURL(chart.GetArtifact())
- chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
+ if c.GetArtifact().URL != newArtifact.URL {
+ r.Storage.SetArtifactURL(c.GetArtifact())
+ c.Status.URL = r.Storage.SetHostname(c.Status.URL)
}
- return chart, nil
+ return c, nil
}
// Ensure artifact directory exists
err = r.Storage.MkdirAll(newArtifact)
if err != nil {
err = fmt.Errorf("unable to create chart directory: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Acquire a lock for the artifact
unlock, err := r.Storage.Lock(newArtifact)
if err != nil {
err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer unlock()
// Copy the packaged chart to the artifact path
if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil {
err = fmt.Errorf("failed to write chart package to storage: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Update symlink
cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name))
if err != nil {
err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil
+ return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil
}
-func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact,
- chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) {
+func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, c sourcev1.HelmChart,
+ workDir string, force bool) (sourcev1.HelmChart, error) {
// Create temporary working directory to untar into
sourceDir := filepath.Join(workDir, "source")
if err := os.Mkdir(sourceDir, 0700); err != nil {
err = fmt.Errorf("failed to create temporary directory to untar source into: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Open the tarball artifact file and untar files into working directory
f, err := os.Open(r.Storage.LocalPath(source))
if err != nil {
err = fmt.Errorf("artifact open error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
if _, err = untar.Untar(f, sourceDir); err != nil {
_ = f.Close()
err = fmt.Errorf("artifact untar error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
if err =f.Close(); err != nil {
err = fmt.Errorf("artifact close error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- chartPath, err := securejoin.SecureJoin(sourceDir, chart.Spec.Chart)
+ chartPath, err := securejoin.SecureJoin(sourceDir, c.Spec.Chart)
if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Setup dependency manager
authDir := filepath.Join(workDir, "creds")
if err = os.Mkdir(authDir, 0700); err != nil {
err = fmt.Errorf("failed to create temporaRy directory for dependency credentials: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- dm := helm.NewDependencyManager(
- helm.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, chart.GetNamespace())),
+ dm := chart.NewDependencyManager(
+ chart.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())),
)
defer dm.Clear()
// Get any cached chart
var cachedChart string
- if artifact := chart.Status.Artifact; artifact != nil {
+ if artifact := c.Status.Artifact; artifact != nil {
cachedChart = artifact.Path
}
- buildsOpts := helm.BuildOptions{
- ValueFiles: chart.GetValuesFiles(),
+ buildsOpts := chart.BuildOptions{
+ ValueFiles: c.GetValuesFiles(),
CachedChart: cachedChart,
Force: force,
}
// Add revision metadata to chart build
- if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision {
+ if c.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision {
// Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix.
splitRev := strings.Split(source.Revision, "/")
buildsOpts.VersionMetadata = splitRev[len(splitRev)-1]
}
// Build chart
- chartB := helm.NewLocalChartBuilder(dm)
- build, err := chartB.Build(ctx, helm.LocalChartReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts)
+ chartB := chart.NewLocalBuilder(dm)
+ build, err := chartB.Build(ctx, chart.LocalReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts)
if err != nil {
- return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.ChartPackageFailedReason, err.Error()), err
}
- newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version,
+ newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version,
fmt.Sprintf("%s-%s.tgz", build.Name, build.Version))
// If the path of the returned build equals the cache path,
// there are no changes to the chart
if build.Path == cachedChart {
// Ensure hostname is updated
- if chart.GetArtifact().URL != newArtifact.URL {
- r.Storage.SetArtifactURL(chart.GetArtifact())
- chart.Status.URL = r.Storage.SetHostname(chart.Status.URL)
+ if c.GetArtifact().URL != newArtifact.URL {
+ r.Storage.SetArtifactURL(c.GetArtifact())
+ c.Status.URL = r.Storage.SetHostname(c.Status.URL)
}
- return chart, nil
+ return c, nil
}
// Ensure artifact directory exists
err = r.Storage.MkdirAll(newArtifact)
if err != nil {
err = fmt.Errorf("unable to create chart directory: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Acquire a lock for the artifact
unlock, err := r.Storage.Lock(newArtifact)
if err != nil {
err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer unlock()
// Copy the packaged chart to the artifact path
if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil {
err = fmt.Errorf("failed to write chart package to storage: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Update symlink
- cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chart.Name))
+ cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name))
if err != nil {
err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil
+ return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil
}
// TODO(hidde): factor out to helper?
-func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) helm.GetChartRepositoryCallback {
- return func(url string) (*helm.ChartRepository, error) {
+func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback {
+ return func(url string) (*repository.ChartRepository, error) {
repo, err := r.resolveDependencyRepository(ctx, url, namespace)
if err != nil {
if errors.ReasonForError(err) != metav1.StatusReasonUnknown {
@@ -528,21 +530,21 @@ func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.C
},
}
}
- clientOpts := []getter.Option{
- getter.WithURL(repo.Spec.URL),
- getter.WithTimeout(repo.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repo.Spec.PassCredentials),
+ clientOpts := []extgetter.Option{
+ extgetter.WithURL(repo.Spec.URL),
+ extgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil {
return nil, err
} else if secret != nil {
- opts, err := helm.ClientOptionsFromSecret(dir, *secret)
+ opts, err := getter.ClientOptionsFromSecret(dir, *secret)
if err != nil {
return nil, err
}
clientOpts = append(clientOpts, opts...)
}
- chartRepo, err := helm.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts)
+ chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts)
if err != nil {
return nil, err
}
@@ -663,7 +665,7 @@ func (r *HelmChartReconciler) indexHelmRepositoryByURL(o client.Object) []string
if !ok {
panic(fmt.Sprintf("Expected a HelmRepository, got %T", o))
}
- u := helm.NormalizeChartRepositoryURL(repo.Spec.URL)
+ u := repository.NormalizeURL(repo.Spec.URL)
if u != "" {
return []string{u}
}
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index 794a912e3..8ab87201d 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -23,12 +23,8 @@ import (
"os"
"time"
- "github.com/fluxcd/pkg/apis/meta"
- "github.com/fluxcd/pkg/runtime/events"
- "github.com/fluxcd/pkg/runtime/metrics"
- "github.com/fluxcd/pkg/runtime/predicates"
"github.com/go-logr/logr"
- "helm.sh/helm/v3/pkg/getter"
+ extgetter "helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,8 +38,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/predicate"
+ "github.com/fluxcd/pkg/apis/meta"
+ "github.com/fluxcd/pkg/runtime/events"
+ "github.com/fluxcd/pkg/runtime/metrics"
+ "github.com/fluxcd/pkg/runtime/predicates"
+
+ "github.com/fluxcd/source-controller/internal/helm/getter"
+ "github.com/fluxcd/source-controller/internal/helm/repository"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
- "github.com/fluxcd/source-controller/internal/helm"
)
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=get;list;watch;create;update;patch;delete
@@ -56,7 +58,7 @@ type HelmRepositoryReconciler struct {
client.Client
Scheme *runtime.Scheme
Storage *Storage
- Getters getter.Providers
+ Getters extgetter.Providers
EventRecorder kuberecorder.EventRecorder
ExternalEventRecorder *events.Recorder
MetricsRecorder *metrics.Recorder
@@ -168,74 +170,74 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil
}
-func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.HelmRepository) (sourcev1.HelmRepository, error) {
- clientOpts := []getter.Option{
- getter.WithURL(repository.Spec.URL),
- getter.WithTimeout(repository.Spec.Timeout.Duration),
- getter.WithPassCredentialsAll(repository.Spec.PassCredentials),
+func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.HelmRepository) (sourcev1.HelmRepository, error) {
+ clientOpts := []extgetter.Option{
+ extgetter.WithURL(repo.Spec.URL),
+ extgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
- if repository.Spec.SecretRef != nil {
+ if repo.Spec.SecretRef != nil {
name := types.NamespacedName{
- Namespace: repository.GetNamespace(),
- Name: repository.Spec.SecretRef.Name,
+ Namespace: repo.GetNamespace(),
+ Name: repo.Spec.SecretRef.Name,
}
var secret corev1.Secret
err := r.Client.Get(ctx, name, &secret)
if err != nil {
err = fmt.Errorf("auth secret error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err
}
authDir, err := os.MkdirTemp("", "helm-repository-")
if err != nil {
err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err
}
defer os.RemoveAll(authDir)
- opts, err := helm.ClientOptionsFromSecret(authDir, secret)
+ opts, err := getter.ClientOptionsFromSecret(authDir, secret)
if err != nil {
err = fmt.Errorf("auth options error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err
}
clientOpts = append(clientOpts, opts...)
}
- chartRepo, err := helm.NewChartRepository(repository.Spec.URL, "", r.Getters, clientOpts)
+ chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts)
if err != nil {
switch err.(type) {
case *url.Error:
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.URLInvalidReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.URLInvalidReason, err.Error()), err
default:
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
}
}
revision, err := chartRepo.CacheIndex()
if err != nil {
err = fmt.Errorf("failed to download repository index: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
}
defer chartRepo.RemoveCache()
- artifact := r.Storage.NewArtifactFor(repository.Kind,
- repository.ObjectMeta.GetObjectMeta(),
+ artifact := r.Storage.NewArtifactFor(repo.Kind,
+ repo.ObjectMeta.GetObjectMeta(),
revision,
fmt.Sprintf("index-%s.yaml", revision))
// Return early on unchanged index
- if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) &&
- repository.GetArtifact().HasRevision(artifact.Revision) {
- if artifact.URL != repository.GetArtifact().URL {
- r.Storage.SetArtifactURL(repository.GetArtifact())
- repository.Status.URL = r.Storage.SetHostname(repository.Status.URL)
+ if apimeta.IsStatusConditionTrue(repo.Status.Conditions, meta.ReadyCondition) &&
+ repo.GetArtifact().HasRevision(artifact.Revision) {
+ if artifact.URL != repo.GetArtifact().URL {
+ r.Storage.SetArtifactURL(repo.GetArtifact())
+ repo.Status.URL = r.Storage.SetHostname(repo.Status.URL)
}
- return repository, nil
+ return repo, nil
}
// Load the cached repository index to ensure it passes validation
if err := chartRepo.LoadFromCache(); err != nil {
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
}
defer chartRepo.Unload()
@@ -243,14 +245,14 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
err = r.Storage.MkdirAll(artifact)
if err != nil {
err = fmt.Errorf("unable to create repository index directory: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// Acquire lock
unlock, err := r.Storage.Lock(artifact)
if err != nil {
err = fmt.Errorf("unable to acquire lock: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer unlock()
@@ -258,10 +260,10 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
storageTarget := r.Storage.LocalPath(artifact)
if storageTarget == "" {
err := fmt.Errorf("failed to calcalute local storage path to store artifact to")
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil {
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
// TODO(hidde): it would be better to make the Storage deal with this
artifact.Checksum = chartRepo.Checksum
@@ -271,11 +273,11 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
indexURL, err := r.Storage.Symlink(artifact, "index.yaml")
if err != nil {
err = fmt.Errorf("storage error: %w", err)
- return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
+ return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
message := fmt.Sprintf("Fetched revision: %s", artifact.Revision)
- return sourcev1.HelmRepositoryReady(repository, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil
+ return sourcev1.HelmRepositoryReady(repo, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil
}
func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.HelmRepository) (ctrl.Result, error) {
diff --git a/internal/helm/chart_builder.go b/internal/helm/chart/builder.go
similarity index 70%
rename from internal/helm/chart_builder.go
rename to internal/helm/chart/builder.go
index 4177983c6..3698d02c1 100644
--- a/internal/helm/chart_builder.go
+++ b/internal/helm/chart/builder.go
@@ -14,49 +14,51 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"context"
"fmt"
"os"
"path/filepath"
+ "regexp"
"strings"
- "github.com/fluxcd/source-controller/internal/fs"
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chartutil"
+
+ "github.com/fluxcd/source-controller/internal/fs"
)
-// ChartReference holds information to locate a chart.
-type ChartReference interface {
- // Validate returns an error if the ChartReference is not valid according
+// Reference holds information to locate a chart.
+type Reference interface {
+ // Validate returns an error if the Reference is not valid according
// to the spec of the interface implementation.
Validate() error
}
-// LocalChartReference contains sufficient information to locate a chart on the
+// LocalReference contains sufficient information to locate a chart on the
// local filesystem.
-type LocalChartReference struct {
- // BaseDir used as chroot during build operations.
+type LocalReference struct {
+ // WorkDir used as chroot during build operations.
// File references are not allowed to traverse outside it.
- BaseDir string
+ WorkDir string
// Path of the chart on the local filesystem.
Path string
}
-// Validate returns an error if the LocalChartReference does not have
+// Validate returns an error if the LocalReference does not have
// a Path set.
-func (r LocalChartReference) Validate() error {
+func (r LocalReference) Validate() error {
if r.Path == "" {
return fmt.Errorf("no path set for local chart reference")
}
return nil
}
-// RemoteChartReference contains sufficient information to look up a chart in
+// RemoteReference contains sufficient information to look up a chart in
// a ChartRepository.
-type RemoteChartReference struct {
+type RemoteReference struct {
// Name of the chart.
Name string
// Version of the chart.
@@ -64,25 +66,29 @@ type RemoteChartReference struct {
Version string
}
-// Validate returns an error if the RemoteChartReference does not have
+// Validate returns an error if the RemoteReference does not have
// a Name set.
-func (r RemoteChartReference) Validate() error {
+func (r RemoteReference) Validate() error {
if r.Name == "" {
return fmt.Errorf("no name set for remote chart reference")
}
+ name := regexp.MustCompile("^([-a-z0-9]*)$")
+ if !name.MatchString(r.Name) {
+ return fmt.Errorf("invalid chart name '%s': a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", r.Name)
+ }
return nil
}
-// ChartBuilder is capable of building a (specific) ChartReference.
-type ChartBuilder interface {
- // Build builds and packages a Helm chart with the given ChartReference
- // and BuildOptions and writes it to p. It returns the ChartBuild result,
- // or an error. It may return an error for unsupported ChartReference
+// Builder is capable of building a (specific) chart Reference.
+type Builder interface {
+ // Build builds and packages a Helm chart with the given Reference
+ // and BuildOptions and writes it to p. It returns the Build result,
+ // or an error. It may return an error for unsupported Reference
// implementations.
- Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error)
+ Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error)
}
-// BuildOptions provides a list of options for ChartBuilder.Build.
+// BuildOptions provides a list of options for Builder.Build.
type BuildOptions struct {
// VersionMetadata can be set to SemVer build metadata as defined in
// the spec, and is included during packaging.
@@ -109,9 +115,9 @@ func (o BuildOptions) GetValueFiles() []string {
return o.ValueFiles
}
-// ChartBuild contains the ChartBuilder.Build result, including specific
+// Build contains the Builder.Build result, including specific
// information about the built chart like ResolvedDependencies.
-type ChartBuild struct {
+type Build struct {
// Path is the absolute path to the packaged chart.
Path string
// Name of the packaged chart.
@@ -124,14 +130,14 @@ type ChartBuild struct {
// ResolvedDependencies is the number of local and remote dependencies
// collected by the DependencyManager before building the chart.
ResolvedDependencies int
- // Packaged indicates if the ChartBuilder has packaged the chart.
+ // Packaged indicates if the Builder has packaged the chart.
// This can for example be false if ValueFiles is empty and the chart
// source was already packaged.
Packaged bool
}
-// Summary returns a human-readable summary of the ChartBuild.
-func (b *ChartBuild) Summary() string {
+// Summary returns a human-readable summary of the Build.
+func (b *Build) Summary() string {
if b == nil {
return "no chart build"
}
@@ -155,15 +161,15 @@ func (b *ChartBuild) Summary() string {
return s.String()
}
-// String returns the Path of the ChartBuild.
-func (b *ChartBuild) String() string {
+// String returns the Path of the Build.
+func (b *Build) String() string {
if b != nil {
return b.Path
}
return ""
}
-// packageToPath attempts to package the given chart.Chart to the out filepath.
+// packageToPath attempts to package the given chart to the out filepath.
func packageToPath(chart *helmchart.Chart, out string) error {
o, err := os.MkdirTemp("", "chart-build-*")
if err != nil {
diff --git a/internal/helm/chart_builder_local.go b/internal/helm/chart/builder_local.go
similarity index 90%
rename from internal/helm/chart_builder_local.go
rename to internal/helm/chart/builder_local.go
index 13e5dbe9c..037a2fe18 100644
--- a/internal/helm/chart_builder_local.go
+++ b/internal/helm/chart/builder_local.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"context"
@@ -24,27 +24,28 @@ import (
"github.com/Masterminds/semver/v3"
securejoin "github.com/cyphar/filepath-securejoin"
- "github.com/fluxcd/pkg/runtime/transform"
"helm.sh/helm/v3/pkg/chart/loader"
"sigs.k8s.io/yaml"
+
+ "github.com/fluxcd/pkg/runtime/transform"
)
type localChartBuilder struct {
dm *DependencyManager
}
-// NewLocalChartBuilder returns a ChartBuilder capable of building a Helm
-// chart with a LocalChartReference. For chart references pointing to a
+// NewLocalBuilder returns a Builder capable of building a Helm
+// chart with a LocalReference. For chart references pointing to a
// directory, the DependencyManager is used to resolve missing local and
// remote dependencies.
-func NewLocalChartBuilder(dm *DependencyManager) ChartBuilder {
+func NewLocalBuilder(dm *DependencyManager) Builder {
return &localChartBuilder{
dm: dm,
}
}
-func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) {
- localRef, ok := ref.(LocalChartReference)
+func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) {
+ localRef, ok := ref.(LocalReference)
if !ok {
return nil, fmt.Errorf("expected local chart reference")
}
@@ -53,14 +54,14 @@ func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p str
return nil, err
}
- // Load the chart metadata from the LocalChartReference to ensure it points
+ // Load the chart metadata from the LocalReference to ensure it points
// to a chart
curMeta, err := LoadChartMetadata(localRef.Path)
if err != nil {
return nil, err
}
- result := &ChartBuild{}
+ result := &Build{}
result.Name = curMeta.Name
// Set build specific metadata if instructed
@@ -101,7 +102,7 @@ func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p str
// Merge chart values, if instructed
var mergedValues map[string]interface{}
if len(opts.GetValueFiles()) > 0 {
- if mergedValues, err = mergeFileValues(localRef.BaseDir, opts.ValueFiles); err != nil {
+ if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValueFiles); err != nil {
return nil, fmt.Errorf("failed to merge value files: %w", err)
}
}
diff --git a/internal/helm/chart_builder_local_test.go b/internal/helm/chart/builder_local_test.go
similarity index 96%
rename from internal/helm/chart_builder_local_test.go
rename to internal/helm/chart/builder_local_test.go
index c2f16d694..477d24890 100644
--- a/internal/helm/chart_builder_local_test.go
+++ b/internal/helm/chart/builder_local_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"os"
@@ -99,16 +99,16 @@ func Test_copyFileToPath(t *testing.T) {
}{
{
name: "copies input file",
- in: "testdata/local-index.yaml",
+ in: "../testdata/local-index.yaml",
},
{
name: "invalid input file",
- in: "testdata/invalid.tgz",
+ in: "../testdata/invalid.tgz",
wantErr: "failed to open file to copy from",
},
{
name: "invalid input directory",
- in: "testdata/charts",
+ in: "../testdata/charts",
wantErr: "failed to read from source during copy",
},
}
diff --git a/internal/helm/chart_builder_remote.go b/internal/helm/chart/builder_remote.go
similarity index 91%
rename from internal/helm/chart_builder_remote.go
rename to internal/helm/chart/builder_remote.go
index 18ff317d8..ce1953655 100644
--- a/internal/helm/chart_builder_remote.go
+++ b/internal/helm/chart/builder_remote.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"context"
@@ -24,28 +24,31 @@ import (
"path/filepath"
"github.com/Masterminds/semver/v3"
- "github.com/fluxcd/pkg/runtime/transform"
- "github.com/fluxcd/source-controller/internal/fs"
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"sigs.k8s.io/yaml"
+
+ "github.com/fluxcd/pkg/runtime/transform"
+
+ "github.com/fluxcd/source-controller/internal/fs"
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
type remoteChartBuilder struct {
- remote *ChartRepository
+ remote *repository.ChartRepository
}
-// NewRemoteChartBuilder returns a ChartBuilder capable of building a Helm
-// chart with a RemoteChartReference from the given ChartRepository.
-func NewRemoteChartBuilder(repository *ChartRepository) ChartBuilder {
+// NewRemoteBuilder returns a Builder capable of building a Helm
+// chart with a RemoteReference from the given Index.
+func NewRemoteBuilder(repository *repository.ChartRepository) Builder {
return &remoteChartBuilder{
remote: repository,
}
}
-func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) {
- remoteRef, ok := ref.(RemoteChartReference)
+func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) {
+ remoteRef, ok := ref.(RemoteReference)
if !ok {
return nil, fmt.Errorf("expected remote chart reference")
}
@@ -59,13 +62,13 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p stri
}
defer b.remote.Unload()
- // Get the current version for the RemoteChartReference
+ // Get the current version for the RemoteReference
cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version)
if err != nil {
return nil, fmt.Errorf("failed to get chart version for remote reference: %w", err)
}
- result := &ChartBuild{}
+ result := &Build{}
result.Name = cv.Name
result.Version = cv.Version
// Set build specific metadata if instructed
diff --git a/internal/helm/chart_builder_remote_test.go b/internal/helm/chart/builder_remote_test.go
similarity index 92%
rename from internal/helm/chart_builder_remote_test.go
rename to internal/helm/chart/builder_remote_test.go
index 260bcbce1..b7a2dae2f 100644
--- a/internal/helm/chart_builder_remote_test.go
+++ b/internal/helm/chart/builder_remote_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"testing"
@@ -104,9 +104,9 @@ func Test_pathIsDir(t *testing.T) {
p string
want bool
}{
- {name: "directory", p: "testdata/", want: true},
- {name: "file", p: "testdata/local-index.yaml", want: false},
- {name: "not found error", p: "testdata/does-not-exist.yaml", want: false},
+ {name: "directory", p: "../testdata/", want: true},
+ {name: "file", p: "../testdata/local-index.yaml", want: false},
+ {name: "not found error", p: "../testdata/does-not-exist.yaml", want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart/builder_test.go
similarity index 89%
rename from internal/helm/chart_builder_test.go
rename to internal/helm/chart/builder_test.go
index a4252be8f..92aec74f1 100644
--- a/internal/helm/chart_builder_test.go
+++ b/internal/helm/chart/builder_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"encoding/hex"
@@ -30,18 +30,18 @@ import (
func TestChartBuildResult_String(t *testing.T) {
g := NewWithT(t)
- var result *ChartBuild
+ var result *Build
g.Expect(result.String()).To(Equal(""))
- result = &ChartBuild{}
+ result = &Build{}
g.Expect(result.String()).To(Equal(""))
- result = &ChartBuild{Path: "/foo/"}
+ result = &Build{Path: "/foo/"}
g.Expect(result.String()).To(Equal("/foo/"))
}
func Test_packageToPath(t *testing.T) {
g := NewWithT(t)
- chart, err := loader.Load("testdata/charts/helmchart-0.1.0.tgz")
+ chart, err := loader.Load("../testdata/charts/helmchart-0.1.0.tgz")
g.Expect(err).ToNot(HaveOccurred())
g.Expect(chart).ToNot(BeNil())
diff --git a/internal/helm/dependency_manager.go b/internal/helm/chart/dependency_manager.go
similarity index 81%
rename from internal/helm/dependency_manager.go
rename to internal/helm/chart/dependency_manager.go
index b8cd78571..2fa1df32c 100644
--- a/internal/helm/dependency_manager.go
+++ b/internal/helm/chart/dependency_manager.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"context"
@@ -31,18 +31,20 @@ import (
"golang.org/x/sync/semaphore"
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
+
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
-// GetChartRepositoryCallback must return a ChartRepository for the URL,
-// or an error describing why it could not be returned.
-type GetChartRepositoryCallback func(url string) (*ChartRepository, error)
+// GetChartRepositoryCallback must return a repository.ChartRepository for the
+// URL, or an error describing why it could not be returned.
+type GetChartRepositoryCallback func(url string) (*repository.ChartRepository, error)
// DependencyManager manages dependencies for a Helm chart.
type DependencyManager struct {
- // repositories contains a map of ChartRepository indexed by their
+ // repositories contains a map of Index indexed by their
// normalized URL. It is used as a lookup table for missing
// dependencies.
- repositories map[string]*ChartRepository
+ repositories map[string]*repository.ChartRepository
// getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback
// which returned result is cached to repositories.
@@ -56,11 +58,12 @@ type DependencyManager struct {
mu sync.Mutex
}
+// DependencyManagerOption configures an option on a DependencyManager.
type DependencyManagerOption interface {
applyToDependencyManager(dm *DependencyManager)
}
-type WithRepositories map[string]*ChartRepository
+type WithRepositories map[string]*repository.ChartRepository
func (o WithRepositories) applyToDependencyManager(dm *DependencyManager) {
dm.repositories = o
@@ -98,9 +101,9 @@ func (dm *DependencyManager) Clear() []error {
}
// Build compiles a set of missing dependencies from chart.Chart, and attempts to
-// resolve and build them using the information from ChartReference.
+// resolve and build them using the information from Reference.
// It returns the number of resolved local and remote dependencies, or an error.
-func (dm *DependencyManager) Build(ctx context.Context, ref ChartReference, chart *helmchart.Chart) (int, error) {
+func (dm *DependencyManager) Build(ctx context.Context, ref Reference, chart *helmchart.Chart) (int, error) {
// Collect dependency metadata
var (
deps = chart.Dependencies()
@@ -132,9 +135,9 @@ type chartWithLock struct {
// build adds the given list of deps to the chart with the configured number of
// concurrent workers. If the chart.Chart references a local dependency but no
-// LocalChartReference is given, or any dependency could not be added, an error
+// LocalReference is given, or any dependency could not be added, an error
// is returned. The first error it encounters cancels all other workers.
-func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, chart *helmchart.Chart, deps map[string]*helmchart.Dependency) error {
+func (dm *DependencyManager) build(ctx context.Context, ref Reference, c *helmchart.Chart, deps map[string]*helmchart.Dependency) error {
current := dm.concurrent
if current <= 0 {
current = 1
@@ -143,7 +146,7 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char
group, groupCtx := errgroup.WithContext(ctx)
group.Go(func() error {
sem := semaphore.NewWeighted(current)
- chart := &chartWithLock{Chart: chart}
+ c := &chartWithLock{Chart: c}
for name, dep := range deps {
name, dep := name, dep
if err := sem.Acquire(groupCtx, 1); err != nil {
@@ -152,17 +155,17 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char
group.Go(func() (err error) {
defer sem.Release(1)
if isLocalDep(dep) {
- localRef, ok := ref.(LocalChartReference)
+ localRef, ok := ref.(LocalReference)
if !ok {
err = fmt.Errorf("failed to add local dependency '%s': no local chart reference", name)
return
}
- if err = dm.addLocalDependency(localRef, chart, dep); err != nil {
+ if err = dm.addLocalDependency(localRef, c, dep); err != nil {
err = fmt.Errorf("failed to add local dependency '%s': %w", name, err)
}
return
}
- if err = dm.addRemoteDependency(chart, dep); err != nil {
+ if err = dm.addRemoteDependency(c, dep); err != nil {
err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err)
}
return
@@ -175,7 +178,7 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char
// addLocalDependency attempts to resolve and add the given local chart.Dependency
// to the chart.
-func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *chartWithLock, dep *helmchart.Dependency) error {
+func (dm *DependencyManager) addLocalDependency(ref LocalReference, c *chartWithLock, dep *helmchart.Dependency) error {
sLocalChartPath, err := dm.secureLocalChartPath(ref, dep)
if err != nil {
return err
@@ -197,7 +200,7 @@ func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *
ch, err := loader.Load(sLocalChartPath)
if err != nil {
return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w",
- strings.TrimPrefix(sLocalChartPath, ref.BaseDir), dep.Repository, err)
+ strings.TrimPrefix(sLocalChartPath, ref.WorkDir), dep.Repository, err)
}
ver, err := semver.NewVersion(ch.Metadata.Version)
@@ -210,9 +213,9 @@ func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *
return err
}
- chart.mu.Lock()
- chart.AddDependency(ch)
- chart.mu.Unlock()
+ c.mu.Lock()
+ c.AddDependency(ch)
+ c.mu.Unlock()
return nil
}
@@ -249,19 +252,19 @@ func (dm *DependencyManager) addRemoteDependency(chart *chartWithLock, dep *helm
}
// resolveRepository first attempts to resolve the url from the repositories, falling back
-// to getRepositoryCallback if set. It returns the resolved ChartRepository, or an error.
-func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) {
+// to getRepositoryCallback if set. It returns the resolved Index, or an error.
+func (dm *DependencyManager) resolveRepository(url string) (_ *repository.ChartRepository, err error) {
dm.mu.Lock()
defer dm.mu.Unlock()
- nUrl := NormalizeChartRepositoryURL(url)
+ nUrl := repository.NormalizeURL(url)
if _, ok := dm.repositories[nUrl]; !ok {
if dm.getRepositoryCallback == nil {
err = fmt.Errorf("no chart repository for URL '%s'", nUrl)
return
}
if dm.repositories == nil {
- dm.repositories = map[string]*ChartRepository{}
+ dm.repositories = map[string]*repository.ChartRepository{}
}
if dm.repositories[nUrl], err = dm.getRepositoryCallback(nUrl); err != nil {
err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err)
@@ -273,8 +276,8 @@ func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository,
// secureLocalChartPath returns the secure absolute path of a local dependency.
// It does not allow the dependency's path to be outside the scope of
-// LocalChartReference.BaseDir.
-func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep *helmchart.Dependency) (string, error) {
+// LocalReference.WorkDir.
+func (dm *DependencyManager) secureLocalChartPath(ref LocalReference, dep *helmchart.Dependency) (string, error) {
localUrl, err := url.Parse(dep.Repository)
if err != nil {
return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err)
@@ -282,11 +285,11 @@ func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep *
if localUrl.Scheme != "" && localUrl.Scheme != "file" {
return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository)
}
- relPath, err := filepath.Rel(ref.BaseDir, ref.Path)
+ relPath, err := filepath.Rel(ref.WorkDir, ref.Path)
if err != nil {
- return "", err
+ relPath = ref.Path
}
- return securejoin.SecureJoin(ref.BaseDir, filepath.Join(relPath, localUrl.Host, localUrl.Path))
+ return securejoin.SecureJoin(ref.WorkDir, filepath.Join(relPath, localUrl.Host, localUrl.Path))
}
// collectMissing returns a map with reqs that are missing from current,
diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go
similarity index 84%
rename from internal/helm/dependency_manager_test.go
rename to internal/helm/chart/dependency_manager_test.go
index 388eff1f4..825fb3b1a 100644
--- a/internal/helm/dependency_manager_test.go
+++ b/internal/helm/chart/dependency_manager_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"context"
@@ -29,26 +29,9 @@ import (
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/repo"
-)
-var (
- // helmPackageFile contains the path to a Helm package in the v2 format
- // without any dependencies
- helmPackageFile = "testdata/charts/helmchart-0.1.0.tgz"
- chartName = "helmchart"
- chartVersion = "0.1.0"
- chartLocalRepository = "file://../helmchart"
- remoteDepFixture = helmchart.Dependency{
- Name: chartName,
- Version: chartVersion,
- Repository: "https://example.com/charts",
- }
- // helmPackageV1File contains the path to a Helm package in the v1 format,
- // including dependencies in a requirements.yaml file which should be
- // loaded
- helmPackageV1File = "testdata/charts/helmchartwithdeps-v1-0.3.0.tgz"
- chartNameV1 = "helmchartwithdeps-v1"
- chartVersionV1 = "0.3.0"
+ "github.com/fluxcd/source-controller/internal/helm/getter"
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
func TestDependencyManager_Build(t *testing.T) {
@@ -56,7 +39,7 @@ func TestDependencyManager_Build(t *testing.T) {
name string
baseDir string
path string
- repositories map[string]*ChartRepository
+ repositories map[string]*repository.ChartRepository
getChartRepositoryCallback GetChartRepositoryCallback
want int
wantChartFunc func(g *WithT, c *helmchart.Chart)
@@ -70,13 +53,13 @@ func TestDependencyManager_Build(t *testing.T) {
//},
{
name: "build failure returns error",
- baseDir: "testdata/charts",
+ baseDir: "./../testdata/charts",
path: "helmchartwithdeps",
wantErr: "failed to add remote dependency 'grafana': no chart repository for URL",
},
{
name: "no dependencies returns zero",
- baseDir: "testdata/charts",
+ baseDir: "./../testdata/charts",
path: "helmchart",
want: 0,
},
@@ -91,7 +74,7 @@ func TestDependencyManager_Build(t *testing.T) {
got, err := NewDependencyManager(
WithRepositories(tt.repositories),
WithRepositoryCallback(tt.getChartRepositoryCallback),
- ).Build(context.TODO(), LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, chart)
+ ).Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
@@ -135,7 +118,7 @@ func TestDependencyManager_build(t *testing.T) {
g := NewWithT(t)
dm := NewDependencyManager()
- err := dm.build(context.TODO(), LocalChartReference{}, &helmchart.Chart{}, tt.deps)
+ err := dm.build(context.TODO(), LocalReference{}, &helmchart.Chart{}, tt.deps)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
return
@@ -180,7 +163,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
Version: chartVersion,
Repository: "file://../../../absolutely/invalid",
},
- wantErr: "no chart found at 'testdata/charts/absolutely/invalid'",
+ wantErr: "no chart found at '../testdata/charts/absolutely/invalid'",
},
{
name: "invalid chart archive",
@@ -207,7 +190,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
dm := NewDependencyManager()
chart := &helmchart.Chart{}
- err := dm.addLocalDependency(LocalChartReference{BaseDir: "testdata/charts", Path: "helmchartwithdeps"},
+ err := dm.addLocalDependency(LocalReference{WorkDir: "../testdata/charts", Path: "helmchartwithdeps"},
&chartWithLock{Chart: chart}, tt.dep)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
@@ -222,23 +205,23 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
func TestDependencyManager_addRemoteDependency(t *testing.T) {
g := NewWithT(t)
- chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz")
+ chartB, err := os.ReadFile("../testdata/charts/helmchart-0.1.0.tgz")
g.Expect(err).ToNot(HaveOccurred())
g.Expect(chartB).ToNot(BeEmpty())
tests := []struct {
name string
- repositories map[string]*ChartRepository
+ repositories map[string]*repository.ChartRepository
dep *helmchart.Dependency
wantFunc func(g *WithT, c *helmchart.Chart)
wantErr string
}{
{
name: "adds remote dependency",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
- Client: &mockGetter{
- response: chartB,
+ Client: &getter.MockGetter{
+ Response: chartB,
},
Index: &repo.IndexFile{
Entries: map[string]repo.ChartVersions{
@@ -266,7 +249,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "resolve repository error",
- repositories: map[string]*ChartRepository{},
+ repositories: map[string]*repository.ChartRepository{},
dep: &helmchart.Dependency{
Repository: "https://example.com",
},
@@ -274,7 +257,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "strategic load error",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
CachePath: "/invalid/cache/path/foo",
RWMutex: &sync.RWMutex{},
@@ -287,7 +270,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "repository get error",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
Index: &repo.IndexFile{},
RWMutex: &sync.RWMutex{},
@@ -300,7 +283,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "repository version constraint error",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
Index: &repo.IndexFile{
Entries: map[string]repo.ChartVersions{
@@ -326,7 +309,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "repository chart download error",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
Index: &repo.IndexFile{
Entries: map[string]repo.ChartVersions{
@@ -352,9 +335,9 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
},
{
name: "chart load error",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {
- Client: &mockGetter{},
+ Client: &getter.MockGetter{},
Index: &repo.IndexFile{
Entries: map[string]repo.ChartVersions{
chartName: {
@@ -404,40 +387,40 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
func TestDependencyManager_resolveRepository(t *testing.T) {
tests := []struct {
name string
- repositories map[string]*ChartRepository
+ repositories map[string]*repository.ChartRepository
getChartRepositoryCallback GetChartRepositoryCallback
url string
- want *ChartRepository
- wantRepositories map[string]*ChartRepository
+ want *repository.ChartRepository
+ wantRepositories map[string]*repository.ChartRepository
wantErr string
}{
{
name: "resolves from repositories index",
url: "https://example.com",
- repositories: map[string]*ChartRepository{
+ repositories: map[string]*repository.ChartRepository{
"https://example.com/": {URL: "https://example.com"},
},
- want: &ChartRepository{URL: "https://example.com"},
+ want: &repository.ChartRepository{URL: "https://example.com"},
},
{
name: "resolves from callback",
url: "https://example.com",
- getChartRepositoryCallback: func(url string) (*ChartRepository, error) {
- return &ChartRepository{URL: "https://example.com"}, nil
+ getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) {
+ return &repository.ChartRepository{URL: "https://example.com"}, nil
},
- want: &ChartRepository{URL: "https://example.com"},
- wantRepositories: map[string]*ChartRepository{
+ want: &repository.ChartRepository{URL: "https://example.com"},
+ wantRepositories: map[string]*repository.ChartRepository{
"https://example.com/": {URL: "https://example.com"},
},
},
{
name: "error from callback",
url: "https://example.com",
- getChartRepositoryCallback: func(url string) (*ChartRepository, error) {
+ getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) {
return nil, errors.New("a very unique error")
},
wantErr: "a very unique error",
- wantRepositories: map[string]*ChartRepository{},
+ wantRepositories: map[string]*repository.ChartRepository{},
},
{
name: "error on not found",
@@ -518,7 +501,7 @@ func TestDependencyManager_secureLocalChartPath(t *testing.T) {
g := NewWithT(t)
dm := NewDependencyManager()
- got, err := dm.secureLocalChartPath(LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, tt.dep)
+ got, err := dm.secureLocalChartPath(LocalReference{WorkDir: tt.baseDir, Path: tt.path}, tt.dep)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
diff --git a/internal/helm/chart.go b/internal/helm/chart/metadata.go
similarity index 96%
rename from internal/helm/chart.go
rename to internal/helm/chart/metadata.go
index 4f89cab61..24e452089 100644
--- a/internal/helm/chart.go
+++ b/internal/helm/chart/metadata.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"archive/tar"
@@ -33,6 +33,8 @@ import (
helmchart "helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chartutil"
"sigs.k8s.io/yaml"
+
+ "github.com/fluxcd/source-controller/internal/helm"
)
// OverwriteChartDefaultValues overwrites the chart default values file with the given data.
@@ -115,8 +117,8 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) {
if stat.IsDir() {
return nil, fmt.Errorf("'%s' is a directory", stat.Name())
}
- if stat.Size() > MaxChartFileSize {
- return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), MaxChartFileSize)
+ if stat.Size() > helm.MaxChartFileSize {
+ return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartFileSize)
}
}
@@ -142,8 +144,8 @@ func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) {
}
return nil, err
}
- if stat.Size() > MaxChartSize {
- return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), MaxChartSize)
+ if stat.Size() > helm.MaxChartSize {
+ return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartSize)
}
f, err := os.Open(archive)
diff --git a/internal/helm/chart_test.go b/internal/helm/chart/metadata_test.go
similarity index 85%
rename from internal/helm/chart_test.go
rename to internal/helm/chart/metadata_test.go
index ac7114e87..f2294ff6b 100644
--- a/internal/helm/chart_test.go
+++ b/internal/helm/chart/metadata_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package chart
import (
"testing"
@@ -25,6 +25,19 @@ import (
)
var (
+ // helmPackageFile contains the path to a Helm package in the v2 format
+ // without any dependencies
+ helmPackageFile = "../testdata/charts/helmchart-0.1.0.tgz"
+ chartName = "helmchart"
+ chartVersion = "0.1.0"
+
+ // helmPackageV1File contains the path to a Helm package in the v1 format,
+ // including dependencies in a requirements.yaml file which should be
+ // loaded
+ helmPackageV1File = "../testdata/charts/helmchartwithdeps-v1-0.3.0.tgz"
+ chartNameV1 = "helmchartwithdeps-v1"
+ chartVersionV1 = "0.3.0"
+
originalValuesFixture = []byte(`override: original
`)
chartFilesFixture = []*helmchart.File{
@@ -123,21 +136,21 @@ func TestLoadChartMetadataFromDir(t *testing.T) {
}{
{
name: "Loads from dir",
- dir: "testdata/charts/helmchart",
+ dir: "../testdata/charts/helmchart",
wantName: "helmchart",
wantVersion: "0.1.0",
},
{
name: "Loads from v1 dir including requirements.yaml",
- dir: "testdata/charts/helmchartwithdeps-v1",
+ dir: "../testdata/charts/helmchartwithdeps-v1",
wantName: chartNameV1,
wantVersion: chartVersionV1,
wantDependencyCount: 1,
},
{
name: "Error if no Chart.yaml",
- dir: "testdata/charts/",
- wantErr: "testdata/charts/Chart.yaml: no such file or directory",
+ dir: "../testdata/charts/",
+ wantErr: "../testdata/charts/Chart.yaml: no such file or directory",
},
}
for _, tt := range tests {
@@ -186,12 +199,12 @@ func TestLoadChartMetadataFromArchive(t *testing.T) {
},
{
name: "Error on not found",
- archive: "testdata/invalid.tgz",
+ archive: "../testdata/invalid.tgz",
wantErr: "no such file or directory",
},
{
name: "Error if no Chart.yaml",
- archive: "testdata/charts/empty.tgz",
+ archive: "../testdata/charts/empty.tgz",
wantErr: "no 'Chart.yaml' found",
},
}
diff --git a/internal/helm/getter.go b/internal/helm/getter/getter.go
similarity index 99%
rename from internal/helm/getter.go
rename to internal/helm/getter/getter.go
index 1ca8b0e9b..583bac5f7 100644
--- a/internal/helm/getter.go
+++ b/internal/helm/getter/getter.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package getter
import (
"fmt"
diff --git a/internal/helm/getter_test.go b/internal/helm/getter/getter_test.go
similarity index 99%
rename from internal/helm/getter_test.go
rename to internal/helm/getter/getter_test.go
index 2c55e7cbb..6437e5b35 100644
--- a/internal/helm/getter_test.go
+++ b/internal/helm/getter/getter_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package getter
import (
"os"
diff --git a/internal/helm/getter/mock.go b/internal/helm/getter/mock.go
new file mode 100644
index 000000000..91cd2b7bc
--- /dev/null
+++ b/internal/helm/getter/mock.go
@@ -0,0 +1,41 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package getter
+
+import (
+ "bytes"
+
+ "helm.sh/helm/v3/pkg/getter"
+)
+
+// MockGetter can be used as a simple mocking getter.Getter implementation.
+type MockGetter struct {
+ Response []byte
+
+ requestedURL string
+}
+
+func (g *MockGetter) Get(u string, _ ...getter.Option) (*bytes.Buffer, error) {
+ g.requestedURL = u
+ r := g.Response
+ return bytes.NewBuffer(r), nil
+}
+
+// LastGet returns the last requested URL for Get.
+func (g *MockGetter) LastGet() string {
+ return g.requestedURL
+}
diff --git a/internal/helm/repository.go b/internal/helm/repository/chart_repository.go
similarity index 98%
rename from internal/helm/repository.go
rename to internal/helm/repository/chart_repository.go
index eb9e668a1..638355f80 100644
--- a/internal/helm/repository.go
+++ b/internal/helm/repository/chart_repository.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package repository
import (
"bytes"
@@ -36,6 +36,8 @@ import (
"sigs.k8s.io/yaml"
"github.com/fluxcd/pkg/version"
+
+ "github.com/fluxcd/source-controller/internal/helm"
)
var ErrNoChartIndex = errors.New("no chart index")
@@ -241,8 +243,8 @@ func (r *ChartRepository) LoadFromFile(path string) error {
}
return err
}
- if stat.Size() > MaxIndexSize {
- return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), MaxIndexSize)
+ if stat.Size() > helm.MaxIndexSize {
+ return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), helm.MaxIndexSize)
}
b, err := os.ReadFile(path)
if err != nil {
@@ -350,7 +352,7 @@ func (r *ChartRepository) HasCacheFile() bool {
}
// Unload can be used to signal the Go garbage collector the Index can
-// be freed from memory if the ChartRepository object is expected to
+// be freed from memory if the Index object is expected to
// continue to exist in the stack for some time.
func (r *ChartRepository) Unload() {
if r == nil {
diff --git a/internal/helm/repository_test.go b/internal/helm/repository/chart_repository_test.go
similarity index 93%
rename from internal/helm/repository_test.go
rename to internal/helm/repository/chart_repository_test.go
index 9c124b791..b6f191f3b 100644
--- a/internal/helm/repository_test.go
+++ b/internal/helm/repository/chart_repository_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package repository
import (
"bytes"
@@ -27,39 +27,29 @@ import (
. "github.com/onsi/gomega"
"helm.sh/helm/v3/pkg/chart"
- "helm.sh/helm/v3/pkg/getter"
+ helmgetter "helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/repo"
+
+ "github.com/fluxcd/source-controller/internal/helm/getter"
)
var now = time.Now()
const (
- testFile = "testdata/local-index.yaml"
- chartmuseumTestFile = "testdata/chartmuseum-index.yaml"
- unorderedTestFile = "testdata/local-index-unordered.yaml"
+ testFile = "../testdata/local-index.yaml"
+ chartmuseumTestFile = "../testdata/chartmuseum-index.yaml"
+ unorderedTestFile = "../testdata/local-index-unordered.yaml"
)
-// mockGetter can be used as a simple mocking getter.Getter implementation.
-type mockGetter struct {
- requestedURL string
- response []byte
-}
-
-func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) {
- g.requestedURL = url
- r := g.response
- return bytes.NewBuffer(r), nil
-}
-
func TestNewChartRepository(t *testing.T) {
repositoryURL := "https://example.com"
- providers := getter.Providers{
- getter.Provider{
+ providers := helmgetter.Providers{
+ helmgetter.Provider{
Schemes: []string{"https"},
- New: getter.NewHTTPGetter,
+ New: helmgetter.NewHTTPGetter,
},
}
- options := []getter.Option{getter.WithBasicAuth("username", "password")}
+ options := []helmgetter.Option{helmgetter.WithBasicAuth("username", "password")}
t.Run("should construct chart repository", func(t *testing.T) {
g := NewWithT(t)
@@ -230,7 +220,7 @@ func TestChartRepository_DownloadChart(t *testing.T) {
g := NewWithT(t)
t.Parallel()
- mg := mockGetter{}
+ mg := getter.MockGetter{}
r := &ChartRepository{
URL: tt.url,
Client: &mg,
@@ -241,7 +231,7 @@ func TestChartRepository_DownloadChart(t *testing.T) {
g.Expect(res).To(BeNil())
return
}
- g.Expect(mg.requestedURL).To(Equal(tt.wantURL))
+ g.Expect(mg.LastGet()).To(Equal(tt.wantURL))
g.Expect(res).ToNot(BeNil())
g.Expect(err).ToNot(HaveOccurred())
})
@@ -254,7 +244,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) {
b, err := os.ReadFile(chartmuseumTestFile)
g.Expect(err).ToNot(HaveOccurred())
- mg := mockGetter{response: b}
+ mg := getter.MockGetter{Response: b}
r := &ChartRepository{
URL: "https://example.com",
Client: &mg,
@@ -263,7 +253,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) {
buf := bytes.NewBuffer([]byte{})
g.Expect(r.DownloadIndex(buf)).To(Succeed())
g.Expect(buf.Bytes()).To(Equal(b))
- g.Expect(mg.requestedURL).To(Equal(r.URL + "/index.yaml"))
+ g.Expect(mg.LastGet()).To(Equal(r.URL + "/index.yaml"))
g.Expect(err).To(BeNil())
}
@@ -384,8 +374,8 @@ func TestChartRepository_LoadIndexFromFile(t *testing.T) {
func TestChartRepository_CacheIndex(t *testing.T) {
g := NewWithT(t)
- mg := mockGetter{response: []byte("foo")}
- expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.response))
+ mg := getter.MockGetter{Response: []byte("foo")}
+ expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.Response))
r := newChartRepository()
r.URL = "https://example.com"
@@ -399,7 +389,7 @@ func TestChartRepository_CacheIndex(t *testing.T) {
g.Expect(r.CachePath).To(BeARegularFile())
b, _ := os.ReadFile(r.CachePath)
- g.Expect(b).To(Equal(mg.response))
+ g.Expect(b).To(Equal(mg.Response))
g.Expect(sum).To(BeEquivalentTo(expectSum))
}
diff --git a/internal/helm/utils.go b/internal/helm/repository/utils.go
similarity index 77%
rename from internal/helm/utils.go
rename to internal/helm/repository/utils.go
index ff2221c61..b02b13782 100644
--- a/internal/helm/utils.go
+++ b/internal/helm/repository/utils.go
@@ -1,5 +1,5 @@
/*
-Copyright 2020 The Flux authors
+Copyright 2021 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package helm
+package repository
import "strings"
-// NormalizeChartRepositoryURL ensures repository urls are normalized
-func NormalizeChartRepositoryURL(url string) string {
+// NormalizeURL normalizes a ChartRepository URL by ensuring it ends with a
+// single "/".
+func NormalizeURL(url string) string {
if url != "" {
return strings.TrimRight(url, "/") + "/"
}
diff --git a/internal/helm/repository/utils_test.go b/internal/helm/repository/utils_test.go
new file mode 100644
index 000000000..fe4cf80ee
--- /dev/null
+++ b/internal/helm/repository/utils_test.go
@@ -0,0 +1,44 @@
+package repository
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+)
+
+func TestNormalizeURL(t *testing.T) {
+ tests := []struct {
+ name string
+ url string
+ want string
+ }{
+ {
+ name: "with slash",
+ url: "http://example.com/",
+ want: "http://example.com/",
+ },
+ {
+ name: "without slash",
+ url: "http://example.com",
+ want: "http://example.com/",
+ },
+ {
+ name: "double slash",
+ url: "http://example.com//",
+ want: "http://example.com/",
+ },
+ {
+ name: "empty",
+ url: "",
+ want: "",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ got := NormalizeURL(tt.url)
+ g.Expect(got).To(Equal(tt.want))
+ })
+ }
+}
diff --git a/internal/helm/utils_test.go b/internal/helm/utils_test.go
deleted file mode 100644
index 62a9e92c2..000000000
--- a/internal/helm/utils_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-Copyright 2021 The Flux authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package helm
-
-import (
- "testing"
-
- . "github.com/onsi/gomega"
-)
-
-func TestNormalizeChartRepositoryURL(t *testing.T) {
- tests := []struct {
- name string
- url string
- want string
- }{
- {
- name: "with slash",
- url: "http://example.com/",
- want: "http://example.com/",
- },
- {
- name: "without slash",
- url: "http://example.com",
- want: "http://example.com/",
- },
- {
- name: "double slash",
- url: "http://example.com//",
- want: "http://example.com/",
- },
- {
- name: "empty",
- url: "",
- want: "",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- g := NewWithT(t)
-
- got := NormalizeChartRepositoryURL(tt.url)
- g.Expect(got).To(Equal(tt.want))
- })
- }
-}
From 32e19ebcd0e1a75b08c752dd616eb1b4a742dbed Mon Sep 17 00:00:00 2001
From: Hidde Beydals
Date: Tue, 16 Nov 2021 09:50:07 +0100
Subject: [PATCH 0229/1397] controllers: more tidying of wiring
Dealing with some loose ends around making observations, and code
style.
The loaded byes of a chart are used as a revision to ensure e.g.
periodic builds with unstable ordering of items do not trigger a false
positive.
Signed-off-by: Hidde Beydals
---
controllers/helmchart_controller.go | 84 +++++++++---------------
controllers/helmchart_controller_test.go | 24 -------
controllers/helmrepository_controller.go | 38 +++++------
3 files changed, 46 insertions(+), 100 deletions(-)
diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go
index d31f6c2bb..3c1be0a7d 100644
--- a/controllers/helmchart_controller.go
+++ b/controllers/helmchart_controller.go
@@ -22,13 +22,12 @@ import (
"net/url"
"os"
"path/filepath"
- "regexp"
"strings"
"time"
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/go-logr/logr"
- extgetter "helm.sh/helm/v3/pkg/getter"
+ helmgetter "helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -69,7 +68,7 @@ type HelmChartReconciler struct {
client.Client
Scheme *runtime.Scheme
Storage *Storage
- Getters extgetter.Providers
+ Getters helmgetter.Providers
EventRecorder kuberecorder.EventRecorder
ExternalEventRecorder *events.Recorder
MetricsRecorder *metrics.Recorder
@@ -199,7 +198,7 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
// Create working directory
- workDir, err := os.MkdirTemp("", chart.Kind + "-" + chart.Namespace + "-" + chart.Name + "-")
+ workDir, err := os.MkdirTemp("", chart.Kind+"-"+chart.Namespace+"-"+chart.Name+"-")
if err != nil {
err = fmt.Errorf("failed to create temporary working directory: %w", err)
chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error())
@@ -216,21 +215,6 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
var reconcileErr error
switch typedSource := source.(type) {
case *sourcev1.HelmRepository:
- // TODO: move this to a validation webhook once the discussion around
- // certificates has settled: https://github.com/fluxcd/image-reflector-controller/issues/69
- if err := validHelmChartName(chart.Spec.Chart); err != nil {
- reconciledChart = sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error())
- log.Error(err, "validation failed")
- if err := r.updateStatus(ctx, req, reconciledChart.Status); err != nil {
- log.Info(fmt.Sprintf("%v", reconciledChart.Status))
- log.Error(err, "unable to update status")
- return ctrl.Result{Requeue: true}, err
- }
- r.event(ctx, reconciledChart, events.EventSeverityError, err.Error())
- r.recordReadiness(ctx, reconciledChart)
- // Do not requeue as there is no chance on recovery.
- return ctrl.Result{Requeue: false}, nil
- }
reconciledChart, reconcileErr = r.fromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), workDir, changed)
case *sourcev1.GitRepository, *sourcev1.Bucket:
reconciledChart, reconcileErr = r.fromTarballArtifact(ctx, *typedSource.GetArtifact(), *chart.DeepCopy(),
@@ -309,10 +293,10 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm
func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourcev1.HelmRepository, c sourcev1.HelmChart,
workDir string, force bool) (sourcev1.HelmChart, error) {
// Configure Index getter options
- clientOpts := []extgetter.Option{
- extgetter.WithURL(repo.Spec.URL),
- extgetter.WithTimeout(repo.Spec.Timeout.Duration),
- extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
+ clientOpts := []helmgetter.Option{
+ helmgetter.WithURL(repo.Spec.URL),
+ helmgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
if secret, err := r.getHelmRepositorySecret(ctx, &repo); err != nil {
return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err
@@ -423,7 +407,7 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so
err = fmt.Errorf("artifact untar error: %w", err)
return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- if err =f.Close(); err != nil {
+ if err = f.Close(); err != nil {
err = fmt.Errorf("artifact close error: %w", err)
return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
@@ -440,20 +424,17 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so
return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err
}
dm := chart.NewDependencyManager(
- chart.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())),
+ chart.WithRepositoryCallback(r.namespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())),
)
defer dm.Clear()
- // Get any cached chart
- var cachedChart string
- if artifact := c.Status.Artifact; artifact != nil {
- cachedChart = artifact.Path
- }
-
+ // Configure builder options, including any previously cached chart
buildsOpts := chart.BuildOptions{
- ValueFiles: c.GetValuesFiles(),
- CachedChart: cachedChart,
- Force: force,
+ ValueFiles: c.GetValuesFiles(),
+ Force: force,
+ }
+ if artifact := c.Status.Artifact; artifact != nil {
+ buildsOpts.CachedChart = artifact.Path
}
// Add revision metadata to chart build
@@ -465,7 +446,7 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so
// Build chart
chartB := chart.NewLocalBuilder(dm)
- build, err := chartB.Build(ctx, chart.LocalReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts)
+ build, err := chartB.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts)
if err != nil {
return sourcev1.HelmChartNotReady(c, sourcev1.ChartPackageFailedReason, err.Error()), err
}
@@ -475,7 +456,8 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so
// If the path of the returned build equals the cache path,
// there are no changes to the chart
- if build.Path == cachedChart {
+ if apimeta.IsStatusConditionTrue(c.Status.Conditions, meta.ReadyCondition) &&
+ build.Path == buildsOpts.CachedChart {
// Ensure hostname is updated
if c.GetArtifact().URL != newArtifact.URL {
r.Storage.SetArtifactURL(c.GetArtifact())
@@ -515,11 +497,17 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so
return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil
}
-// TODO(hidde): factor out to helper?
-func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback {
+// namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback
+// scoped to the given namespace. Credentials for retrieved v1beta1.HelmRepository
+// objects are stored in the given directory.
+// The returned callback returns a repository.ChartRepository configured with the
+// retrieved v1beta1.HelmRepository, or a shim with defaults if no object could
+// be found.
+func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback {
return func(url string) (*repository.ChartRepository, error) {
repo, err := r.resolveDependencyRepository(ctx, url, namespace)
if err != nil {
+ // Return Kubernetes client errors, but ignore others
if errors.ReasonForError(err) != metav1.StatusReasonUnknown {
return nil, err
}
@@ -530,10 +518,10 @@ func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.C
},
}
}
- clientOpts := []extgetter.Option{
- extgetter.WithURL(repo.Spec.URL),
- extgetter.WithTimeout(repo.Spec.Timeout.Duration),
- extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
+ clientOpts := []helmgetter.Option{
+ helmgetter.WithURL(repo.Spec.URL),
+ helmgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil {
return nil, err
@@ -801,18 +789,6 @@ func (r *HelmChartReconciler) requestsForBucketChange(o client.Object) []reconci
return reqs
}
-// validHelmChartName returns an error if the given string is not a
-// valid Helm chart name; a valid name must be lower case letters
-// and numbers, words may be separated with dashes (-).
-// Ref: https://helm.sh/docs/chart_best_practices/conventions/#chart-names
-func validHelmChartName(s string) error {
- chartFmt := regexp.MustCompile("^([-a-z0-9]*)$")
- if !chartFmt.MatchString(s) {
- return fmt.Errorf("invalid chart name %q, a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", s)
- }
- return nil
-}
-
func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart sourcev1.HelmChart) {
if r.MetricsRecorder == nil {
return
diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go
index ceb30842f..82df1bc35 100644
--- a/controllers/helmchart_controller_test.go
+++ b/controllers/helmchart_controller_test.go
@@ -25,7 +25,6 @@ import (
"path"
"path/filepath"
"strings"
- "testing"
"time"
"github.com/fluxcd/pkg/apis/meta"
@@ -1327,26 +1326,3 @@ var _ = Describe("HelmChartReconciler", func() {
})
})
})
-
-func Test_validHelmChartName(t *testing.T) {
- tests := []struct {
- name string
- chart string
- expectErr bool
- }{
- {"valid", "drupal", false},
- {"valid dash", "nginx-lego", false},
- {"valid dashes", "aws-cluster-autoscaler", false},
- {"valid alphanum", "ng1nx-leg0", false},
- {"invalid slash", "artifactory/invalid", true},
- {"invalid dot", "in.valid", true},
- {"invalid uppercase", "inValid", true},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if err := validHelmChartName(tt.chart); (err != nil) != tt.expectErr {
- t.Errorf("validHelmChartName() error = %v, expectErr %v", err, tt.expectErr)
- }
- })
- }
-}
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index 8ab87201d..5a29a7734 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -24,7 +24,7 @@ import (
"time"
"github.com/go-logr/logr"
- extgetter "helm.sh/helm/v3/pkg/getter"
+ helmgetter "helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,9 +43,9 @@ import (
"github.com/fluxcd/pkg/runtime/metrics"
"github.com/fluxcd/pkg/runtime/predicates"
+ sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"github.com/fluxcd/source-controller/internal/helm/getter"
"github.com/fluxcd/source-controller/internal/helm/repository"
- sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=get;list;watch;create;update;patch;delete
@@ -58,7 +58,7 @@ type HelmRepositoryReconciler struct {
client.Client
Scheme *runtime.Scheme
Storage *Storage
- Getters extgetter.Providers
+ Getters helmgetter.Providers
EventRecorder kuberecorder.EventRecorder
ExternalEventRecorder *events.Recorder
MetricsRecorder *metrics.Recorder
@@ -171,10 +171,10 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque
}
func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.HelmRepository) (sourcev1.HelmRepository, error) {
- clientOpts := []extgetter.Option{
- extgetter.WithURL(repo.Spec.URL),
- extgetter.WithTimeout(repo.Spec.Timeout.Duration),
- extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
+ clientOpts := []helmgetter.Option{
+ helmgetter.WithURL(repo.Spec.URL),
+ helmgetter.WithTimeout(repo.Spec.Timeout.Duration),
+ helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials),
}
if repo.Spec.SecretRef != nil {
name := types.NamespacedName{
@@ -189,7 +189,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err
}
- authDir, err := os.MkdirTemp("", "helm-repository-")
+ authDir, err := os.MkdirTemp("", repo.Kind+"-"+repo.Namespace+"-"+repo.Name+"-")
if err != nil {
err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err)
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err
@@ -213,7 +213,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
}
}
- revision, err := chartRepo.CacheIndex()
+ checksum, err := chartRepo.CacheIndex()
if err != nil {
err = fmt.Errorf("failed to download repository index: %w", err)
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
@@ -222,12 +222,12 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.
artifact := r.Storage.NewArtifactFor(repo.Kind,
repo.ObjectMeta.GetObjectMeta(),
- revision,
- fmt.Sprintf("index-%s.yaml", revision))
+ "",
+ fmt.Sprintf("index-%s.yaml", checksum))
// Return early on unchanged index
if apimeta.IsStatusConditionTrue(repo.Status.Conditions, meta.ReadyCondition) &&
- repo.GetArtifact().HasRevision(artifact.Revision) {
+ (repo.GetArtifact() != nil && repo.GetArtifact().Checksum == checksum) {
if artifact.URL != repo.GetArtifact().URL {
r.Storage.SetArtifactURL(repo.GetArtifact())
repo.Status.URL = r.Storage.SetHostname(repo.Status.URL)
@@ -239,7 +239,9 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.
if err := chartRepo.LoadFromCache(); err != nil {
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err
}
- defer chartRepo.Unload()
+ // The repository checksum is the SHA256 of the loaded bytes, after sorting
+ artifact.Revision = chartRepo.Checksum
+ chartRepo.Unload()
// Create artifact dir
err = r.Storage.MkdirAll(artifact)
@@ -257,17 +259,9 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.
defer unlock()
// Save artifact to storage
- storageTarget := r.Storage.LocalPath(artifact)
- if storageTarget == "" {
- err := fmt.Errorf("failed to calcalute local storage path to store artifact to")
- return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
- }
- if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil {
+ if err = r.Storage.CopyFromPath(&artifact, chartRepo.CachePath); err != nil {
return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err
}
- // TODO(hidde): it would be better to make the Storage deal with this
- artifact.Checksum = chartRepo.Checksum
- artifact.LastUpdateTime = metav1.Now()
// Update index symlink
indexURL, err := r.Storage.Symlink(artifact, "index.yaml")
From 7c910e37a2bed1debcaa70dde942d4dc4884ca6d Mon Sep 17 00:00:00 2001
From: Sunny
Date: Tue, 16 Nov 2021 16:26:05 +0530
Subject: [PATCH 0230/1397] internal/helm: local builder & dep manager test
Add more chart local builder and dependency manager tests.
Signed-off-by: Sunny
---
go.mod | 1 +
go.sum | 7 +
internal/helm/chart/builder_local_test.go | 211 ++++++++++++++++++
.../helm/chart/dependency_manager_test.go | 84 ++++++-
4 files changed, 294 insertions(+), 9 deletions(-)
diff --git a/go.mod b/go.mod
index c4503b710..5246fc455 100644
--- a/go.mod
+++ b/go.mod
@@ -38,6 +38,7 @@ require (
github.com/minio/minio-go/v7 v7.0.10
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.14.0
+ github.com/otiai10/copy v1.7.0
github.com/spf13/pflag v1.0.5
github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect
github.com/yvasiyarov/gorelic v0.0.7 // indirect
diff --git a/go.sum b/go.sum
index 593aa3e0b..a252cf16f 100644
--- a/go.sum
+++ b/go.sum
@@ -738,6 +738,13 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
+github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
+github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
+github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go
index 477d24890..1e0acb744 100644
--- a/internal/helm/chart/builder_local_test.go
+++ b/internal/helm/chart/builder_local_test.go
@@ -17,14 +17,225 @@ limitations under the License.
package chart
import (
+ "context"
"os"
"path/filepath"
+ "sync"
"testing"
. "github.com/onsi/gomega"
+ "github.com/otiai10/copy"
helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
+ "helm.sh/helm/v3/pkg/chartutil"
+ "helm.sh/helm/v3/pkg/repo"
+
+ "github.com/fluxcd/source-controller/internal/helm/getter"
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
+func TestLocalBuilder_Build(t *testing.T) {
+ g := NewWithT(t)
+
+ // Prepare chart repositories to be used for charts with remote dependency.
+ chartB, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chartB).ToNot(BeEmpty())
+ mockRepo := func() *repository.ChartRepository {
+ return &repository.ChartRepository{
+ Client: &getter.MockGetter{
+ Response: chartB,
+ },
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ "grafana": {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: "grafana",
+ Version: "6.17.4",
+ },
+ URLs: []string{"https://example.com/grafana.tgz"},
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ }
+ }
+
+ tests := []struct {
+ name string
+ reference Reference
+ buildOpts BuildOptions
+ valueFiles []helmchart.File
+ repositories map[string]*repository.ChartRepository
+ dependentChartPaths []string
+ wantValues chartutil.Values
+ wantVersion string
+ wantPackaged bool
+ wantErr string
+ }{
+ {
+ name: "invalid reference",
+ reference: RemoteReference{},
+ wantErr: "expected local chart reference",
+ },
+ {
+ name: "invalid local reference - no path",
+ reference: LocalReference{},
+ wantErr: "no path set for local chart reference",
+ },
+ {
+ name: "invalid local reference - no file",
+ reference: LocalReference{Path: "/tmp/non-existent-path.xyz"},
+ wantErr: "no such file or directory",
+ },
+ {
+ name: "invalid version metadata",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart"},
+ buildOpts: BuildOptions{VersionMetadata: "^"},
+ wantErr: "Invalid Metadata string",
+ },
+ {
+ name: "with version metadata",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart"},
+ buildOpts: BuildOptions{VersionMetadata: "foo"},
+ wantVersion: "0.1.0+foo",
+ wantPackaged: true,
+ },
+ // TODO: Test setting BuildOptions CachedChart and Force.
+ {
+ name: "already packaged chart",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart-0.1.0.tgz"},
+ wantVersion: "0.1.0",
+ wantPackaged: false,
+ },
+ {
+ name: "default values",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart"},
+ wantValues: chartutil.Values{
+ "replicaCount": float64(1),
+ },
+ wantVersion: "0.1.0",
+ wantPackaged: true,
+ },
+ {
+ name: "with value files",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart"},
+ buildOpts: BuildOptions{
+ ValueFiles: []string{"custom-values1.yaml", "custom-values2.yaml"},
+ },
+ valueFiles: []helmchart.File{
+ {
+ Name: "custom-values1.yaml",
+ Data: []byte(`replicaCount: 11
+nameOverride: "foo-name-override"`),
+ },
+ {
+ Name: "custom-values2.yaml",
+ Data: []byte(`replicaCount: 20
+fullnameOverride: "full-foo-name-override"`),
+ },
+ },
+ wantValues: chartutil.Values{
+ "replicaCount": float64(20),
+ "nameOverride": "foo-name-override",
+ "fullnameOverride": "full-foo-name-override",
+ },
+ wantVersion: "0.1.0",
+ wantPackaged: true,
+ },
+ {
+ name: "chart with dependencies",
+ reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps"},
+ repositories: map[string]*repository.ChartRepository{
+ "https://grafana.github.io/helm-charts/": mockRepo(),
+ },
+ dependentChartPaths: []string{"./../testdata/charts/helmchart"},
+ wantVersion: "0.1.0",
+ wantPackaged: true,
+ },
+ {
+ name: "v1 chart",
+ reference: LocalReference{Path: "./../testdata/charts/helmchart-v1"},
+ wantValues: chartutil.Values{
+ "replicaCount": float64(1),
+ },
+ wantVersion: "0.2.0",
+ wantPackaged: true,
+ },
+ {
+ name: "v1 chart with dependencies",
+ reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps-v1"},
+ repositories: map[string]*repository.ChartRepository{
+ "https://grafana.github.io/helm-charts/": mockRepo(),
+ },
+ dependentChartPaths: []string{"./../testdata/charts/helmchart-v1"},
+ wantVersion: "0.3.0",
+ wantPackaged: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ workDir, err := os.MkdirTemp("", "local-builder-")
+ g.Expect(err).ToNot(HaveOccurred())
+ defer os.RemoveAll(workDir)
+
+ // Only if the reference is a LocalReference, set the WorkDir.
+ localRef, ok := tt.reference.(LocalReference)
+ if ok {
+ localRef.WorkDir = workDir
+ tt.reference = localRef
+ }
+
+ // Write value file in the base dir.
+ for _, f := range tt.valueFiles {
+ vPath := filepath.Join(workDir, f.Name)
+ g.Expect(os.WriteFile(vPath, f.Data, 0644)).ToNot(HaveOccurred())
+ }
+
+ // Write chart dependencies in the base dir.
+ for _, dcp := range tt.dependentChartPaths {
+ // Construct the chart path relative to the testdata chart.
+ helmchartDir := filepath.Join(workDir, "testdata", "charts", filepath.Base(dcp))
+ g.Expect(copy.Copy(dcp, helmchartDir)).ToNot(HaveOccurred())
+ }
+
+ // Target path with name similar to the workDir.
+ targetPath := workDir + ".tgz"
+ defer os.RemoveAll(targetPath)
+
+ dm := NewDependencyManager(
+ WithRepositories(tt.repositories),
+ )
+
+ b := NewLocalBuilder(dm)
+ cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts)
+
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(cb).To(BeZero())
+ return
+ }
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value")
+ g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path")
+
+ // Load the resulting chart and verify the values.
+ resultChart, err := loader.Load(cb.Path)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion))
+
+ for k, v := range tt.wantValues {
+ g.Expect(v).To(Equal(resultChart.Values[k]))
+ }
+ })
+ }
+}
+
func Test_mergeFileValues(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/helm/chart/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go
index 825fb3b1a..da4b70a67 100644
--- a/internal/helm/chart/dependency_manager_test.go
+++ b/internal/helm/chart/dependency_manager_test.go
@@ -35,6 +35,36 @@ import (
)
func TestDependencyManager_Build(t *testing.T) {
+ g := NewWithT(t)
+
+ // Mock chart used as grafana chart in the test below. The cached repository
+ // takes care of the actual grafana related details in the chart index.
+ chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chartGrafana).ToNot(BeEmpty())
+
+ mockRepo := func() *repository.ChartRepository {
+ return &repository.ChartRepository{
+ Client: &getter.MockGetter{
+ Response: chartGrafana,
+ },
+ Index: &repo.IndexFile{
+ Entries: map[string]repo.ChartVersions{
+ "grafana": {
+ &repo.ChartVersion{
+ Metadata: &helmchart.Metadata{
+ Name: "grafana",
+ Version: "6.17.4",
+ },
+ URLs: []string{"https://example.com/grafana.tgz"},
+ },
+ },
+ },
+ },
+ RWMutex: &sync.RWMutex{},
+ }
+ }
+
tests := []struct {
name string
baseDir string
@@ -45,12 +75,6 @@ func TestDependencyManager_Build(t *testing.T) {
wantChartFunc func(g *WithT, c *helmchart.Chart)
wantErr string
}{
- //{
- // // TODO(hidde): add various happy paths
- //},
- //{
- // // TODO(hidde): test Chart.lock
- //},
{
name: "build failure returns error",
baseDir: "./../testdata/charts",
@@ -61,7 +85,44 @@ func TestDependencyManager_Build(t *testing.T) {
name: "no dependencies returns zero",
baseDir: "./../testdata/charts",
path: "helmchart",
- want: 0,
+ wantChartFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(0))
+ },
+ want: 0,
+ },
+ {
+ name: "no dependency returns zero - v1",
+ baseDir: "./../testdata/charts",
+ path: "helmchart-v1",
+ wantChartFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(0))
+ },
+ want: 0,
+ },
+ {
+ name: "build with dependencies using lock file",
+ baseDir: "./../testdata/charts",
+ path: "helmchartwithdeps",
+ repositories: map[string]*repository.ChartRepository{
+ "https://grafana.github.io/helm-charts/": mockRepo(),
+ },
+ getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) {
+ return &repository.ChartRepository{URL: "https://grafana.github.io/helm-charts/"}, nil
+ },
+ wantChartFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(2))
+ g.Expect(c.Lock.Dependencies).To(HaveLen(3))
+ },
+ want: 2,
+ },
+ {
+ name: "build with dependencies - v1",
+ baseDir: "./../testdata/charts",
+ path: "helmchartwithdeps-v1",
+ wantChartFunc: func(g *WithT, c *helmchart.Chart) {
+ g.Expect(c.Dependencies()).To(HaveLen(1))
+ },
+ want: 1,
},
}
for _, tt := range tests {
@@ -71,10 +132,11 @@ func TestDependencyManager_Build(t *testing.T) {
chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path))
g.Expect(err).ToNot(HaveOccurred())
- got, err := NewDependencyManager(
+ dm := NewDependencyManager(
WithRepositories(tt.repositories),
WithRepositoryCallback(tt.getChartRepositoryCallback),
- ).Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart)
+ )
+ got, err := dm.Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
@@ -198,6 +260,10 @@ func TestDependencyManager_addLocalDependency(t *testing.T) {
return
}
g.Expect(err).ToNot(HaveOccurred())
+
+ if tt.wantFunc != nil {
+ tt.wantFunc(g, chart)
+ }
})
}
}
From 753abed30cf25ab901c3f895f460d80779f520e2 Mon Sep 17 00:00:00 2001
From: Sunny
Date: Tue, 16 Nov 2021 20:23:52 +0530
Subject: [PATCH 0231/1397] internal/helm: add remote builder tests
- For remote builds, if the build option has a version metadata, the
chart should be repackaged with the provided version.
- Update internal/helm/testdata/charts/helmchart-0.1.0.tgz to include
value files for testing merge chart values.
Signed-off-by: Sunny
---
internal/helm/chart/builder_remote.go | 7 +-
internal/helm/chart/builder_remote_test.go | 187 ++++++++++++++++++
.../helm/testdata/charts/helmchart-0.1.0.tgz | Bin 3277 -> 3354 bytes
3 files changed, 192 insertions(+), 2 deletions(-)
diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go
index ce1953655..2caceb39c 100644
--- a/internal/helm/chart/builder_remote.go
+++ b/internal/helm/chart/builder_remote.go
@@ -100,8 +100,9 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o
return nil, fmt.Errorf("failed to download chart for remote reference: %w", err)
}
- // Use literal chart copy from remote if no custom value files options are set
- if len(opts.GetValueFiles()) == 0 {
+ // Use literal chart copy from remote if no custom value files options are
+ // set or build option version metadata isn't set.
+ if len(opts.GetValueFiles()) == 0 && opts.VersionMetadata == "" {
if err = validatePackageAndWriteToPath(res, p); err != nil {
return nil, err
}
@@ -127,6 +128,8 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o
result.ValueFiles = opts.GetValueFiles()
}
+ chart.Metadata.Version = result.Version
+
// Package the chart with the custom values
if err = packageToPath(chart, p); err != nil {
return nil, err
diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go
index b7a2dae2f..431ac0a6c 100644
--- a/internal/helm/chart/builder_remote_test.go
+++ b/internal/helm/chart/builder_remote_test.go
@@ -17,13 +17,200 @@ limitations under the License.
package chart
import (
+ "bytes"
+ "context"
+ "math/rand"
+ "os"
+ "strings"
+ "sync"
"testing"
+ "time"
. "github.com/onsi/gomega"
helmchart "helm.sh/helm/v3/pkg/chart"
+ "helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
+ helmgetter "helm.sh/helm/v3/pkg/getter"
+
+ "github.com/fluxcd/source-controller/internal/helm/repository"
)
+var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890")
+
+func randStringRunes(n int) string {
+ b := make([]rune, n)
+ for i := range b {
+ b[i] = letterRunes[rand.Intn(len(letterRunes))]
+ }
+ return string(b)
+}
+
+func init() {
+ rand.Seed(time.Now().UnixNano())
+}
+
+// mockIndexChartGetter returns specific response for index and chart queries.
+type mockIndexChartGetter struct {
+ IndexResponse []byte
+ ChartResponse []byte
+ requestedURL string
+}
+
+func (g *mockIndexChartGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error) {
+ g.requestedURL = u
+ r := g.ChartResponse
+ if strings.HasSuffix(u, "index.yaml") {
+ r = g.IndexResponse
+ }
+ return bytes.NewBuffer(r), nil
+}
+
+func (g *mockIndexChartGetter) LastGet() string {
+ return g.requestedURL
+}
+
+func TestRemoteBuilder_Build(t *testing.T) {
+ g := NewWithT(t)
+
+ chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(chartGrafana).ToNot(BeEmpty())
+
+ index := []byte(`
+apiVersion: v1
+entries:
+ grafana:
+ - urls:
+ - https://example.com/grafana.tgz
+ description: string
+ version: 6.17.4
+`)
+
+ mockGetter := &mockIndexChartGetter{
+ IndexResponse: index,
+ ChartResponse: chartGrafana,
+ }
+
+ mockRepo := func() *repository.ChartRepository {
+ return &repository.ChartRepository{
+ URL: "https://grafana.github.io/helm-charts/",
+ Client: mockGetter,
+ RWMutex: &sync.RWMutex{},
+ }
+ }
+
+ tests := []struct {
+ name string
+ reference Reference
+ buildOpts BuildOptions
+ repository *repository.ChartRepository
+ wantValues chartutil.Values
+ wantVersion string
+ wantPackaged bool
+ wantErr string
+ }{
+ {
+ name: "invalid reference",
+ reference: LocalReference{},
+ wantErr: "expected remote chart reference",
+ },
+ {
+ name: "invalid reference - no name",
+ reference: RemoteReference{},
+ wantErr: "no name set for remote chart reference",
+ },
+ {
+ name: "chart not in repo",
+ reference: RemoteReference{Name: "foo"},
+ repository: mockRepo(),
+ wantErr: "failed to get chart version for remote reference",
+ },
+ {
+ name: "chart version not in repo",
+ reference: RemoteReference{Name: "grafana", Version: "1.1.1"},
+ repository: mockRepo(),
+ wantErr: "failed to get chart version for remote reference",
+ },
+ {
+ name: "invalid version metadata",
+ reference: RemoteReference{Name: "grafana"},
+ repository: mockRepo(),
+ buildOpts: BuildOptions{VersionMetadata: "^"},
+ wantErr: "Invalid Metadata string",
+ },
+ {
+ name: "with version metadata",
+ reference: RemoteReference{Name: "grafana"},
+ repository: mockRepo(),
+ buildOpts: BuildOptions{VersionMetadata: "foo"},
+ wantVersion: "6.17.4+foo",
+ wantPackaged: true,
+ },
+ // TODO: Test setting BuildOptions CachedChart and Force.
+ {
+ name: "default values",
+ reference: RemoteReference{Name: "grafana"},
+ repository: mockRepo(),
+ wantVersion: "0.1.0",
+ wantValues: chartutil.Values{
+ "replicaCount": float64(1),
+ },
+ },
+ {
+ name: "merge values",
+ reference: RemoteReference{Name: "grafana"},
+ buildOpts: BuildOptions{
+ ValueFiles: []string{"a.yaml", "b.yaml", "c.yaml"},
+ },
+ repository: mockRepo(),
+ wantVersion: "6.17.4",
+ wantValues: chartutil.Values{
+ "a": "b",
+ "b": "d",
+ },
+ wantPackaged: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ targetPath := "/tmp/remote-chart-builder-" + randStringRunes(5) + ".tgz"
+ defer os.RemoveAll(targetPath)
+
+ if tt.repository != nil {
+ _, err := tt.repository.CacheIndex()
+ g.Expect(err).ToNot(HaveOccurred())
+ // Cleanup the cache index path.
+ defer os.Remove(tt.repository.CachePath)
+ }
+
+ b := NewRemoteBuilder(tt.repository)
+
+ cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts)
+
+ if tt.wantErr != "" {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
+ g.Expect(cb).To(BeZero())
+ return
+ }
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value")
+ g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path")
+
+ // Load the resulting chart and verify the values.
+ resultChart, err := loader.Load(cb.Path)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion))
+
+ for k, v := range tt.wantValues {
+ g.Expect(v).To(Equal(resultChart.Values[k]))
+ }
+ })
+ }
+}
+
func Test_mergeChartValues(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/helm/testdata/charts/helmchart-0.1.0.tgz b/internal/helm/testdata/charts/helmchart-0.1.0.tgz
index f64a32eeeb54fc24a44390478a3adf5bcb5cf754..1ffdde531f6c9e4ee48963ba512a302970994d14 100644
GIT binary patch
delta 3327
zcmV~>nKMg}uYx;qe}X4}UF48{|q8xflLnUd_pU
zCqOd#1*M{bM=(9`NR}->_JZDV5PAu!Skg?}ckkeTk)>dF42*1&CdQG
z1NPwzMA3tl8Dr!DNX&$yTF6-hd`$U`
z2ax5A4Fsd{EQ0gVh0p_u3b`(Z4RXdVF_x%R1Ydu)o`2}$6s4pIM&SG2C}(W#rKeC%
zX^ijU*h=96?89Y-F&$AO
zAV#1kxJE<^$Q1?__k09gXi%tu@T8z+bZ)4<2I2@&1!LqW4M)focKVt<@Vs%NKyYrN4B%ompBNc|v1o<(pFhM`eKJ6cRKGNg=ZipnVnAdw;~pF8mG
z!w35;kt82En|%^O{|{xTW=dmPQ;5v@J9QzrL1@1Ihe8=auMov5%5ZQIPVeGgzFl(w9W0?dcVMZ^x?kThc$FcA}GzX!;Op15rZpjr8}Z-y$nXD@8r
zTp`fWOcffgnC&}yomO6+L?v<=qgt-rKM``36UJsBF&1fxJhA&s3s82O&mbv`e>vK2
zKz}WOA(k0*xxkCaohD1ot*naCj#a83BiOlre&?q;i<
zeF(p)&+ON~$NyLHf9ajO7au>rzYO$^exMk0NBlnwkB=Jhzkl36*v0?HfL;JUpno%z
zTBZ*F`1ApWGh?s3h%*YoSQX)kioE%}19Swz7nd`ZzO-<}uvl0ON%$DynQ>qcovdM7
z$7XN=D(t|zOZRr7bvB