diff --git a/infrastructure/clair_db.tf b/infrastructure/clair_db.tf index 25aff7f..09d3279 100644 --- a/infrastructure/clair_db.tf +++ b/infrastructure/clair_db.tf @@ -3,7 +3,7 @@ resource "aws_db_instance" "clair_db" { instance_class = "db.t3.micro" allocated_storage = 50 engine = "postgres" - engine_version = "14.2" + engine_version = var.clair_db_version name = "clair" username = "clair" password = var.db_password diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 94a4028..a2f14e2 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -1,3 +1,15 @@ +terraform { + required_providers { + aws = { + version = "3.47.0" + } + tls = { + source = "hashicorp/tls" + version = "3.4.0" + } + } +} + provider "aws" { region = var.region profile = var.aws_profile @@ -69,6 +81,11 @@ data "template_file" "quay_template" { builder_ssh_keypair = "${var.builder_ssh_keypair}" registry_state = local.is_secondary == 1 ? "readonly" : "normal" + + enable_monitoring = var.enable_monitoring + prometheus_image = "${var.prometheus_image}" + grafana_image = "${var.grafana_image}" + prometheus_host = var.enable_monitoring ? "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}" : "" } } diff --git a/infrastructure/monitoring_dns.tf b/infrastructure/monitoring_dns.tf new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/infrastructure/monitoring_dns.tf @@ -0,0 +1 @@ + diff --git a/infrastructure/monitoring_lb_service.tf b/infrastructure/monitoring_lb_service.tf new file mode 100644 index 0000000..41fb799 --- /dev/null +++ b/infrastructure/monitoring_lb_service.tf @@ -0,0 +1,44 @@ +resource "kubernetes_service" "prometheus_lb_service" { + count = var.enable_monitoring ? 1 : 0 + metadata { + name = "${var.prefix}-prometheus-lb" + namespace = "${kubernetes_namespace.quay_ns.metadata[0].name}" + } + + spec { + selector = { + "app" = "prometheus-app" + } + port { + name = "prometheus" + port = 9090 + target_port = 9090 + } + type = "LoadBalancer" + } +} + +resource "kubernetes_service" "grafana_lb_service" { + count = var.enable_monitoring ? 1 : 0 + metadata { + name = "${var.prefix}-grafana-lb" + namespace = "${kubernetes_namespace.quay_ns.metadata[0].name}" + } + spec { + selector = { + "app" = "grafana-app" + } + port { + name = "grafana-http" + port = 80 + target_port = 3000 + } + port { + name = "grafana-https" + port = 443 + target_port = 3000 + } + type = "LoadBalancer" + } + +} diff --git a/infrastructure/outputs.tf b/infrastructure/outputs.tf index dfd8e02..7cc1dbb 100644 --- a/infrastructure/outputs.tf +++ b/infrastructure/outputs.tf @@ -40,6 +40,19 @@ output "quay_hostname" { sensitive = false } +output "prometheus_hostname" { + description = "Prometheus hostname" + value = var.enable_monitoring ? "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}" : null + sensitive = false +} + +output "grafana_hostname" { + description = "Grafana hostname" + value = var.enable_monitoring ? "grafana-${var.prefix}.${data.aws_route53_zone.zone.name}" : null + sensitive = false + +} + output "lb_name" { description = "Quay hostname" value = "${kubernetes_service.quay_lb_service.status.0.load_balancer.0.ingress.0.hostname}" diff --git a/infrastructure/quay_alb.tf b/infrastructure/quay_alb.tf index 333ad6a..00f8ba9 100644 --- a/infrastructure/quay_alb.tf +++ b/infrastructure/quay_alb.tf @@ -74,4 +74,28 @@ resource "aws_alb_target_group" "quay_alb_grpc_target_group" { } } +resource "aws_alb_listener" "quay_alb_metrics_listener" { + load_balancer_arn = aws_lb.quay_alb.arn + port = "9091" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_acm_certificate.quay_domain_cert.arn + + default_action { + type = "forward" + target_group_arn = aws_alb_target_group.quay_alb_grpc_target_group.arn + } +} + +resource "aws_alb_target_group" "quay_alb_metrics_target_group" { + name = "${var.prefix}-alb-metrics-tg" + port = "9091" + protocol = "HTTPS" + target_type = "ip" + vpc_id = module.quay_vpc.vpc_id + health_check { + port = 443 + } +} + /* TODO: Add IPs of ELB automatically to the target group */ diff --git a/infrastructure/quay_db.tf b/infrastructure/quay_db.tf index 226c815..1a101e5 100644 --- a/infrastructure/quay_db.tf +++ b/infrastructure/quay_db.tf @@ -3,7 +3,7 @@ resource "aws_db_instance" "quay_db" { instance_class = "db.t3.micro" allocated_storage = 5 engine = "mysql" - engine_version = "5.7.33" + engine_version = var.quay_db_version name = "quay" username = "quay" password = var.db_password diff --git a/infrastructure/quay_deployment.yaml.tpl b/infrastructure/quay_deployment.yaml.tpl index f47da71..e47b1a6 100644 --- a/infrastructure/quay_deployment.yaml.tpl +++ b/infrastructure/quay_deployment.yaml.tpl @@ -181,6 +181,10 @@ spec: protocol: TCP port: 55443 targetPort: 55443 + - name: metrics + protocol: TCP + port: 9091 + targetPort: 9091 selector: quay-component: quay-app --- @@ -210,6 +214,10 @@ spec: protocol: TCP port: 55443 targetPort: 55443 + - name: metrics + protocol: TCP + port: 9091 + targetPort: 9091 selector: quay-component: quay-app --- @@ -252,6 +260,7 @@ stringData: config.yaml: | + REGISTRY_STATE: ${registry_state} ALLOW_PULLS_WITHOUT_STRICT_LOGGING: false AUTHENTICATION_TYPE: Database DATABASE_SECRET_KEY: db-secret-key @@ -373,6 +382,10 @@ spec: protocol: TCP port: 55443 targetPort: 55443 + - name: metrics + protocol: TCP + port: 9091 + targetPort: 9091 selector: quay-component: quay-app --- @@ -506,7 +519,7 @@ stringData: ${ssl_key} config.yaml: | - REGISTRY_STATE=${registry_state} + REGISTRY_STATE: ${registry_state} ALLOW_PULLS_WITHOUT_STRICT_LOGGING: false AUTHENTICATION_TYPE: Database DATABASE_SECRET_KEY: db-secret-key @@ -524,6 +537,8 @@ stringData: s3_access_key: ${s3_access_key_id} s3_secret_key: ${s3_secret_key} s3_bucket: ${s3_bucket_name} + s3_region: us-east-1 + cloudfront_distribution_org_overrides: {} storage_path: "/images" DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: - s3_us_west_1 @@ -615,3 +630,123 @@ stringData: url: "" %{ endif } + +%{ if enable_monitoring} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: ${namespace} +data: + prometheus.yml: | + global: + scrape_interval: 15s + evaluation_interval: 15s + scrape_configs: + - job_name: 'quay' + static_configs: + - targets: ["${quay_route_host}:9091"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: ${namespace} + labels: + app: prometheus-app +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-app + template: + metadata: + labels: + app: prometheus-app + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + containers: + - name: prometheus + image: ${prometheus_image} + args: + - '--storage.tsdb.retention=6h' + - '--storage.tsdb.path=/prometheus' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - name: web + containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus + - name: prometheus-storage-volume + mountPath: /prometheus/ + restartPolicy: Always + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-config + - name: prometheus-storage-volume + emptyDir: {} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: "${namespace}" +data: + prometheus.yaml: |- + { + "apiVersion": 1, + "datasources": [ + { + "access": "proxy", + "editable": true, + "name": quay-prometheus, + "orgId": 1, + "type": prometheus, + "url": "http://${prometheus_host}:9090", + "version": 1 + } + ] + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: "${namespace}" +spec: + replicas: 1 + selector: + matchLabels: + app: grafana-app + template: + metadata: + name: grafana-app + labels: + app: grafana-app + spec: + containers: + - name: grafana + image: ${grafana_image} + ports: + - name: grafana + containerPort: 3000 + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-storage + - mountPath: /etc/grafana/provisioning/datasources + name: grafana-datasources + readOnly: false + volumes: + - name: grafana-storage + emptyDir: {} + - name: grafana-datasources + configMap: + defaultMode: 420 + name: grafana-datasources +%{ endif } diff --git a/infrastructure/quay_dns.tf b/infrastructure/quay_dns.tf index 1ffcf2e..67f6192 100644 --- a/infrastructure/quay_dns.tf +++ b/infrastructure/quay_dns.tf @@ -21,4 +21,21 @@ resource "aws_route53_record" "quay_hostname" { records = ["${kubernetes_service.quay_lb_service.status.0.load_balancer.0.ingress.0.hostname}"] } +resource "aws_route53_record" "prometheus_hostname" { + count = var.enable_monitoring ? 1 : 0 + zone_id = data.aws_route53_zone.zone.zone_id + name = "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}" + type = "CNAME" + ttl = "300" + records = ["${kubernetes_service.prometheus_lb_service[0].status.0.load_balancer.0.ingress.0.hostname}"] +} + +resource "aws_route53_record" "grafana_hostname" { + count = var.enable_monitoring ? 1 : 0 + zone_id = data.aws_route53_zone.zone.zone_id + name = "grafana-${var.prefix}.${data.aws_route53_zone.zone.name}" + type = "CNAME" + ttl = "300" + records = ["${kubernetes_service.grafana_lb_service[0].status.0.load_balancer.0.ingress.0.hostname}"] +} diff --git a/infrastructure/quay_lb_service.tf b/infrastructure/quay_lb_service.tf index 172179e..3594e74 100644 --- a/infrastructure/quay_lb_service.tf +++ b/infrastructure/quay_lb_service.tf @@ -28,6 +28,11 @@ resource "kubernetes_service" "quay_lb_service" { port = 55443 target_port = 55443 } + port { + name = "metrics" + port = 9091 + target_port = 9091 + } type = "LoadBalancer" } diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf index 2ccef3a..076b4d0 100644 --- a/infrastructure/variables.tf +++ b/infrastructure/variables.tf @@ -129,3 +129,33 @@ variable "redis_azs" { type = list default = ["us-east-1a", "us-east-1b"] } + +variable "enable_monitoring" { + description = "enable prometheus/grafana monitoring for quay" + type = bool + default = false +} + +variable "prometheus_image" { + description = "image for prometheus container" + type = string + default = "prom/prometheus" +} + +variable "grafana_image" { + description = "image for grafana container" + type = string + default = "grafana/grafana" +} + +variable "quay_db_version" { + description = "version of quay's database" + type = string + default = "5.7.41" +} + +variable "clair_db_version" { + description = "version of clair's database" + type = string + default = "14.2" +}