Add grafana monitoring

infrastructure/clair_db.tf

@@ -3,7 +3,7 @@ resource "aws_db_instance" "clair_db" {
   instance_class         = "db.t3.micro"
   allocated_storage      = 50
   engine                 = "postgres"
-  engine_version         = "14.2"
+  engine_version         = var.clair_db_version
   name                   = "clair"
   username               = "clair"
   password               = var.db_password

infrastructure/main.tf

@@ -1,3 +1,15 @@
+terraform {
+  required_providers {
+    aws  = {
+      version = "3.47.0"
+    }
+    tls = {
+      source = "hashicorp/tls"
+      version = "3.4.0"
+    }
+  }
+}
+
 provider "aws" {
   region = var.region
   profile = var.aws_profile
@@ -69,6 +81,11 @@ data "template_file" "quay_template" {
     builder_ssh_keypair = "${var.builder_ssh_keypair}"
 
     registry_state = local.is_secondary == 1 ? "readonly" : "normal"
+
+    enable_monitoring = var.enable_monitoring
+    prometheus_image = "${var.prometheus_image}"
+    grafana_image = "${var.grafana_image}"
+    prometheus_host = var.enable_monitoring ? "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}" : ""
   }
 }
 

infrastructure/monitoring_dns.tf

@@ -0,0 +1 @@
+

infrastructure/monitoring_lb_service.tf

@@ -0,0 +1,44 @@
+resource "kubernetes_service" "prometheus_lb_service" {
+  count = var.enable_monitoring ? 1 : 0
+  metadata {
+    name = "${var.prefix}-prometheus-lb"
+    namespace = "${kubernetes_namespace.quay_ns.metadata[0].name}"
+  }
+
+  spec {
+    selector = {
+      "app" = "prometheus-app"
+    }
+    port {
+      name        = "prometheus"
+      port        = 9090
+      target_port = 9090
+    }
+    type = "LoadBalancer"
+  }
+}
+
+resource "kubernetes_service" "grafana_lb_service" {
+  count = var.enable_monitoring ? 1 : 0
+  metadata {
+    name = "${var.prefix}-grafana-lb"
+    namespace = "${kubernetes_namespace.quay_ns.metadata[0].name}"
+  }
+  spec {
+    selector = {
+      "app" = "grafana-app"
+    }
+    port {
+      name        = "grafana-http"
+      port        = 80
+      target_port = 3000
+    }
+    port {
+      name        = "grafana-https"
+      port        = 443
+      target_port = 3000
+    }
+    type = "LoadBalancer"
+  }
+
+}

infrastructure/outputs.tf

@@ -40,6 +40,19 @@ output "quay_hostname" {
   sensitive   = false
 }
 
+output "prometheus_hostname" {
+  description = "Prometheus hostname"
+  value = var.enable_monitoring ? "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}" : null
+  sensitive = false
+}
+
+output "grafana_hostname" {
+  description = "Grafana hostname"
+  value = var.enable_monitoring ? "grafana-${var.prefix}.${data.aws_route53_zone.zone.name}" : null
+  sensitive = false
+
+}
+
 output "lb_name" {
   description = "Quay hostname"
   value = "${kubernetes_service.quay_lb_service.status.0.load_balancer.0.ingress.0.hostname}"

infrastructure/quay_alb.tf

@@ -74,4 +74,28 @@ resource "aws_alb_target_group" "quay_alb_grpc_target_group" {
   }
 }
 
+resource "aws_alb_listener" "quay_alb_metrics_listener" {
+  load_balancer_arn = aws_lb.quay_alb.arn
+  port              = "9091"
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-2016-08"
+  certificate_arn   = aws_acm_certificate.quay_domain_cert.arn
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.quay_alb_grpc_target_group.arn
+  }
+}
+
+resource "aws_alb_target_group" "quay_alb_metrics_target_group" {
+  name     = "${var.prefix}-alb-metrics-tg"
+  port     = "9091"
+  protocol = "HTTPS"
+  target_type = "ip"
+  vpc_id   = module.quay_vpc.vpc_id
+  health_check {
+     port = 443
+  }
+}
+
 /* TODO: Add IPs of ELB automatically to the target group */

infrastructure/quay_db.tf

@@ -3,7 +3,7 @@ resource "aws_db_instance" "quay_db" {
   instance_class         = "db.t3.micro"
   allocated_storage      = 5
   engine                 = "mysql"
-  engine_version         = "5.7.33"
+  engine_version         = var.quay_db_version
   name                   = "quay"
   username               = "quay"
   password               = var.db_password

infrastructure/quay_deployment.yaml.tpl

@@ -181,6 +181,10 @@ spec:
       protocol: TCP
       port: 55443
       targetPort: 55443
+    - name: metrics
+      protocol: TCP
+      port: 9091
+      targetPort: 9091
   selector:
     quay-component: quay-app
 ---
@@ -210,6 +214,10 @@ spec:
       protocol: TCP
       port: 55443
       targetPort: 55443
+    - name: metrics
+      protocol: TCP
+      port: 9091
+      targetPort: 9091
   selector:
     quay-component: quay-app
 ---
@@ -252,6 +260,7 @@ stringData:
 
   config.yaml: |
 
+    REGISTRY_STATE: ${registry_state}
     ALLOW_PULLS_WITHOUT_STRICT_LOGGING: false
     AUTHENTICATION_TYPE: Database
     DATABASE_SECRET_KEY: db-secret-key
@@ -373,6 +382,10 @@ spec:
       protocol: TCP
       port: 55443
       targetPort: 55443
+    - name: metrics
+      protocol: TCP
+      port: 9091
+      targetPort: 9091
   selector:
     quay-component: quay-app
 ---
@@ -506,7 +519,7 @@ stringData:
     ${ssl_key}
 
   config.yaml: |
-    REGISTRY_STATE=${registry_state}
+    REGISTRY_STATE: ${registry_state}
     ALLOW_PULLS_WITHOUT_STRICT_LOGGING: false
     AUTHENTICATION_TYPE: Database
     DATABASE_SECRET_KEY: db-secret-key
@@ -524,6 +537,8 @@ stringData:
         s3_access_key: ${s3_access_key_id}
         s3_secret_key: ${s3_secret_key}
         s3_bucket: ${s3_bucket_name}
+        s3_region: us-east-1
+        cloudfront_distribution_org_overrides: {}
         storage_path: "/images"
     DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS:
     - s3_us_west_1
@@ -615,3 +630,123 @@ stringData:
             url: ""
 
 %{ endif }
+
+%{ if enable_monitoring}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: ${namespace}
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+      evaluation_interval: 15s
+    scrape_configs:
+      - job_name: 'quay'
+        static_configs:
+          - targets: ["${quay_route_host}:9091"]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: ${namespace}
+  labels:
+    app: prometheus-app
+spec: 
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus-app
+  template:
+    metadata:
+      labels:
+        app: prometheus-app
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
+    spec:
+      containers:
+      - name: prometheus
+        image: ${prometheus_image}
+        args:
+          - '--storage.tsdb.retention=6h'
+          - '--storage.tsdb.path=/prometheus'
+          - '--config.file=/etc/prometheus/prometheus.yml'
+        ports:
+          - name: web
+            containerPort: 9090
+        volumeMounts:
+          - name: prometheus-config-volume
+            mountPath: /etc/prometheus
+          - name: prometheus-storage-volume
+            mountPath: /prometheus/
+      restartPolicy: Always
+      volumes:
+      - name: prometheus-config-volume
+        configMap:
+          defaultMode: 420
+          name: prometheus-config
+      - name: prometheus-storage-volume
+        emptyDir: {}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  namespace: "${namespace}"
+data:
+  prometheus.yaml: |-
+    {
+        "apiVersion": 1,
+        "datasources": [
+            {
+                "access": "proxy",
+                "editable": true,
+                "name": quay-prometheus,
+                "orgId": 1,
+                "type": prometheus,
+                "url": "http://${prometheus_host}:9090",
+                "version": 1
+            }
+        ]
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: "${namespace}"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana-app
+  template:
+    metadata:
+      name: grafana-app
+      labels:
+        app: grafana-app
+    spec:
+      containers:
+      - name: grafana
+        image: ${grafana_image}
+        ports:
+          - name: grafana
+            containerPort: 3000
+        volumeMounts:
+          - mountPath: /var/lib/grafana
+            name: grafana-storage
+          - mountPath: /etc/grafana/provisioning/datasources
+            name: grafana-datasources
+            readOnly: false
+      volumes:
+        - name: grafana-storage
+          emptyDir: {}
+        - name: grafana-datasources
+          configMap:
+            defaultMode: 420
+            name: grafana-datasources
+%{ endif }

infrastructure/quay_dns.tf

@@ -21,4 +21,21 @@ resource "aws_route53_record" "quay_hostname" {
   records = ["${kubernetes_service.quay_lb_service.status.0.load_balancer.0.ingress.0.hostname}"]
 }
 
+resource "aws_route53_record" "prometheus_hostname" {
+  count = var.enable_monitoring ? 1 : 0
+  zone_id = data.aws_route53_zone.zone.zone_id
+  name = "prometheus-${var.prefix}.${data.aws_route53_zone.zone.name}"
+  type = "CNAME"
+  ttl = "300"
+  records = ["${kubernetes_service.prometheus_lb_service[0].status.0.load_balancer.0.ingress.0.hostname}"]
+}
+
+resource "aws_route53_record" "grafana_hostname" {
+  count = var.enable_monitoring ? 1 : 0
+  zone_id = data.aws_route53_zone.zone.zone_id
+  name = "grafana-${var.prefix}.${data.aws_route53_zone.zone.name}"
+  type = "CNAME"
+  ttl = "300"
+  records = ["${kubernetes_service.grafana_lb_service[0].status.0.load_balancer.0.ingress.0.hostname}"]
+}
 

infrastructure/quay_lb_service.tf

@@ -28,6 +28,11 @@ resource "kubernetes_service" "quay_lb_service" {
       port        = 55443
       target_port = 55443
     }
+    port {
+      name        = "metrics"
+      port        = 9091
+      target_port = 9091
+    }
 
     type = "LoadBalancer"
   }

infrastructure/variables.tf

@@ -129,3 +129,33 @@ variable "redis_azs" {
   type = list
   default = ["us-east-1a", "us-east-1b"]
 }
+
+variable "enable_monitoring" {
+  description = "enable prometheus/grafana monitoring for quay"
+  type = bool
+  default = false
+}
+
+variable "prometheus_image" {
+  description = "image for prometheus container"
+  type = string
+  default = "prom/prometheus"
+}
+
+variable "grafana_image" {
+  description = "image for grafana container"
+  type = string
+  default = "grafana/grafana"
+}
+
+variable "quay_db_version" {
+  description = "version of quay's database"
+  type = string
+  default = "5.7.41"
+}
+
+variable "clair_db_version" {
+  description = "version of clair's database"
+  type = string
+  default = "14.2"
+}