Keep scrape config in line with the new Prometheus scrape config

This is triggered by grafana/jsonnet-libs#261 .

The above PR removes the `instance` label. As it has turned out (see
PR linked above), a sane `instance` label in Prometheus has to be
unique, and that includes the case where a single container exposes
metrics on two different endpoints. However, that scenario would still
only result in one log stream for Loki to scrape.

Therefore, Loki and Prometheus need to sync via target labels uniquely
identifying a container (rather than a metrics endpoint). Those labels
are namespace, pod, container, also added here.

This commit removes the `container_name` label. It is the same as the
`container` label and was already added to Loki previously. However,
the `container_name` label is deprecated and has disappeared in K8s
1.16, so that it will soon become useless for direct joining.

Author: beorn7

production/helm/promtail/templates/configmap.yaml

@@ -47,11 +47,11 @@ data:
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_name
-        target_label: instance
+        target_label: pod
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_container_name
-        target_label: container_name
+        target_label: container
       - replacement: /var/log/pods/*$1/*.log
         separator: /
         source_labels:
@@ -94,11 +94,11 @@ data:
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_name
-        target_label: instance
+        target_label: pod
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_container_name
-        target_label: container_name
+        target_label: container
       - replacement: /var/log/pods/*$1/*.log
         separator: /
         source_labels:
@@ -147,11 +147,11 @@ data:
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_name
-        target_label: instance
+        target_label: pod
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_container_name
-        target_label: container_name
+        target_label: container
       - replacement: /var/log/pods/*$1/*.log
         separator: /
         source_labels:
@@ -202,11 +202,11 @@ data:
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_name
-        target_label: instance
+        target_label: pod
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_container_name
-        target_label: container_name
+        target_label: container
       - replacement: /var/log/pods/*$1/*.log
         separator: /
         source_labels:
@@ -250,11 +250,11 @@ data:
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_name
-        target_label: instance
+        target_label: pod
       - action: replace
         source_labels:
         - __meta_kubernetes_pod_container_name
-        target_label: container_name
+        target_label: container
       - replacement: /var/log/pods/*$1/*.log
         separator: /
         source_labels:

production/ksonnet/promtail/scrape_config.libsonnet

@@ -31,7 +31,7 @@ config {
         regex: '__meta_kubernetes_pod_label_(.+)',
       },
 
-      // Rename jobs to be <namespace>/<name, from pod name label>
+      // Rename jobs to be <namespace>/<service>.
       {
         source_labels: ['__meta_kubernetes_namespace', '__service__'],
         action: 'replace',
@@ -40,25 +40,24 @@ config {
         replacement: '$1',
       },
 
-      // But also include the namespace as a separate label, for routing alerts
+      // But also include the namespace, pod, container as separate
+      // labels. They uniquely identify a container. They are also
+      // identical to the target labels configured in Prometheus
+      // (but note that Loki does not use an instance label).
       {
         source_labels: ['__meta_kubernetes_namespace'],
         action: 'replace',
         target_label: 'namespace',
       },
-
-      // Rename instances to be the pod name
       {
         source_labels: ['__meta_kubernetes_pod_name'],
         action: 'replace',
-        target_label: 'instance',
+        target_label: 'pod',  // Not 'pod_name', which disappeared in K8s 1.16.
       },
-
-      // Include container_name label
       {
         source_labels: ['__meta_kubernetes_pod_container_name'],
         action: 'replace',
-        target_label: 'container_name',
+        target_label: 'container',  // Not 'container_name', which disappeared in K8s 1.16.
       },
 
       // Kubernetes puts logs under subdirectories keyed pod UID and container_name.
@@ -76,7 +75,6 @@ config {
       // Scrape config to scrape any pods with a 'name' label.
       gen_scrape_config('kubernetes-pods-name', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
-
           // Use name label as __service__.
           {
             source_labels: ['__meta_kubernetes_pod_label_name'],
@@ -85,7 +83,7 @@ config {
         ],
       },
 
-      // Scrape config to scrape any pods with a 'app' label.
+      // Scrape config to scrape any pods with an 'app' label.
       gen_scrape_config('kubernetes-pods-app', '__meta_kubernetes_pod_uid') {
         prelabel_config:: [
           // Drop pods with a 'name' label.  They will have already been added by