adding integrationbs docker-compose service for assets loading into o…

…pensearch / dashboards

Signed-off-by: YANGDB <[email protected]>

.env

@@ -43,7 +43,7 @@ EMAIL_SERVICE_PORT=6060
 EMAIL_SERVICE_ADDR=http://emailservice:${EMAIL_SERVICE_PORT}
 
 # Feature Flag Service
-FEATURE_FLAG_SERVICE_PORT=8081
+FEATURE_FLAG_SERVICE_PORT=8881
 FEATURE_FLAG_SERVICE_ADDR=featureflagservice:${FEATURE_FLAG_SERVICE_PORT}
 FEATURE_FLAG_SERVICE_HOST=feature-flag-service
 FEATURE_FLAG_GRPC_SERVICE_PORT=50053

add_hosts_locally.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Local host that are used to help developing and debugging the OTEL demo locally
+
+# The IP address you want to associate with the hostname
+IP="127.0.0.1"
+
+# The hostname you want to associate with the IP address
+OPENSEARCH_HOST="opensearch-node1"
+OPENSEARCH_DASHBOARD="opensearch-dashboards"
+OTEL_STORE="frontend"
+OTEL_LOADER="loadgenerator"
+PROMETHEUS="prometheus"
+
+# Add the entry to the /etc/hosts file
+echo "$IP    $OPENSEARCH_HOST" | sudo tee -a /etc/hosts
+echo "$IP    $OPENSEARCH_DASHBOARD" | sudo tee -a /etc/hosts
+echo "$IP    $OTEL_STORE" | sudo tee -a /etc/hosts
+echo "$IP    $PROMETHEUS" | sudo tee -a /etc/hosts
+echo "$IP    $OTEL_LOADER" | sudo tee -a /etc/hosts

docker-compose.yml

@@ -220,7 +220,7 @@ services:
           memory: 175M
     restart: unless-stopped
     ports:
-      - "${FEATURE_FLAG_SERVICE_PORT}"                                  # Feature Flag Service UI
+      - "${FEATURE_FLAG_SERVICE_PORT}:${FEATURE_FLAG_SERVICE_PORT}"     # Feature Flag Service UI
       - "${FEATURE_FLAG_GRPC_SERVICE_PORT}"                             # Feature Flag Service gRPC API
     environment:
       - FEATURE_FLAG_SERVICE_PORT
@@ -306,6 +306,7 @@ services:
       - quoteservice
       - recommendationservice
       - shippingservice
+      - paymentservice
     logging: *logging
 
   # Load Generator
@@ -463,6 +464,7 @@ services:
       - frontend
       - fluentbit
       - otelcol
+      - loadgenerator
     links:
       - fluentbit
     logging:
@@ -712,7 +714,7 @@ services:
     volumes:
       - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://opensearch-node1:9200/_cluster/health?wait_for_status=yellow"]
+      test: ["CMD", "curl", "-f", "https://opensearch-node1:9200/_cluster/health?wait_for_status=yellow"]
       interval: 5s
       timeout: 25s
       retries: 4
@@ -755,10 +757,14 @@ services:
       - opensearch-node1
 
 #   Observability OSD Integrations
-#  observability-integration:
-#    build: ./src/integrations/Dockerfile
-#    depends_on:
-#      - opensearch-dashboards
+  integrations:
+    build:
+      context: ./src/integrations
+      dockerfile: Dockerfile
+    volumes:
+      - ./src/integrations:/integrations
+    depends_on:
+      - opensearch-dashboards
 
   # *****
   # Tests

src/fluent-bit/fluent-bit.conf

@@ -1,6 +1,7 @@
 [SERVICE]
     Parsers_File parsers.conf
-
+    Log_Level info
+    Daemon off
 [INPUT]
     Name forward
     Port 24224
@@ -19,12 +20,17 @@
 
 [OUTPUT]
     Name  opensearch
-    Match nginx.access
-    Host  opensearch
+    Match nginx.*
+    Host  opensearch-node1
     Port  9200
+    tls   On
+    tls.verify Off
+    HTTP_Passwd     admin
+    HTTP_User       admin
     Index sso_logs-nginx-prod
+    Generate_ID On
+    Retry_Limit False
     Suppress_Type_Name On
-
 [OUTPUT]
     Name stdout
     Match nginx.access

src/integrations/Dockerfile

@@ -1,8 +1,10 @@
-FROM alpine:latest
+FROM python:3.9
 
-RUN apk update && apk add curl netcat-openbsd
+WORKDIR .
 
-COPY install-observability.sh /install-observability.sh
-RUN chmod +x /install-observability.sh
+COPY src/requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
 
-CMD /install-observability.sh
+COPY . .
+
+CMD [ "python", "./src/install.py" ]

src/integrations/datasource/prometheus.json

@@ -0,0 +1,8 @@
+{
+  "name" : "prometheus",
+  "connector": "prometheus",
+  "properties" : {
+    "prometheus.uri" : "http://prometheus:9090"
+  },
+  "allowedRoles" : ["prometheus_access"]
+}

src/integrations/indices/data-stream.json

@@ -0,0 +1,3 @@
+{
+  "data-stream" : ["sso_logs-nginx-prod","opensearch-prod"]
+}

src/integrations/install-observability.sh

@@ -12,27 +12,5 @@ until nc -z -v -w30 "$host" "$port"; do
 done
 
 echo "Service is up and ready! Executing command:"
-# create datasource for prometheus
-echo "create datasource for prometheus"
+echo "importing assets into opensearch"
 exec $cmd
-
-# create index template for logs,traces,metrics
-echo "create index template for logs,traces,metrics,serviceMaps"
-exec $cmd
-
-# create data-streams for logs,traces,metrics
-echo "create data-streams for logs,traces,metrics"
-exec $cmd
-
-# create email channel for alerting
-echo " create email channel for alerting"
-exec $cmd
-
-# create saves searches for services KPI
-echo " create saves searches for services KPI"
-exec $cmd
-
-# create Alerts queries for monitoring services
-echo " create Alerts queries for monitoring services
-exec $cmd
-

src/integrations/mapping-components/cloud.mapping

@@ -0,0 +1,70 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "cloud": {
+          "properties": {
+            "provider": {
+              "type": "keyword"
+            },
+            "availability_zone": {
+              "type": "keyword"
+            },
+            "region": {
+              "type": "keyword"
+            },
+            "machine": {
+              "type": "object",
+              "properties": {
+                "type": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "account": {
+              "type": "object",
+              "properties": {
+                "id": {
+                  "type": "keyword"
+                },
+                "name": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "service": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "project": {
+              "type": "object",
+              "properties": {
+                "id": {
+                  "type": "keyword"
+                },
+                "name": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "instance": {
+              "type": "object",
+              "properties": {
+                "id": {
+                  "type": "keyword"
+                },
+                "name": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/integrations/mapping-components/communication.mapping

@@ -0,0 +1,96 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "communication": {
+          "properties": {
+            "sock.family": {
+              "type": "keyword",
+              "ignore_above": 256
+            },
+            "source": {
+              "type": "object",
+              "properties": {
+                "address": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 1024
+                    }
+                  }
+                },
+                "domain": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 1024
+                    }
+                  }
+                },
+                "bytes": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                },
+                "ip": {
+                  "type": "ip"
+                },
+                "port": {
+                  "type": "long"
+                },
+                "mac": {
+                  "type": "keyword",
+                  "ignore_above": 1024
+                },
+                "packets": {
+                  "type": "long"
+                }
+              }
+            },
+            "destination": {
+              "type": "object",
+              "properties": {
+                "address": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 1024
+                    }
+                  }
+                },
+                "domain": {
+                  "type": "text",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword",
+                      "ignore_above": 1024
+                    }
+                  }
+                },
+                "bytes": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                },
+                "ip": {
+                  "type": "ip"
+                },
+                "port": {
+                  "type": "long"
+                },
+                "mac": {
+                  "type": "keyword",
+                  "ignore_above": 1024
+                },
+                "packets": {
+                  "type": "long"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}