adding datasource for query prometheus

Signed-off-by: YANGDB <[email protected]>

add_hosts_locally.sh

@@ -1,3 +1,6 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
 #!/bin/bash
 # Local host that are used to help developing and debugging the OTEL demo locally
 

docker-compose.yml

@@ -714,7 +714,7 @@ services:
     volumes:
       - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
     healthcheck:
-      test: ["CMD", "curl", "-f", "https://opensearch-node1:9200/_cluster/health?wait_for_status=yellow"]
+      test: ["CMD", "curl", "-f", "https://opensearch-node1:9200/_cluster/health?wait_for_status=yellow", "-ku admin:admin"]
       interval: 5s
       timeout: 25s
       retries: 4
@@ -745,7 +745,7 @@ services:
 
   # OpenSearch store - dashboard
   opensearch-dashboards:
-    image: opensearchproject/opensearch-dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes
+    image: opensearchproject/opensearch-dashboards:2.6.0 # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes
     container_name: opensearch-dashboards
     ports:
       - 5601:5601 # Map host port 5601 to container port 5601

local-config.md

@@ -0,0 +1,33 @@
+# Local Dev Config
+
+Use the following configuration to locally run and test the OTEL demo:
+
+- For additional help go [here](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/)
+
+### Raise your host's ulimits:
+Rais the upper liits for OpenSearch to be able handling high I/O :
+
+`sudo sysctl -w vm.max_map_count=512000`
+
+### Map domain name to local dns
+
+run the [following script](add_hosts_locally.sh) to map the docker-compose service names to your local dns
+
+```text
+# The hostname you want to associate with the IP address
+
+OPENSEARCH_HOST="opensearch-node1"
+OPENSEARCH_DASHBOARD="opensearch-dashboards"
+OTEL_STORE="frontend"
+OTEL_LOADER="loadgenerator"
+PROMETHEUS="prometheus"
+
+# Add the entry to the /etc/hosts file
+
+echo "$IP    $OPENSEARCH_HOST" | sudo tee -a /etc/hosts
+echo "$IP    $OPENSEARCH_DASHBOARD" | sudo tee -a /etc/hosts
+echo "$IP    $OTEL_STORE" | sudo tee -a /etc/hosts
+echo "$IP    $PROMETHEUS" | sudo tee -a /etc/hosts
+echo "$IP    $OTEL_LOADER" | sudo tee -a /etc/hosts
+
+```

src/integrations/src/install.py

@@ -96,6 +96,25 @@ def create_mapping_templates(client):
                 except OpenSearchException as e:
                     logger.error(f'Failed to create index template: {template_name}, error: {str(e)}')
 
+# create data streams
+def create_data_streams():
+    with open('../indices/data-stream.json', 'r') as f:
+        data = json.load(f)
+        data_streams = data.get('data-stream', [])
+
+        for ds in data_streams:
+            try:
+                response = requests.put(
+                    url=f'https://{opensearch_host}:9200/_data_stream/{ds}',
+                    auth=HTTPBasicAuth('admin', 'admin'),
+                    headers={'Content-Type': 'application/json'},
+                    verify=False  # Disable SSL verification
+                )
+                response.raise_for_status()  # Raise an exception if the request failed
+                logger.info(f'Successfully created data stream: {ds}')
+            except requests.HTTPError as e:
+                logger.error(f'Failed to create data stream: {ds}, error: {str(e)}')
+
 # load dashboards in the display folder
 def load_dashboards():
     dashboard_dir = '../display/'
@@ -122,23 +141,26 @@ def load_dashboards():
                     logger.error(f'Failed to load dashboard: {dashboard_name}, error: {str(e)}')
 
 # create the data_streams based on the list given in the data-stream.json file
-def create_data_streams():
-    with open('../indices/data-stream.json', 'r') as f:
-        data = json.load(f)
-        data_streams = data.get('data-stream', [])
-
-        for ds in data_streams:
-            try:
-                response = requests.post(
-                    url=f'https://{opensearch_host}:9200/_query/_datasources',
-                    auth=HTTPBasicAuth('admin', 'admin'),
-                    headers={'Content-Type': 'application/json'},
-                    verify=False  # Disable SSL verification
-                )
-                response.raise_for_status()  # Raise an exception if the request failed
-                logger.info(f'Successfully created data stream: {ds}')
-            except requests.HTTPError as e:
-                logger.error(f'Failed to create data stream: {ds}, error: {str(e)}')
+def create_datasources():
+    datasource_dir = '../datasource/'
+    for filename in os.listdir(datasource_dir):
+        if filename.endswith('.json'):
+            with open(os.path.join(datasource_dir, filename), 'r') as f:
+                datasource = json.load(f)
+                try:
+                    response = requests.post(
+                        url=f'https://{opensearch_host}:9200/_plugins/_query/_datasources',
+                        auth=HTTPBasicAuth('admin', 'admin'),
+                        json=datasource,
+                        verify=False,  # Disable SSL verification
+                        headers={'Content-Type': 'application/json'}
+                    )
+                    response.raise_for_status()  # Raise an exception if the request failed
+                    logger.info(f'Successfully created datasource: {filename}')
+                    print(f'Successfully created datasource: {filename}')
+                except requests.HTTPError as e:
+                    print(f'Failed to create datasource: {filename}, error: {str(e)}')
+                    logger.error(f'Failed to create datasource: {filename}, error: {str(e)}')
 
 
 if __name__ == '__main__':
@@ -148,4 +170,4 @@ def create_data_streams():
     create_mapping_templates(client)
     create_data_streams()
     load_dashboards()
-    create_data_streams()
+    create_datasources()