From 3babcc79b41f214635b06b63c7ab4f39f27acdd9 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Thu, 14 Dec 2017 13:42:04 +1100 Subject: [PATCH 01/18] Set version to 6.2.0 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index a2add87..79a31e3 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "7.0.0-alpha1"} +{"version": "6.2.0"} From 5e4f362a98042b061ddc740a246bf5de608d2e1a Mon Sep 17 00:00:00 2001 From: Louis-David Perron Date: Thu, 25 Jan 2018 12:48:10 -0500 Subject: [PATCH 02/18] Added support for alphanumeric configuration parameters --- build/elasticsearch/bin/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/elasticsearch/bin/docker-entrypoint.sh b/build/elasticsearch/bin/docker-entrypoint.sh index e0133a8..d455328 100755 --- a/build/elasticsearch/bin/docker-entrypoint.sh +++ b/build/elasticsearch/bin/docker-entrypoint.sh @@ -51,7 +51,7 @@ declare -a es_opts while IFS='=' read -r envvar_key envvar_value do # Elasticsearch env vars need to have at least two dot separated lowercase words, e.g. `cluster.name` - if [[ "$envvar_key" =~ ^[a-z_]+\.[a-z_]+ ]]; then + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ ]]; then if [[ ! -z $envvar_value ]]; then es_opt="-E${envvar_key}=${envvar_value}" es_opts+=("${es_opt}") From 12b07eebba3e095feb6bcd9f532a5c7bde58b463 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Tue, 6 Feb 2018 23:59:33 +0100 Subject: [PATCH 03/18] Bump version to 6.2.1 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 79a31e3..215f613 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "6.2.0"} +{"version": "6.2.1"} From 4d9786cde8b708683ba0ae577f19a210879094cf Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Thu, 8 Feb 2018 23:29:20 +0100 Subject: [PATCH 04/18] Bump version to 6.2.2 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 215f613..f48bc46 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "6.2.1"} +{"version": "6.2.2"} From a7114d4260b22ec679bfe561bd6715a8714165bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20Gr=C3=A9au?= Date: Tue, 20 Feb 2018 18:18:13 +0100 Subject: [PATCH 05/18] Bump version to 6.2.3 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index f48bc46..5154dea 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "6.2.2"} +{"version": "6.2.3"} From db6a458b9d57fdfadc480e63cd0fcdcbad593395 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Wed, 31 Jan 2018 05:13:50 +0100 Subject: [PATCH 06/18] Add "from-snapshot" Make target --- .gitignore | 1 + Makefile | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.gitignore b/.gitignore index 64b077a..3564e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build/elasticsearch/Dockerfile build/elasticsearch/Dockerfile-* build/elasticsearch/elasticsearch*.tar.gz tests/docker-compose*.yml +/snapshots diff --git a/Makefile b/Makefile index b34dd27..e7af2a2 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,26 @@ build-from-local-artifacts: venv dockerfile docker-compose ) docker kill elasticsearch-docker-artifact-server +# Build images from the latest snapshots on snapshots.elastic.co +from-snapshot: + rm -rf snapshots + + mkdir -p snapshots/elasticsearch/distribution/tar/build/distributions + (cd snapshots/elasticsearch/distribution/tar/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-$(ELASTIC_VERSION)-SNAPSHOT.tar.gz) + + mkdir -p snapshots/x-pack-elasticsearch/plugin/build/distributions + (cd snapshots/x-pack-elasticsearch/plugin/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch-plugins/x-pack/x-pack-$(ELASTIC_VERSION)-SNAPSHOT.zip) + + for plugin in ingest-user-agent ingest-geoip; do \ + mkdir -p snapshots/elasticsearch/plugins/$$plugin/build/distributions; \ + (cd snapshots/elasticsearch/plugins/$$plugin/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch-plugins/$$plugin/$$plugin-$(ELASTIC_VERSION)-SNAPSHOT.zip); \ + done + + ARTIFACTS_DIR=$$PWD/snapshots make release-manager-snapshot + # Push the images to the dedicated push endpoint at "push.docker.elastic.co" push: test $(foreach FLAVOR, $(IMAGE_FLAVORS), \ From ab1fcb47869a995b0017b663a38a6d93559d3f50 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Wed, 14 Mar 2018 05:09:21 +0100 Subject: [PATCH 07/18] Continue entrypoint after creating keystore --- build/elasticsearch/bin/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/elasticsearch/bin/docker-entrypoint.sh b/build/elasticsearch/bin/docker-entrypoint.sh index d455328..3288d5d 100755 --- a/build/elasticsearch/bin/docker-entrypoint.sh +++ b/build/elasticsearch/bin/docker-entrypoint.sh @@ -80,8 +80,8 @@ if bin/elasticsearch-plugin list -s | grep -q x-pack; then # node at this step, we can't enforce the presence of this env # var. if [[ -n "$ELASTIC_PASSWORD" ]]; then - [[ -f config/elasticsearch.keystore ]] || run_as_other_user_if_needed "bin/elasticsearch-keystore" "create" - run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | bin/elasticsearch-keystore add -x 'bootstrap.password' + [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create) + (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password') fi fi From d645be2a82184d049e65ddf688f7b85022d06e27 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Wed, 21 Mar 2018 07:58:48 +1100 Subject: [PATCH 08/18] Bump to version 6.2.4 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index 5154dea..bf53319 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "6.2.3"} +{"version": "6.2.4"} From 9a1f8fdf36975437e1a6c2f3c0b36afc32c83d8b Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 18 Apr 2018 14:22:45 +0100 Subject: [PATCH 09/18] Bump version to 6.2.5 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index bf53319..7257386 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "6.2.4"} +{"version": "6.2.5"} From 543db259f06cec5b21f2964bb20fa470e65a719b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 15 May 2018 11:19:44 -0400 Subject: [PATCH 10/18] Do not use host network for building image Today the Elasticsearch Docker build relies on the host network for building the Docker image. In particular, it requires port 8000 on the host to be available so that the Docker build can connect to the artifacts server that is created during an earlier phase of the build. I ran into a strange issue where the build would repeatedly fail if using the host network. In particular, the issue that I ran into is that the setup package installed during yum update could not be installed because /etc/hosts could not be extracted from the package. It is not clear to me why this fails with the host network, but once we surmised this was a possbility and switched to not use the host network, the issue went anyway. Whatever the issue is, it's inherently desirable to not use the host network during the build anyway. Instead, we can connect the containers over a dedicated bridge network and not rely on port 8000 being available on the host. Fortunately, this fixes the issue that I was running into as well. Co-authored-by: Dimitrios Liappis --- Makefile | 9 ++++++--- templates/Dockerfile.j2 | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index e7af2a2..00300c3 100644 --- a/Makefile +++ b/Makefile @@ -98,19 +98,22 @@ release-manager-release: clean # Build from artifacts on the local filesystem, using an http server (running # in a container) to provide the artifacts to the Dockerfile. build-from-local-artifacts: venv dockerfile docker-compose + -docker network create elasticsearch-docker-build docker run --rm -d --name=elasticsearch-docker-artifact-server \ - --network=host -v $(ARTIFACTS_DIR):/mnt \ + --network=elasticsearch-docker-build -v $(ARTIFACTS_DIR):/mnt \ python:3 bash -c 'cd /mnt && python3 -m http.server' - timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done' + docker run --rm --network=elasticsearch-docker-build centos:7 \ + timeout 120 bash -c 'until curl -s elasticsearch-docker-artifact-server:8000 > /dev/null; do sleep 1; done' -$(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -f puffy -w 160 "Building: $(FLAVOR)"; \ - docker build --network=host -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch || \ + docker build --network=elasticsearch-docker-build -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch || \ (docker kill elasticsearch-docker-artifact-server; false); \ if [[ $(FLAVOR) == $(DEFAULT_IMAGE_FLAVOR) ]]; then \ docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG); \ fi; \ ) docker kill elasticsearch-docker-artifact-server + -docker network rm elasticsearch-docker-build # Build images from the latest snapshots on snapshots.elastic.co from-snapshot: diff --git a/templates/Dockerfile.j2 b/templates/Dockerfile.j2 index fe35ea3..5cf3857 100644 --- a/templates/Dockerfile.j2 +++ b/templates/Dockerfile.j2 @@ -9,9 +9,9 @@ {% set tarball = 'elasticsearch-%s.tar.gz' % elastic_version -%} {% if artifacts_dir -%} -{% set url_root = 'http://localhost:8000/elasticsearch/distribution/tar/build/distributions' -%} -{% set x_pack_url_root = 'http://localhost:8000/x-pack-elasticsearch/plugin/build/distributions' -%} -{% set plugin_url_root = 'http://localhost:8000/elasticsearch/plugins' -%} +{% set url_root = 'http://elasticsearch-docker-artifact-server:8000/elasticsearch/distribution/tar/build/distributions' -%} +{% set x_pack_url_root = 'http://elasticsearch-docker-artifact-server:8000/x-pack-elasticsearch/plugin/build/distributions' -%} +{% set plugin_url_root = 'http://elasticsearch-docker-artifact-server:8000/elasticsearch/plugins' -%} {% elif staging_build_num -%} {% set version_tag = version_tag + '-' + staging_build_num -%} {% set url_root = 'https://staging.elastic.co/%s/downloads/elasticsearch' % version_tag -%} From aa45fa157635b8bf96cf452bfc81d7c8faf01d65 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Fri, 25 May 2018 09:56:22 +1000 Subject: [PATCH 11/18] Clean datadirs before testing --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 00300c3..60d0ae6 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ all: build test # Test specified versions without building test: lint docker-compose + docker run --rm -v "$(PWD):/mnt" bash rm -rf /mnt/tests/datadir1 /mnt/tests/datadir2 $(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -w 160 -f puffy "test: $(FLAVOR) single"; \ ./bin/pytest --image-flavor=$(FLAVOR) --single-node tests; \ From 19609537a2bf9b6e617aad3cce21f3c5d7257681 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Fri, 25 May 2018 09:59:07 +1000 Subject: [PATCH 12/18] Double timeout for tests --- tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 9e88e2a..4abf131 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,7 +9,7 @@ import os retry_settings = { - 'stop_max_delay': 30000, + 'stop_max_delay': 60000, 'wait_exponential_multiplier': 100, 'wait_exponential_max': 10000 } From e6d9e03de18bd0c7cf797e5981ea73fc05452e61 Mon Sep 17 00:00:00 2001 From: Toby McLaughlin Date: Mon, 4 Jun 2018 11:43:11 +1000 Subject: [PATCH 13/18] Add build instructions to README #163 --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 791c279..6cf4595 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,20 @@ A full build and test requires: - GNU Make - Python 3.5 with Virtualenv +## Running a build +To build an image with the latest nightly snapshot of Elasticsearch, run: +``` +make from-snapshot +``` + +To build an image with a released version of Elasticsearch, check out the corresponding +branch for the version, and run Make while specifying the exact version desired. +Like this: +``` +git checkout 6.2 +ELASTIC_VERSION=6.2.4 make +``` + ## Contributing, issues and testing Acceptance tests for the image are located in the test directory, and can be invoked with `make test`. From d2a4500ccf0c3384134f3530f2eea1cd046e9c6f Mon Sep 17 00:00:00 2001 From: Thomas Soens Date: Thu, 24 Mar 2022 09:26:03 +0100 Subject: [PATCH 14/18] Disable xpack ml & security --- build/elasticsearch/elasticsearch.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/elasticsearch/elasticsearch.yml b/build/elasticsearch/elasticsearch.yml index b950fe5..8645fa5 100644 --- a/build/elasticsearch/elasticsearch.yml +++ b/build/elasticsearch/elasticsearch.yml @@ -5,3 +5,5 @@ network.host: 0.0.0.0 # set to 1 to allow single node clusters # Details: https://github.com/elastic/elasticsearch/pull/17288 discovery.zen.minimum_master_nodes: 1 +xpack.security.enabled: false +xpack.ml.enabled: false From 32b866907fb9c236c60bbcf5582b79d7ed412b6e Mon Sep 17 00:00:00 2001 From: Thomas Soens Date: Thu, 24 Mar 2022 09:31:18 +0100 Subject: [PATCH 15/18] Add install of requirements in readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6cf4595..3a9bb03 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ A full build and test requires: - Docker - GNU Make - Python 3.5 with Virtualenv +- install all requirements in requirements.txt with pip3 ## Running a build To build an image with the latest nightly snapshot of Elasticsearch, run: From 1fbbb5e49ca51f1a429def394df90b523cd72bdd Mon Sep 17 00:00:00 2001 From: Thomas Soens Date: Mon, 25 Apr 2022 15:29:22 +0200 Subject: [PATCH 16/18] =?UTF-8?q?=C3=81dd=20instructions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3a9bb03..68dfdb9 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,11 @@ To build an image with a released version of Elasticsearch, check out the corres branch for the version, and run Make while specifying the exact version desired. Like this: ``` -git checkout 6.2 -ELASTIC_VERSION=6.2.4 make +(git checkout 6.4.2-A1) +ELASTIC_VERSION=6.4.2 make build +docker tag docker.elastic.co/elasticsearch/elasticsearch-platinum:6.4.2 dckr.skryv.com/elastic-m1:6.4.2 ``` +Newer build scripts do not work. This build script download the correct artifacts. ## Contributing, issues and testing From d6fc9e6d5b7d6a798b4074827226a18217a0254c Mon Sep 17 00:00:00 2001 From: Thomas Soens Date: Thu, 28 Apr 2022 11:23:56 +0200 Subject: [PATCH 17/18] multiplatform build and push --- Makefile | 5 +---- README.md | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 60d0ae6..5d03fc8 100644 --- a/Makefile +++ b/Makefile @@ -83,10 +83,7 @@ run-cluster: build docker-compose build: clean dockerfile $(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -f puffy -w 160 "Building: $(FLAVOR)"; \ - docker build -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch; \ - if [[ $(FLAVOR) == $(DEFAULT_IMAGE_FLAVOR) ]]; then \ - docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG); \ - fi; \ + docker buildx build --platform=linux/amd64,linux/arm64 -o type=image,name=$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG),push=true -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch; \ ) diff --git a/README.md b/README.md index 68dfdb9..1c4a365 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,12 @@ To build an image with the latest nightly snapshot of Elasticsearch, run: make from-snapshot ``` -To build an image with a released version of Elasticsearch, check out the corresponding +To build and push an image with a released version of Elasticsearch, check out the corresponding branch for the version, and run Make while specifying the exact version desired. Like this: ``` (git checkout 6.4.2-A1) -ELASTIC_VERSION=6.4.2 make build -docker tag docker.elastic.co/elasticsearch/elasticsearch-platinum:6.4.2 dckr.skryv.com/elastic-m1:6.4.2 +ELASTIC_VERSION=6.4.2 IMAGE_FLAVORS=platinum ELASTIC_REGISTRY=dckr.skryv.com make build ``` Newer build scripts do not work. This build script download the correct artifacts. From 717f47dbb692dfc634c2d025665afc2290da8104 Mon Sep 17 00:00:00 2001 From: Thomas Soens Date: Wed, 17 Aug 2022 09:29:37 +0200 Subject: [PATCH 18/18] Correct typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c4a365..fd903a8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ To build and push an image with a released version of Elasticsearch, check out t branch for the version, and run Make while specifying the exact version desired. Like this: ``` -(git checkout 6.4.2-A1) +(git checkout 6.4.2-M1) ELASTIC_VERSION=6.4.2 IMAGE_FLAVORS=platinum ELASTIC_REGISTRY=dckr.skryv.com make build ``` Newer build scripts do not work. This build script download the correct artifacts.