diff --git a/.gitignore b/.gitignore index 64b077a..3564e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build/elasticsearch/Dockerfile build/elasticsearch/Dockerfile-* build/elasticsearch/elasticsearch*.tar.gz tests/docker-compose*.yml +/snapshots diff --git a/Makefile b/Makefile index b34dd27..5d03fc8 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ all: build test # Test specified versions without building test: lint docker-compose + docker run --rm -v "$(PWD):/mnt" bash rm -rf /mnt/tests/datadir1 /mnt/tests/datadir2 $(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -w 160 -f puffy "test: $(FLAVOR) single"; \ ./bin/pytest --image-flavor=$(FLAVOR) --single-node tests; \ @@ -82,10 +83,7 @@ run-cluster: build docker-compose build: clean dockerfile $(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -f puffy -w 160 "Building: $(FLAVOR)"; \ - docker build -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch; \ - if [[ $(FLAVOR) == $(DEFAULT_IMAGE_FLAVOR) ]]; then \ - docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG); \ - fi; \ + docker buildx build --platform=linux/amd64,linux/arm64 -o type=image,name=$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG),push=true -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch; \ ) @@ -98,19 +96,42 @@ release-manager-release: clean # Build from artifacts on the local filesystem, using an http server (running # in a container) to provide the artifacts to the Dockerfile. build-from-local-artifacts: venv dockerfile docker-compose + -docker network create elasticsearch-docker-build docker run --rm -d --name=elasticsearch-docker-artifact-server \ - --network=host -v $(ARTIFACTS_DIR):/mnt \ + --network=elasticsearch-docker-build -v $(ARTIFACTS_DIR):/mnt \ python:3 bash -c 'cd /mnt && python3 -m http.server' - timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done' + docker run --rm --network=elasticsearch-docker-build centos:7 \ + timeout 120 bash -c 'until curl -s elasticsearch-docker-artifact-server:8000 > /dev/null; do sleep 1; done' -$(foreach FLAVOR, $(IMAGE_FLAVORS), \ pyfiglet -f puffy -w 160 "Building: $(FLAVOR)"; \ - docker build --network=host -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch || \ + docker build --network=elasticsearch-docker-build -t $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) -f build/elasticsearch/Dockerfile-$(FLAVOR) build/elasticsearch || \ (docker kill elasticsearch-docker-artifact-server; false); \ if [[ $(FLAVOR) == $(DEFAULT_IMAGE_FLAVOR) ]]; then \ docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG); \ fi; \ ) docker kill elasticsearch-docker-artifact-server + -docker network rm elasticsearch-docker-build + +# Build images from the latest snapshots on snapshots.elastic.co +from-snapshot: + rm -rf snapshots + + mkdir -p snapshots/elasticsearch/distribution/tar/build/distributions + (cd snapshots/elasticsearch/distribution/tar/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch/elasticsearch-$(ELASTIC_VERSION)-SNAPSHOT.tar.gz) + + mkdir -p snapshots/x-pack-elasticsearch/plugin/build/distributions + (cd snapshots/x-pack-elasticsearch/plugin/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch-plugins/x-pack/x-pack-$(ELASTIC_VERSION)-SNAPSHOT.zip) + + for plugin in ingest-user-agent ingest-geoip; do \ + mkdir -p snapshots/elasticsearch/plugins/$$plugin/build/distributions; \ + (cd snapshots/elasticsearch/plugins/$$plugin/build/distributions && \ + wget https://snapshots.elastic.co/downloads/elasticsearch-plugins/$$plugin/$$plugin-$(ELASTIC_VERSION)-SNAPSHOT.zip); \ + done + + ARTIFACTS_DIR=$$PWD/snapshots make release-manager-snapshot # Push the images to the dedicated push endpoint at "push.docker.elastic.co" push: test diff --git a/README.md b/README.md index 791c279..fd903a8 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,22 @@ A full build and test requires: - Docker - GNU Make - Python 3.5 with Virtualenv +- install all requirements in requirements.txt with pip3 + +## Running a build +To build an image with the latest nightly snapshot of Elasticsearch, run: +``` +make from-snapshot +``` + +To build and push an image with a released version of Elasticsearch, check out the corresponding +branch for the version, and run Make while specifying the exact version desired. +Like this: +``` +(git checkout 6.4.2-M1) +ELASTIC_VERSION=6.4.2 IMAGE_FLAVORS=platinum ELASTIC_REGISTRY=dckr.skryv.com make build +``` +Newer build scripts do not work. This build script download the correct artifacts. ## Contributing, issues and testing diff --git a/build/elasticsearch/bin/docker-entrypoint.sh b/build/elasticsearch/bin/docker-entrypoint.sh index e0133a8..3288d5d 100755 --- a/build/elasticsearch/bin/docker-entrypoint.sh +++ b/build/elasticsearch/bin/docker-entrypoint.sh @@ -51,7 +51,7 @@ declare -a es_opts while IFS='=' read -r envvar_key envvar_value do # Elasticsearch env vars need to have at least two dot separated lowercase words, e.g. `cluster.name` - if [[ "$envvar_key" =~ ^[a-z_]+\.[a-z_]+ ]]; then + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ ]]; then if [[ ! -z $envvar_value ]]; then es_opt="-E${envvar_key}=${envvar_value}" es_opts+=("${es_opt}") @@ -80,8 +80,8 @@ if bin/elasticsearch-plugin list -s | grep -q x-pack; then # node at this step, we can't enforce the presence of this env # var. if [[ -n "$ELASTIC_PASSWORD" ]]; then - [[ -f config/elasticsearch.keystore ]] || run_as_other_user_if_needed "bin/elasticsearch-keystore" "create" - run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | bin/elasticsearch-keystore add -x 'bootstrap.password' + [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create) + (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password') fi fi diff --git a/build/elasticsearch/elasticsearch.yml b/build/elasticsearch/elasticsearch.yml index b950fe5..8645fa5 100644 --- a/build/elasticsearch/elasticsearch.yml +++ b/build/elasticsearch/elasticsearch.yml @@ -5,3 +5,5 @@ network.host: 0.0.0.0 # set to 1 to allow single node clusters # Details: https://github.com/elastic/elasticsearch/pull/17288 discovery.zen.minimum_master_nodes: 1 +xpack.security.enabled: false +xpack.ml.enabled: false diff --git a/templates/Dockerfile.j2 b/templates/Dockerfile.j2 index fe35ea3..5cf3857 100644 --- a/templates/Dockerfile.j2 +++ b/templates/Dockerfile.j2 @@ -9,9 +9,9 @@ {% set tarball = 'elasticsearch-%s.tar.gz' % elastic_version -%} {% if artifacts_dir -%} -{% set url_root = 'http://localhost:8000/elasticsearch/distribution/tar/build/distributions' -%} -{% set x_pack_url_root = 'http://localhost:8000/x-pack-elasticsearch/plugin/build/distributions' -%} -{% set plugin_url_root = 'http://localhost:8000/elasticsearch/plugins' -%} +{% set url_root = 'http://elasticsearch-docker-artifact-server:8000/elasticsearch/distribution/tar/build/distributions' -%} +{% set x_pack_url_root = 'http://elasticsearch-docker-artifact-server:8000/x-pack-elasticsearch/plugin/build/distributions' -%} +{% set plugin_url_root = 'http://elasticsearch-docker-artifact-server:8000/elasticsearch/plugins' -%} {% elif staging_build_num -%} {% set version_tag = version_tag + '-' + staging_build_num -%} {% set url_root = 'https://staging.elastic.co/%s/downloads/elasticsearch' % version_tag -%} diff --git a/tests/fixtures.py b/tests/fixtures.py index 9e88e2a..4abf131 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,7 +9,7 @@ import os retry_settings = { - 'stop_max_delay': 30000, + 'stop_max_delay': 60000, 'wait_exponential_multiplier': 100, 'wait_exponential_max': 10000 } diff --git a/version.json b/version.json index a2add87..7257386 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version": "7.0.0-alpha1"} +{"version": "6.2.5"}