From a55e4bd1f72723e84680bc8371e276ff59a12c97 Mon Sep 17 00:00:00 2001 From: Fred Smith Date: Mon, 29 Mar 2021 20:52:44 -0400 Subject: [PATCH 001/664] add enterprise catalog to sandboxes --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 10a5863d7a7..7ea70c3bb74 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -509,7 +509,7 @@ EOF fi declare -A deploy -plays="prospectus edxapp forum ecommerce credentials discovery analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration xqueue certs demo testcourses registrar program_console learner_portal" +plays="prospectus edxapp forum ecommerce credentials discovery enterprise_catalog analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration xqueue certs demo testcourses registrar program_console learner_portal" for play in $plays; do deploy[$play]=${!play} From 090927f5b3649957f9eafe7efb622fa36673b329 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Thu, 8 Apr 2021 11:41:14 -0400 Subject: [PATCH 002/664] SRE-911 | Update ansible EC2 tagging to happen separately from instance creation (#6351) --- playbooks/continuous_delivery/launch_instance.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 0c6347b0f8d..66f2e0cbcda 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -65,7 +65,6 @@ - name: Launch EC2 instance ec2: - instance_tags: {"Name" : "{{ automation_prefix }} {{ unique_key_name.stdout }}"} region: "{{ ec2_region }}" key_name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" instance_type: "{{ ec2_instance_type }}" @@ -83,6 +82,16 @@ instance_profile_name: "{{ ec2_instance_profile_name }}" register: ec2_instance_register + - name: Tag EC2 instance + ec2_tag: + region: "{{ ec2_region }}" + resource: "{{ ec2_instance_register.instances[0].instance_id }}" + state: present + tags: + Name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" + retries: 6 + delay: 15 + - name: Wait for SSH to come up wait_for: host: "{{ ec2_instance_register.instances[0].private_ip }}" From 6f696e27fb7d0cb177cebcf4f2f41e2147fdb204 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Thu, 8 Apr 2021 12:10:29 -0400 Subject: [PATCH 003/664] Revert "SRE-911 | Update ansible EC2 tagging to happen separately from instance creation (#6351)" (#6353) This reverts commit 090927f5b3649957f9eafe7efb622fa36673b329. --- playbooks/continuous_delivery/launch_instance.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 66f2e0cbcda..0c6347b0f8d 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -65,6 +65,7 @@ - name: Launch EC2 instance ec2: + instance_tags: {"Name" : "{{ automation_prefix }} {{ unique_key_name.stdout }}"} region: "{{ ec2_region }}" key_name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" instance_type: "{{ ec2_instance_type }}" @@ -82,16 +83,6 @@ instance_profile_name: "{{ ec2_instance_profile_name }}" register: ec2_instance_register - - name: Tag EC2 instance - ec2_tag: - region: "{{ ec2_region }}" - resource: "{{ ec2_instance_register.instances[0].instance_id }}" - state: present - tags: - Name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" - retries: 6 - delay: 15 - - name: Wait for SSH to come up wait_for: host: "{{ ec2_instance_register.instances[0].private_ip }}" From f72fd983ace1e831851c0b21a26307dde7cbdde9 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Thu, 8 Apr 2021 14:12:09 -0400 Subject: [PATCH 004/664] ISRE-911 | Update ansible EC2 tagging to happen separately from instance creation (take-2) (#6354) --- playbooks/continuous_delivery/launch_instance.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 0c6347b0f8d..a53e096728b 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -65,7 +65,6 @@ - name: Launch EC2 instance ec2: - instance_tags: {"Name" : "{{ automation_prefix }} {{ unique_key_name.stdout }}"} region: "{{ ec2_region }}" key_name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" instance_type: "{{ ec2_instance_type }}" @@ -83,6 +82,16 @@ instance_profile_name: "{{ ec2_instance_profile_name }}" register: ec2_instance_register + - name: Tag EC2 instance + ec2_tag: + region: "{{ ec2_region }}" + resource: "{{ ec2_instance_register.instances[0].id }}" + state: present + tags: + Name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" + retries: 6 + delay: 15 + - name: Wait for SSH to come up wait_for: host: "{{ ec2_instance_register.instances[0].private_ip }}" From 65ed22bc21fa829837c0fff39d6f4bfbadab952d Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Wed, 7 Apr 2021 16:04:40 -0300 Subject: [PATCH 005/664] docs: remove venv activation instructions If one runs the Native Installation instructions, which includes running ansible-bootstrap.sh, then activates the bootstrapped ansible virtualenv, then proceeds to run native.sh, this will result in a clobbered python3-pkg-resources package, necessitating its reinstallation. Thus, we remove these instructions from the output of ansible-bootstrap.sh. (cherry picked from commit 3cc14096239e03b9a2ccc4a1f4232d71892c41a7) --- CHANGELOG.md | 9 +++++++-- util/install/ansible-bootstrap.sh | 3 --- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57932b97114..2be1259159f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ # Changelog + All notable changes to this project will be documented in this file. -Add any new changes to the top(right below this line). +Add any new changes to the top (right below this line). + + - 2021-03-08 + - Remove instruction from ansile-bootstrap.sh that instructed people to activate + the virtualenv. This was incorrect for community installations. - - 2021-07-03 + - 2021-03-07 - Role: ecommerce - Added new configuration variable ECOMMERCE_EXTRA_CONFIG_OVERRIDES, which will allow override any ecommerce settings. diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 44890486426..a4c6bb40eb0 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -176,9 +176,6 @@ if [[ "true" == "${RUN_ANSIBLE}" ]]; then ****************************************************************************** Done bootstrapping, edx_ansible is now installed in /edx/app/edx_ansible. - Time to run some plays. Activate the virtual env with - - > . /edx/app/edx_ansible/venvs/edx_ansible/bin/activate ****************************************************************************** EOF From 691b9321fe2cb12dbce9ed93e4d61dba00c14045 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 9 Apr 2021 11:21:36 +0500 Subject: [PATCH 006/664] Update jenkins war and plugins --- playbooks/roles/jenkins_build/defaults/main.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 9297224ceab..3a1c9bc362b 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.263.4 +BUILD_JENKINS_VERSION: jenkins_2.277.2 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: @@ -43,7 +43,7 @@ build_jenkins_plugins_list: version: '2.1' group: 'org.jenkins-ci.plugins' - name: 'aws-credentials' - version: '1.24' + version: '1.28.1' group: 'org.jenkins-ci.plugins' - name: 'aws-java-sdk' version: '1.11.700' @@ -172,7 +172,7 @@ build_jenkins_plugins_list: version: '2.13' group: 'org.jenkins-ci.plugins' - name: 'matrix-auth' - version: '2.6.2' + version: '2.6.6' group: 'org.jenkins-ci.plugins' - name: 'matrix-project' version: '1.18' @@ -208,7 +208,7 @@ build_jenkins_plugins_list: version: '2.9' group: 'org.jenkins-ci.plugins' - name: 'promoted-builds' - version: '3.5' + version: '3.9.1' group: 'org.jenkins-ci.plugins' - name: 'pyenv-pipeline' version: '2.0.1' @@ -267,6 +267,9 @@ build_jenkins_plugins_list: - name: 'translation' version: '1.16' group: 'org.jenkins-ci.plugins' + - name: 'variant' + version: '1.4' + group: 'org.jenkins-ci.plugins' - name: 'violations' version: '0.7.11' group: 'org.jenkins-ci.plugins' From bf8cb3c8dad55026a961df3ed33dba4f16dd9b4d Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 9 Apr 2021 16:14:55 +0500 Subject: [PATCH 007/664] Downgrade jenkins --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 3a1c9bc362b..897d59edbd0 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.277.2 +BUILD_JENKINS_VERSION: jenkins_2.263.4 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: From 89c692281903ccceb962edad9be06f06a4517b36 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Fri, 9 Apr 2021 17:59:31 -0400 Subject: [PATCH 008/664] fix: Update syntax check script This removes all references to Travis since we now run this on a Github Action. --- tests/test_playbooks.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_playbooks.sh b/tests/test_playbooks.sh index 42eddbde4f9..52e4b3abafc 100755 --- a/tests/test_playbooks.sh +++ b/tests/test_playbooks.sh @@ -3,20 +3,20 @@ set -e ROOT_DIR=$PWD cd playbooks ROLE_DIRS=$(/bin/ls -d roles/*) -cat <travis-test.yml +cat <syntax-check-test.yml - name: Play to test all roles hosts: all roles: EOF for role_dir in $ROLE_DIRS; do - echo " - $(basename $role_dir)" >> travis-test.yml + echo " - $(basename $role_dir)" >> syntax-check-test.yml done -ansible-playbook -i localhost, --syntax-check travis-test.yml +ansible-playbook -i localhost, --syntax-check syntax-check-test.yml output_dir="$PWD/test_output/env-dep" mkdir -p $output_dir -ansible-playbook -i localhost, -c local --tags "common_directories,edxapp_cfg" edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir -e EDXAPP_EDXAPP_SECRET_KEY='DUMMY TRAVIS KEY' +ansible-playbook -i localhost, -c local --tags "common_directories,edxapp_cfg" edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir -e EDXAPP_EDXAPP_SECRET_KEY='DUMMY KEY' root_dir=$output_dir environment_deployments="." From 702cb82d7f2dac767708aa8994bfb3ec954fb8c1 Mon Sep 17 00:00:00 2001 From: Jazib Humayun Date: Mon, 12 Apr 2021 18:44:24 +0500 Subject: [PATCH 009/664] feat: Install Docker on Data Engineering New Jenkins --- playbooks/jenkins_data_engineering_new.yml | 1 + .../jenkins_data_engineering_new/defaults/main.yml | 2 ++ .../jenkins_data_engineering_new/tasks/system.yml | 11 +++++++++++ 3 files changed, 14 insertions(+) create mode 100644 playbooks/roles/jenkins_data_engineering_new/tasks/system.yml diff --git a/playbooks/jenkins_data_engineering_new.yml b/playbooks/jenkins_data_engineering_new.yml index 29ddc68b4e9..7a4728686e4 100644 --- a/playbooks/jenkins_data_engineering_new.yml +++ b/playbooks/jenkins_data_engineering_new.yml @@ -33,3 +33,4 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE tags: - newreliconly + - docker-tools diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index c4894d1ebe5..36e5d73048c 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -6,6 +6,8 @@ jenkins_home: '/var/lib/jenkins' JENKINS_SERVER_NAME: 'jenkins-new.analytics.edx.org' jenkins_user: 'jenkins' jenkins_group: 'jenkins' +# the groups that the jenkins user must belong to +jenkins_groups: "jenkins,docker" jenkins_user_home: '/home/{{ jenkins_user }}' jenkins_port: 8080 jenkins_nginx_port: 80 diff --git a/playbooks/roles/jenkins_data_engineering_new/tasks/system.yml b/playbooks/roles/jenkins_data_engineering_new/tasks/system.yml new file mode 100644 index 00000000000..63b65bcf18c --- /dev/null +++ b/playbooks/roles/jenkins_data_engineering_new/tasks/system.yml @@ -0,0 +1,11 @@ +--- +- name: Create jenkins group + group: name={{ jenkins_group }} state=present + +# The Jenkins account needs a login shell because Jenkins uses scp +- name: Add the jenkins user to the group and configure shell + user: + name: '{{ jenkins_user }}' + groups: '{{ jenkins_groups }}' + append: yes + shell: /bin/bash \ No newline at end of file From 4669b7f52c7f315a82c48b90a901363ef6903f72 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 12 Apr 2021 15:49:46 -0400 Subject: [PATCH 010/664] build: mention lilac in the pull request template Remind people that their fixes might be needed on Lilac as well. --- .github/PULL_REQUEST_TEMPLATE.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 322bf24846b..6c7b9de1a19 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,14 @@ Configuration Pull Request --- + + Make sure that the following steps are done before merging: - [ ] A DevOps team member has approved the PR if it is code shared across multiple services and you don't own all of the services. From f62f8dffa6bdc5afe67545f792584ff65e04bfc7 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 13 Apr 2021 09:38:00 -0400 Subject: [PATCH 011/664] Cleanup celery scripts, moved to k8s PSRE-737 --- Makefile | 4 - requirements/celery.in | 8 - requirements/celery_progress.in | 10 - .../check_celery_progress.py | 531 ------------------ .../check_celery_progress/print_queue.py | 232 -------- .../check_celery_progress/requirements.txt | 84 --- .../test_check_celery_progress.py | 164 ------ .../create_celery_dashboard.py | 215 ------- .../update_celery_monitoring/requirements.txt | 21 - .../update_celery_monitoring.py | 292 ---------- 10 files changed, 1561 deletions(-) delete mode 100644 requirements/celery.in delete mode 100644 requirements/celery_progress.in delete mode 100644 util/jenkins/check_celery_progress/check_celery_progress.py delete mode 100644 util/jenkins/check_celery_progress/print_queue.py delete mode 100644 util/jenkins/check_celery_progress/requirements.txt delete mode 100644 util/jenkins/check_celery_progress/test_check_celery_progress.py delete mode 100644 util/jenkins/update_celery_monitoring/create_celery_dashboard.py delete mode 100644 util/jenkins/update_celery_monitoring/requirements.txt delete mode 100644 util/jenkins/update_celery_monitoring/update_celery_monitoring.py diff --git a/Makefile b/Makefile index 8c60d936305..3d89474c8be 100755 --- a/Makefile +++ b/Makefile @@ -29,8 +29,6 @@ upgrade: ## update the pip requirements files to use the latest releases satisfy pip-compile --upgrade -o requirements.txt requirements/base.in pip-compile --upgrade -o playbooks/roles/aws/templates/requirements.txt.j2 requirements/aws.in pip-compile --upgrade -o util/elasticsearch/requirements.txt requirements/elasticsearch.in - pip-compile --upgrade -o util/jenkins/update_celery_monitoring/requirements.txt requirements/celery.in - pip-compile --upgrade -o util/jenkins/check_celery_progress/requirements.txt requirements/celery_progress.in pip-compile --upgrade -o util/jenkins/requirements-cloudflare.txt requirements/cloudflare.in pip-compile --upgrade -o util/pingdom/requirements.txt requirements/pingdom.in pip-compile --upgrade -o util/vpc-tools/requirements.txt requirements/vpc-tools.in @@ -40,8 +38,6 @@ upgrade: ## update the pip requirements files to use the latest releases satisfy requirements.txt \ playbooks/roles/aws/templates/requirements.txt.j2 \ util/elasticsearch/requirements.txt \ - util/jenkins/update_celery_monitoring/requirements.txt \ - util/jenkins/check_celery_progress/requirements.txt \ util/jenkins/requirements-cloudflare.txt \ util/pingdom/requirements.txt \ util/vpc-tools/requirements.txt diff --git a/requirements/celery.in b/requirements/celery.in deleted file mode 100644 index 852051b55d1..00000000000 --- a/requirements/celery.in +++ /dev/null @@ -1,8 +0,0 @@ -# Requirements for the celery/redis monitoring script in util/jenkins - -awscli==1.14.32 -backoff==1.4.3 -boto3==1.5.4 -click==6.7 -futures ; python_version == "2.7" # via s3transfer -redis==3.5.3 diff --git a/requirements/celery_progress.in b/requirements/celery_progress.in deleted file mode 100644 index d8bbababd67..00000000000 --- a/requirements/celery_progress.in +++ /dev/null @@ -1,10 +0,0 @@ -# Requirements for the celery/redis monitoring script in util/jenkins - -awscli==1.14.32 -backoff==1.4.3 -boto3==1.5.4 -click==6.7 -futures ; python_version == "2.7" # via s3transfer -redis==3.5.3 -opsgenie-sdk==0.3.1 -celery==4.4.7 diff --git a/util/jenkins/check_celery_progress/check_celery_progress.py b/util/jenkins/check_celery_progress/check_celery_progress.py deleted file mode 100644 index 3a5cbefb0c2..00000000000 --- a/util/jenkins/check_celery_progress/check_celery_progress.py +++ /dev/null @@ -1,531 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import sys -import pickle -import json -import datetime -import base64 -import zlib -import redis -import click -import backoff -import boto3 -import botocore -import traceback -from itertools import zip_longest -from celery import Celery -from opsgenie.swagger_client import AlertApi -from opsgenie.swagger_client import configuration -from opsgenie.swagger_client.models import CreateAlertRequest, CloseAlertRequest -from opsgenie.swagger_client.rest import ApiException -from textwrap import dedent - - -MAX_TRIES = 5 -QUEUE_AGE_HASH_NAME = "queue_age_monitoring" -DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f' - - -class RedisWrapper(object): - def __init__(self, dev_test_mode=None, *args, **kwargs): - assert isinstance(dev_test_mode, bool) - self.dev_test_mode = dev_test_mode - self.redis = redis.StrictRedis(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def keys(self): - return list(self.redis.keys()) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def type(self, key): - return self.redis.type(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def llen(self, key): - return self.redis.llen(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def lindex(self, key, index): - return self.redis.lindex(key, index) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def hgetall(self, key): - return self.redis.hgetall(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def delete(self, key): - if self.dev_test_mode: - print(("Test Mode: would have run redis.delete({})".format(key))) - else: - return self.redis.delete(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def hset(self, *args): - if self.dev_test_mode: - print(("Test Mode: would have run redis.hset({})".format(args))) - else: - return self.redis.hset(*args) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def hmset(self, *args): - if self.dev_test_mode: - print(("Test Mode: would have run redis.hmset({})".format(args))) - else: - return self.redis.hmset(*args) - - -class CwBotoWrapper(object): - def __init__(self, dev_test_mode=None): - assert isinstance(dev_test_mode, bool) - self.dev_test_mode = dev_test_mode - self.client = boto3.client('cloudwatch') - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def put_metric_data(self, *args, **kwargs): - if self.dev_test_mode: - print(("Test Mode: would have run put_metric_data({},{})".format(args, kwargs))) - else: - return self.client.put_metric_data(*args, **kwargs) - - -def pretty_json(obj): - return json.dumps(obj, indent=4, sort_keys=True) - - -def pretty_state(state): - output = {} - for queue_name, queue_state in state.items(): - output[queue_name] = {} - for key, value in queue_state.items(): - if key == 'first_occurance_time': - value = str_from_datetime(value) - output[queue_name][key] = value - return pretty_json(output) - - -def datetime_from_str(string): - return datetime.datetime.strptime(string, DATE_FORMAT) - - -def str_from_datetime(dt): - return dt.strftime(DATE_FORMAT) - - -def unpack_state(packed_state): - decoded_state = {k.decode("utf-8"): v.decode("utf-8") for k, v in packed_state.items()} - unpacked_state = {} - - for key, value in decoded_state.items(): - decoded_value = json.loads(value) - unpacked_state[key] = { - 'correlation_id': decoded_value['correlation_id'], - 'first_occurance_time': datetime_from_str(decoded_value['first_occurance_time']), - 'alert_created': decoded_value['alert_created'], - } - - return unpacked_state - - -def pack_state(unpacked_state): - packed_state = {} - for queue_name, queue_state in unpacked_state.items(): - dt_str = str_from_datetime(queue_state['first_occurance_time']) - packed_state[queue_name] = json.dumps({ - 'correlation_id': queue_state['correlation_id'], - 'first_occurance_time': dt_str, - 'alert_created': queue_state['alert_created'], - }) - return packed_state - - -def build_new_state(old_state, queue_first_items, current_time): - new_state = {} - for queue_name, first_item in queue_first_items.items(): - # TODO: Handle keys missing in data - correlation_id = first_item['properties']['correlation_id'] - first_occurance_time = current_time - alert_created = False - if queue_name in old_state: - old_correlation_id = old_state[queue_name]['correlation_id'] - alert_created = old_state[queue_name]['alert_created'] - if old_correlation_id == correlation_id: - first_occurance_time = old_state[queue_name]['first_occurance_time'] - - new_state[queue_name] = { - 'correlation_id': correlation_id, - 'first_occurance_time': first_occurance_time, - 'alert_created': alert_created, - } - - return new_state - - -def generate_alert_message(environment, deploy, queue_name, threshold): - return str.format("{}-{} {} queue is stale. Stationary for over {}s", environment, deploy, queue_name, threshold) - - -def generate_alert_alias(environment, deploy, queue_name): - return str.format("{}-{} {} stale celery queue", environment, deploy, queue_name) - - -@backoff.on_exception(backoff.expo, - (ApiException), - max_tries=MAX_TRIES) -def create_alert(opsgenie_api_key, environment, deploy, queue_name, threshold, info, dev_test_mode=None): - assert isinstance(dev_test_mode, bool) - - configuration.api_key['Authorization'] = opsgenie_api_key - configuration.api_key_prefix['Authorization'] = 'GenieKey' - - alert_msg = generate_alert_message(environment, deploy, queue_name, threshold) - alias = generate_alert_alias(environment, deploy, queue_name) - - if dev_test_mode: - print(("Test Mode: would have created Alert: {}".format(alias))) - else: - print(("Creating Alert: {}".format(alias))) - response = AlertApi().create_alert(body=CreateAlertRequest(message=alert_msg, alias=alias, description=info)) - print(('request id: {}'.format(response.request_id))) - print(('took: {}'.format(response.took))) - print(('result: {}'.format(response.result))) - - -@backoff.on_exception(backoff.expo, - (ApiException), - max_tries=MAX_TRIES) -def close_alert(opsgenie_api_key, environment, deploy, queue_name, dev_test_mode=None): - assert isinstance(dev_test_mode, bool) - - configuration.api_key['Authorization'] = opsgenie_api_key - configuration.api_key_prefix['Authorization'] = 'GenieKey' - - alias = generate_alert_alias(environment, deploy, queue_name) - - if dev_test_mode: - print(("Test Mode: would have closed Alert: {}".format(alias))) - else: - print(("Closing Alert: {}".format(alias))) - # Need body=CloseAlertRequest(source="") otherwise OpsGenie API complains that body must be a json object - response = AlertApi().close_alert(identifier=alias, identifier_type='alias', body=CloseAlertRequest(source="")) - print(('request id: {}'.format(response.request_id))) - print(('took: {}'.format(response.took))) - print(('result: {}'.format(response.result))) - - -def extract_body(task): - body = base64.b64decode(task['body']) - body_dict = {} - - if ( - 'headers' in task and - 'compression' in task['headers'] and - task['headers']['compression'] == 'application/x-gzip' - ): - body = zlib.decompress(body) - - if task.get('content-type') == 'application/json': - body_dict = json.loads(body.decode("utf-8")) - elif task.get('content-type') == 'application/x-python-serialize': - body_dict = {k.decode("utf-8"): v for k, v in pickle.loads(body, encoding='bytes').items()} - return body_dict - - -def generate_info( - queue_name, - correlation_id, - body, - active_tasks, - do_alert, - first_occurance_time, - current_time, - next_task_age, - threshold, - default_threshold, - jenkins_build_url, -): - next_task = "Key missing" - args = "Key missing" - kwargs = "Key missing" - - if 'task' in body: - next_task = body['task'] - - if 'args' in body: - args = body['args'] - - if 'kwargs' in body: - kwargs = body['kwargs'] - - output = str.format( - dedent(""" - ============================================= - queue_name = {} - --------------------------------------------- - do_alert = {} - threshold = {} seconds - default_threshold = {} seconds - jenkins_build_url = {} - current_time = {} - --------------------------------------------- - Next Task: - first_occurance_time = {} - age = {} seconds - correlation_id = {} - task_name = {} - args = {} - kwargs = {} - --------------------------------------------- - active_tasks = {} - ============================================= - """), - queue_name, - do_alert, - threshold, - default_threshold, - jenkins_build_url, - current_time, - first_occurance_time, - next_task_age, - correlation_id, - next_task, - args, - kwargs, - active_tasks, - ) - return output - - -def celery_connection(host, port): - celery_client = " " - try: - broker_url = "redis://" + host + ":" + str(port) - celery_client = Celery(broker=broker_url) - except Exception as e: - print(("Exception in connection():", e)) - print(traceback.format_exc()) - return celery_client - - -# Functionality added to get list of currently running tasks -# because Redis returns only the next tasks in the list -def get_active_tasks(celery_control, queue_workers, queue_name): - active_tasks = dict() - redacted_active_tasks = dict() - if queue_name in queue_workers: - workers = queue_workers[queue_name] - try: - for worker, data in celery_control.inspect(workers).active().items(): - for task in data: - active_tasks.setdefault( - task["hostname"], []).append([ - 'task: {}'.format(task.get("name")), - 'args: {}'.format(task.get("args")), - 'kwargs: {}'.format(task.get("kwargs")), - ]) - redacted_active_tasks.setdefault( - task["hostname"], []).append([ - 'task: {}'.format(task.get("name")), - 'args: REDACTED', - 'kwargs: REDACTED', - ]) - except Exception as e: - print(("Exception in get_active_tasks():", e)) - print(traceback.format_exc()) - return (pretty_json(active_tasks), pretty_json(redacted_active_tasks)) - - -@click.command() -@click.option('--host', '-h', default='localhost', - help='Hostname of redis server', required=True) -@click.option('--port', '-p', default=6379, help='Port of redis server') -@click.option('--environment', '-e', required=True) -@click.option('--deploy', '-d', required=True, - help="Deployment (i.e. edx or edge)") -@click.option('--default-threshold', default=300, - help='Default queue maximum item age in seconds') -@click.option('--queue-threshold', type=(str, int), multiple=True, - help='Per queue maximum item age (seconds) in format --queue-threshold' - + ' {queue_name} {threshold}. May be used multiple times.') -@click.option('--opsgenie-api-key', '-k', envvar='OPSGENIE_API_KEY', required=True) -@click.option('--jenkins-build-url', '-j', envvar='BUILD_URL', required=False) -@click.option('--max-metrics', default=20, - help='Maximum number of CloudWatch metrics to publish') -@click.option('--dev-test-mode', is_flag=True, help='Enable dev (no-op) mode') -def check_queues(host, port, environment, deploy, default_threshold, queue_threshold, opsgenie_api_key, - jenkins_build_url, max_metrics, dev_test_mode): - ret_val = 0 - thresholds = dict(queue_threshold) - print(("Default Threshold (seconds): {}".format(default_threshold))) - print(("Per Queue Thresholds (seconds):\n{}".format(pretty_json(thresholds)))) - - timeout = 1 - redis_client = RedisWrapper(host=host, port=port, socket_timeout=timeout, - socket_connect_timeout=timeout, dev_test_mode=dev_test_mode) - celery_control = celery_connection(host, port).control - cloudwatch = CwBotoWrapper(dev_test_mode=dev_test_mode) - - namespace = "celery/{}-{}".format(environment, deploy) - metric_name = 'next_task_age' - dimension = 'queue' - next_task_age_metric_data = [] - - queue_names = set([k.decode() for k in redis_client.keys() - if (redis_client.type(k) == b'list' and - not k.decode().endswith(".pidbox") and - not k.decode().startswith("_kombu"))]) - queue_age_hash = redis_client.hgetall(QUEUE_AGE_HASH_NAME) - - # key: queue name, value: list of worker nodes for each queue - queue_workers = {} - try: - for worker, data in celery_control.inspect().active_queues().items(): - for queue in data: - queue_workers.setdefault(queue['name'], []).append(worker) - except Exception as e: - print(("Exception while getting queue to worker mappings:", e)) - print(traceback.format_exc()) - - old_state = unpack_state(queue_age_hash) - # Temp debugging - print(("DEBUG: old_state\n{}\n".format(pretty_state(old_state)))) - queue_first_items = {} - current_time = datetime.datetime.now() - - for queue_name in queue_names: - # Use -1 to get end of queue, running redis monitor shows that celery - # uses BRPOP to pull items off the right end of the queue, so that's - # what we should be monitoring - queue_first_item = redis_client.lindex(queue_name, -1) - # Check that queue_first_item is not None which is the case if the queue is empty - if queue_first_item is not None: - queue_first_items[queue_name] = json.loads(queue_first_item.decode("utf-8")) - - new_state = build_new_state(old_state, queue_first_items, current_time) - - # Temp debugging - print(("DEBUG: new_state from new_state() function\n{}\n".format(pretty_state(new_state)))) - for queue_name, first_item in queue_first_items.items(): - redacted_body = "" - threshold = default_threshold - if queue_name in thresholds: - threshold = thresholds[queue_name] - - correlation_id = new_state[queue_name]['correlation_id'] - first_occurance_time = new_state[queue_name]['first_occurance_time'] - body = {} - try: - body = extract_body(first_item) - except Exception as error: - print(("ERROR: Unable to extract task body in queue {}, exception {}".format(queue_name, error))) - print(traceback.format_exc()) - ret_val = 1 - redacted_body = {'task': body.get('task'), 'args': 'REDACTED', 'kwargs': 'REDACTED'} - active_tasks, redacted_active_tasks = get_active_tasks(celery_control, queue_workers, queue_name) - next_task_age = (current_time - first_occurance_time).total_seconds() - do_alert = next_task_age > threshold - - next_task_age_metric_data.append({ - 'MetricName': metric_name, - 'Dimensions': [{ - "Name": dimension, - "Value": queue_name - }], - 'Value': next_task_age, - 'Unit': 'Seconds', - }) - - info = generate_info( - queue_name, - correlation_id, - body, - active_tasks, - do_alert, - first_occurance_time, - current_time, - next_task_age, - threshold, - default_threshold, - jenkins_build_url, - ) - redacted_info = generate_info( - queue_name, - correlation_id, - redacted_body, - redacted_active_tasks, - do_alert, - first_occurance_time, - current_time, - next_task_age, - threshold, - default_threshold, - jenkins_build_url, - ) - print(info) - if not new_state[queue_name]['alert_created'] and do_alert: - create_alert(opsgenie_api_key, environment, deploy, queue_name, threshold, redacted_info, - dev_test_mode=dev_test_mode) - new_state[queue_name]['alert_created'] = True - elif new_state[queue_name]['alert_created'] and not do_alert: - close_alert(opsgenie_api_key, environment, deploy, queue_name, dev_test_mode=dev_test_mode) - new_state[queue_name]['alert_created'] = False - - for queue_name in set(old_state.keys()) - set(new_state.keys()): - print(("DEBUG: Checking cleared queue {}".format(queue_name))) - if old_state[queue_name]['alert_created']: - close_alert(opsgenie_api_key, environment, deploy, queue_name, dev_test_mode=dev_test_mode) - - redis_client.delete(QUEUE_AGE_HASH_NAME) - if new_state: - redis_client.hmset(QUEUE_AGE_HASH_NAME, pack_state(new_state)) - # Temp Debugging - print(("DEBUG: new_state pushed to redis\n{}\n".format(pretty_state(new_state)))) - - # Push next_task_age data to cloudwatch for tracking - if len(next_task_age_metric_data) > 0: - for metric_data_grouped in grouper(next_task_age_metric_data, max_metrics): - print(("next_task_age_metric_data {}".format(next_task_age_metric_data))) - cloudwatch.put_metric_data(Namespace=namespace, MetricData=next_task_age_metric_data) - - sys.exit(ret_val) - - -# Stolen right from the itertools recipes -# https://docs.python.org/3/library/itertools.html#itertools-recipes -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - chunks = zip_longest(*args, fillvalue=fillvalue) - # Remove Nones in function - for chunk in chunks: - yield [v for v in chunk if v is not None] - - -if __name__ == '__main__': - check_queues() diff --git a/util/jenkins/check_celery_progress/print_queue.py b/util/jenkins/check_celery_progress/print_queue.py deleted file mode 100644 index 7e495fd4833..00000000000 --- a/util/jenkins/check_celery_progress/print_queue.py +++ /dev/null @@ -1,232 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import sys -import pickle -import json -import datetime -import base64 -import zlib -import redis -import click -import backoff -from celery import Celery -from textwrap import dedent -from pprint import pprint -from six.moves import range - - -MAX_TRIES = 5 -QUEUE_AGE_HASH_NAME = "queue_age_monitoring" -DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f' - - -class RedisWrapper(object): - def __init__(self, *args, **kwargs): - self.redis = redis.StrictRedis(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def keys(self): - return list(self.redis.keys()) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def type(self, key): - return self.redis.type(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def llen(self, key): - return self.redis.llen(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def lindex(self, key, index): - return self.redis.lindex(key, index) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def hgetall(self, key): - return self.redis.hgetall(key) - - -def pretty_json(obj): - return json.dumps(obj, indent=4, sort_keys=True) - - -def unpack_state(packed_state): - decoded_state = {k.decode("utf-8"): v.decode("utf-8") for k, v in packed_state.items()} - unpacked_state = {} - - for key, value in decoded_state.items(): - decoded_value = json.loads(value) - unpacked_state[key] = { - 'correlation_id': decoded_value['correlation_id'], - 'first_occurance_time': datetime_from_str(decoded_value['first_occurance_time']), - 'alert_created': decoded_value['alert_created'], - } - - return unpacked_state - - -def extract_body(task): - body = base64.b64decode(task['body']) - body_dict = {} - - if 'headers' in task and 'compression' in task['headers'] and task['headers']['compression'] == 'application/x-gzip': - body = zlib.decompress(body) - - if task.get('content-type') == 'application/json': - body_dict = json.loads(body.decode("utf-8")) - elif task.get('content-type') == 'application/x-python-serialize': - body_dict = {k.decode("utf-8"): v for k, v in pickle.loads(body, encoding='bytes').items()} - return body_dict - - -def generate_info( - queue_name, - correlation_id, - body, - active_tasks, -): - next_task = "Key missing" - args = "Key missing" - kwargs = "Key missing" - - if 'task' in body: - next_task = body['task'] - - if 'args' in body: - args = body['args'] - - if 'kwargs' in body: - kwargs = body['kwargs'] - - output = str.format( - dedent(""" - ============================================= - queue_name = {} - correlation_id = {} - --------------------------------------------- - active_tasks = {} - --------------------------------------------- - next_task = {} - args = {} - kwargs = {} - ============================================= - """), - queue_name, - correlation_id, - active_tasks, - next_task, - args, - kwargs, - ) - return output - - -def celery_connection(host, port): - celery_client = " " - try: - broker_url = "redis://" + host + ":" + str(port) - celery_client = Celery(broker=broker_url) - except Exception as e: - print(("Exception in connection():", e)) - return celery_client - - -# Functionality added to get list of currently running tasks -# because Redis returns only the next tasks in the list -def get_active_tasks(celery_control, queue_workers, queue_name): - active_tasks = dict() - redacted_active_tasks = dict() - if queue_name in queue_workers: - workers = queue_workers[queue_name] - try: - for worker, data in celery_control.inspect(workers).active().items(): - for task in data: - active_tasks.setdefault( - task["hostname"], []).append([ - 'task: {}'.format(task.get("name")), - 'args: {}'.format(task.get("args")), - 'kwargs: {}'.format(task.get("kwargs")), - ]) - redacted_active_tasks.setdefault( - task["hostname"], []).append([ - 'task: {}'.format(task.get("name")), - 'args: REDACTED', - 'kwargs: REDACTED', - ]) - except Exception as e: - print(("Exception in get_active_tasks():", e)) - return (pretty_json(active_tasks), pretty_json(redacted_active_tasks)) - - -@click.command() -@click.option('--host', '-h', default='localhost', - help='Hostname of redis server', required=True) -@click.option('--port', '-p', default=6379, help='Port of redis server') -@click.option('--queue', '-q', required=True) -@click.option('--items', '-i', default=1, help='Number of items to print') -@click.option('--body/--no-body', default=False, help='Print full body of tasks') -def check_queues(host, port, queue, items, body): - queue_name = queue - ret_val = 0 - - timeout = 1 - redis_client = RedisWrapper(host=host, port=port, socket_timeout=timeout, - socket_connect_timeout=timeout) - celery_control = celery_connection(host, port).control - - # key: queue name, value: list of worker nodes for each queue - queue_workers = {} - try: - for worker, data in celery_control.inspect().active_queues().items(): - for queue in data: - queue_workers.setdefault(queue['name'], []).append(worker) - except Exception as e: - print(("Exception while getting queue to worker mappings:", e)) - - for count in range(items): - print(("Count: {}".format(count))) - queue_first_item = redis_client.lindex(queue_name, count) - # Check that queue_first_item is not None which is the case if the queue is empty - if queue_first_item is not None: - queue_first_item_decoded = json.loads(queue_first_item.decode("utf-8")) - - correlation_id = queue_first_item_decoded['properties']['correlation_id'] - - extracted_body = {} - try: - extracted_body = extract_body(queue_first_item_decoded) - except Exception as error: - print(("ERROR: Unable to extract task body in queue {}, exception {}".format(queue_name, error))) - ret_val = 1 - active_tasks, redacted_active_tasks = get_active_tasks(celery_control, queue_workers, queue_name) - - info = generate_info( - queue_name, - correlation_id, - extracted_body, - active_tasks, - ) - print(info) - if body: - print("BODY") - pprint(extracted_body) - - sys.exit(ret_val) - - -if __name__ == '__main__': - check_queues() diff --git a/util/jenkins/check_celery_progress/requirements.txt b/util/jenkins/check_celery_progress/requirements.txt deleted file mode 100644 index 9091be59df3..00000000000 --- a/util/jenkins/check_celery_progress/requirements.txt +++ /dev/null @@ -1,84 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# make upgrade -# -amqp==2.6.1 - # via kombu -awscli==1.14.32 - # via -r requirements/celery_progress.in -backoff==1.4.3 - # via -r requirements/celery_progress.in -billiard==3.6.4.0 - # via celery -boto3==1.5.4 - # via -r requirements/celery_progress.in -botocore==1.8.36 - # via - # awscli - # boto3 - # s3transfer -celery==4.4.7 - # via -r requirements/celery_progress.in -certifi==2020.12.5 - # via - # opsgenie-sdk - # requests -chardet==4.0.0 - # via requests -click==6.7 - # via -r requirements/celery_progress.in -colorama==0.3.7 - # via awscli -docutils==0.17 - # via - # awscli - # botocore -idna==2.10 - # via requests -jmespath==0.10.0 - # via - # boto3 - # botocore -kombu==4.6.11 - # via celery -opsgenie-sdk==0.3.1 - # via -r requirements/celery_progress.in -pyasn1==0.4.8 - # via rsa -python-dateutil==2.8.1 - # via - # botocore - # opsgenie-sdk -pytz==2021.1 - # via - # celery - # opsgenie-sdk -pyyaml==3.12 - # via awscli -redis==3.5.3 - # via -r requirements/celery_progress.in -requests==2.25.1 - # via opsgenie-sdk -rsa==3.4.2 - # via awscli -s3transfer==0.1.13 - # via - # awscli - # boto3 -six==1.15.0 - # via - # opsgenie-sdk - # python-dateutil -urllib3==1.26.4 - # via - # opsgenie-sdk - # requests -vine==1.3.0 - # via - # amqp - # celery - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/util/jenkins/check_celery_progress/test_check_celery_progress.py b/util/jenkins/check_celery_progress/test_check_celery_progress.py deleted file mode 100644 index 25f6f4cb922..00000000000 --- a/util/jenkins/check_celery_progress/test_check_celery_progress.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import absolute_import -import unittest -import datetime -from datetime import timedelta -from check_celery_progress import build_new_state, datetime_from_str, should_create_alert, pack_state, unpack_state - -class TestCheckCeleryQueues(unittest.TestCase): - - queue_first_items_0 = { - "edx.lms.core.grades_policy_change": { - "body": "", - "headers": { - "compression": "application/x-gzip" - }, - "content-type": "application/json", - "properties": { - "body_encoding": "base64", - "correlation_id": "8a661b24-cf3a-49f6-ba72-824d20d1cc02", - "reply_to": "9d88dd87-c55f-3d95-a725-176db14a16dd", - "delivery_info": { - "priority": 0, - "routing_key": "edx.lms.core.grades_policy_change", - "exchange": "edx.lms.core" - }, - "delivery_mode": 2, - "delivery_tag": "0efb9ecc-2914-4a3c-98fb-358a67d678d8" - }, - "content-encoding": "utf-8" - }, - "edx.lms.core.default": { - "body": "", - "headers": { - "compression": "application/x-gzip" - }, - "content-type": "application/json", - "properties": { - "body_encoding": "base64", - "correlation_id": "27d76be4-408b-4f5b-a132-5ad043465e90", - "reply_to": "2e6506c2-46dd-3dd7-9be0-a03e0e5c2f7f", - "delivery_info": { - "priority": 0, - "routing_key": "edx.lms.core.default", - "exchange": "edx.lms.core" - }, - "delivery_mode": 2, - "delivery_tag": "2627a9a2-1941-4890-8fe3-c477c62b707d" - }, - "content-encoding": "utf-8" - } - } - - - queue_first_items_1 = { - "edx.lms.core.grades_policy_change": { - "body": "", - "headers": { - "compression": "application/x-gzip" - }, - "content-type": "application/json", - "properties": { - "body_encoding": "base64", - "correlation_id": "8a661b24-cf3a-49f6-ba72-824d20d1cc02", - "reply_to": "9d88dd87-c55f-3d95-a725-176db14a16dd", - "delivery_info": { - "priority": 0, - "routing_key": "edx.lms.core.grades_policy_change", - "exchange": "edx.lms.core" - }, - "delivery_mode": 2, - "delivery_tag": "0efb9ecc-2914-4a3c-98fb-358a67d678d8" - }, - "content-encoding": "utf-8" - }, - "edx.lms.core.default": { - "body": "", - "headers": { - "compression": "application/x-gzip" - }, - "content-type": "application/json", - "properties": { - "body_encoding": "base64", - "correlation_id": "27d76be4-408b-4f5b-a132-c0ffee465e90", - "reply_to": "2e6506c2-46dd-3dd7-9be0-a03e0e5c2f7f", - "delivery_info": { - "priority": 0, - "routing_key": "edx.lms.core.default", - "exchange": "edx.lms.core" - }, - "delivery_mode": 2, - "delivery_tag": "2627a9a2-1941-4890-8fe3-c477c62b707d" - }, - "content-encoding": "utf-8" - } - } - - time_0 = datetime_from_str("2018-10-04 11:00:51.111367") - time_1_min = datetime_from_str("2018-10-04 11:01:51.111367") - - threshold = 5 * 60 - - def test_equal_output_if_queues_stuck(self): - state_0 = build_new_state({}, self.queue_first_items_0, self.time_0) - state_1 = build_new_state(state_0, self.queue_first_items_0, self.time_1_min) - self.assertEqual(state_0, state_1) - - def test_build_new_state_missing_alert_created(self): - state_0 = build_new_state({}, self.queue_first_items_0, self.time_0) - state_0['edx.lms.core.default'].pop('alert_created') - state_1 = build_new_state(state_0, self.queue_first_items_0, self.time_1_min) - self.assertFalse(state_1['edx.lms.core.default']['alert_created']) - - def test_build_new_state_alert_created(self): - state_0 = build_new_state({}, self.queue_first_items_0, self.time_0) - state_1 = build_new_state(state_0, self.queue_first_items_0, self.time_1_min) - self.assertFalse(state_0['edx.lms.core.grades_policy_change']['alert_created']) - self.assertFalse(state_0['edx.lms.core.default']['alert_created']) - self.assertFalse(state_1['edx.lms.core.grades_policy_change']['alert_created']) - self.assertFalse(state_1['edx.lms.core.default']['alert_created']) - - def test_build_new_state_alert_created_preserved(self): - state_0 = build_new_state({}, self.queue_first_items_0, self.time_0) - state_0['edx.lms.core.default']['alert_created'] = True - state_1 = build_new_state(state_0, self.queue_first_items_0, self.time_1_min) - self.assertTrue(state_1['edx.lms.core.default']['alert_created']) - - def test_output_1_queue_changed(self): - state_0 = build_new_state({}, self.queue_first_items_0, self.time_0) - state_1 = build_new_state(state_0, self.queue_first_items_1, self.time_1_min) - self.assertEqual(state_0['edx.lms.core.grades_policy_change'], state_1['edx.lms.core.grades_policy_change']) - self.assertEqual(state_1['edx.lms.core.default']['first_occurance_time'], self.time_1_min) - self.assertEqual(state_1['edx.lms.core.default']['correlation_id'], "27d76be4-408b-4f5b-a132-c0ffee465e90") - self.assertFalse(state_1['edx.lms.core.default']['alert_created']) - - def test_should_create_alert_0_delta(self): - first_occurance_time = self.time_0 - result = should_create_alert(first_occurance_time, self.time_0, self.threshold) - self.assertEqual(False, result) - - def test_should_create_alert_under_threshold(self): - first_occurance_time = self.time_0 - timedelta(seconds=self.threshold-60) - result = should_create_alert(first_occurance_time, self.time_0, self.threshold) - self.assertEqual(False, result) - - def test_should_create_alert_over_threshold(self): - first_occurance_time = self.time_0 - timedelta(seconds=self.threshold+60) - result = should_create_alert(first_occurance_time, self.time_0, self.threshold) - self.assertEqual(True, result) - - def test_should_create_alert_negative_delta(self): - first_occurance_time = self.time_0 + timedelta(seconds=self.threshold+60) - result = should_create_alert(first_occurance_time, self.time_0, self.threshold) - self.assertEqual(False, result) - - def test_pack_state(self): - # Round trip state to make sure all fields are preserved - state = build_new_state({}, self.queue_first_items_0, self.time_0) - packed_state = pack_state(state) - encoded_packed_state = {k.encode("utf-8"): v.encode("utf-8") for k, v in packed_state.items()} - unpacked_state = unpack_state(encoded_packed_state) - self.assertEqual(state, unpacked_state) - - -if __name__ == '__main__': - unittest.main() diff --git a/util/jenkins/update_celery_monitoring/create_celery_dashboard.py b/util/jenkins/update_celery_monitoring/create_celery_dashboard.py deleted file mode 100644 index e767c57bf79..00000000000 --- a/util/jenkins/update_celery_monitoring/create_celery_dashboard.py +++ /dev/null @@ -1,215 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import pprint -import re - -import boto3 -import botocore -import backoff -import click -import json - -MAX_TRIES = 1 - -class CwBotoWrapper(object): - def __init__(self): - self.client = boto3.client('cloudwatch') - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def list_metrics(self, *args, **kwargs): - return self.client.list_metrics(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def put_dashboard(self, *args, **kwargs): - return self.client.put_dashboard(*args, **kwargs) - -def generate_dashboard_widget_metrics(cloudwatch, namespace, metric_name, dimension_name, properties={}, include_filter=None, exclude_filter=None, right_axis_items=[]): - # https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Dashboard-Body-Structure.html#CloudWatch-Dashboard-Properties-Metrics-Array-Format - # [Namespace, MetricName, [{DimensionName,DimensionValue}...] [Rendering Properties Object] ] - # ['AWS/EC2', 'CPUUtilization', 'AutoScalingGroupName', 'asg-name', {'period': 60}] - pp = pprint.PrettyPrinter(indent=4) - - metrics = cloudwatch.list_metrics( - Namespace=namespace, MetricName=metric_name, Dimensions=[{"Name": dimension_name}] - ) - - values = [] - - for metric in metrics['Metrics']: - for dimension in metric['Dimensions']: - if dimension['Name'] == dimension_name: - if include_filter is None or re.search(include_filter, dimension['Value'], re.IGNORECASE): - if exclude_filter is None or not re.search(exclude_filter, dimension['Value'], re.IGNROECASE): - values.append(dimension['Value']) - - values.sort() - - new_widget_metrics = [] - for value in values: - value_properties = properties.copy() - value_properties['label'] = value - if value in right_axis_items: - value_properties["yAxis"] = "right" - new_widget_metrics.append([namespace, metric_name, dimension_name, value, value_properties]) - - return new_widget_metrics - -# * means that all arguments after cloudwatch are keyword arguments only and are not positional -def generate_dashboard_widget( - cloudwatch, - *, - x=0, - y, - title, - namespace, - metric_name, - dimension_name, - metrics_properties={}, - include_filter=None, - exclude_filter=None, - height, - width=24, - stacked=False, - region='us-east-1', - period=60, - right_axis_items=[] -): - return { 'type': 'metric', 'height': height, 'width': width, 'x': x, 'y': y, - 'properties': { - 'period': period, 'view': 'timeSeries', 'stacked': stacked, 'region': region, - 'title': "{} (auto-generated)".format(title), - 'metrics': generate_dashboard_widget_metrics(cloudwatch, namespace, metric_name, dimension_name, metrics_properties, - include_filter=include_filter, exclude_filter=exclude_filter, right_axis_items=right_axis_items) - } -} - -@click.command() -@click.option('--environment', '-e', required=True) -@click.option('--deploy', '-d', required=True, - help="Deployment (i.e. edx or edge)") -def generate_dashboard(environment, deploy): - pp = pprint.PrettyPrinter(indent=4) - cloudwatch = CwBotoWrapper() - - dashboard_name = "{}-{}-queues".format(environment, deploy) - celery_namespace = "celery/{}-{}".format(environment, deploy) - xqueue_namespace = "xqueue/{}-{}".format(environment, deploy) - - widgets = [] - width = 24 - y_cord = 0 - region = "us-east-1" - right_axis_items=["edx.lms.core.ace", "edx.lms.core.background_process"] - right_axis_items_age=[] - - height = 9 - - cpu_widget = generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{}-Worker ASG Average CPU".format(environment, deploy), - namespace="AWS/EC2", metric_name="CPUUtilization", dimension_name="AutoScalingGroupName", - include_filter="{}-{}-Worker".format(environment, deploy) - ) - - cpu_widget['properties']['annotations'] = { - 'horizontal': [ - {'label': 'Scale Up', 'value': 90, 'color': '#d62728'}, - {'label': 'Scale Down', 'value': 45, 'color': '#2ca02c'} - ] - } - - cpu_widget['properties']['yAxis'] = {'left': {'min': 0, 'max': 100}} - - widgets.append(cpu_widget) - - y_cord += height - height = 6 - - worker_count_widget = generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{}-Worker Count".format(environment, deploy), - namespace=celery_namespace, metric_name="count", dimension_name="workers" - ) - - worker_count_widget['properties']['stacked'] = True - - widgets.append(worker_count_widget) - - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} All Celery Queues".format(environment, deploy), - namespace=celery_namespace, metric_name="queue_length", dimension_name="queue", - right_axis_items=right_axis_items - )) - - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} All Queues Next Task Age".format(environment, deploy), - namespace=celery_namespace, metric_name="next_task_age", dimension_name="queue", - right_axis_items=right_axis_items_age - )) - - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} LMS Queues".format(environment, deploy), - namespace=celery_namespace, metric_name="queue_length", dimension_name="queue", - include_filter="^edx.lms", - right_axis_items=right_axis_items - )) - - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} CMS Queues".format(environment, deploy), - namespace=celery_namespace, metric_name="queue_length", dimension_name="queue", - include_filter="^edx.cms", - right_axis_items=right_axis_items - )) - - if deploy == 'edx' and environment == 'prod': - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} Xqueue Queues".format(environment, deploy), - namespace=xqueue_namespace, metric_name="queue_length", dimension_name="queue", - )) - - - if deploy in ["edx", "edge"]: - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} Ecommerce".format(environment, deploy), - namespace=celery_namespace, metric_name="queue_length", dimension_name="queue", - include_filter="^ecommerce\.", - )) - - y_cord += height - height = 9 - - widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} Legacy Celery (Ecommerce) should be 0".format(environment, deploy), - namespace=celery_namespace, metric_name="queue_length", dimension_name="queue", - include_filter="celery", - )) - - dashboard_body = { 'widgets': widgets } - - print("Dasboard Body") - pp.pprint(dashboard_body) - - cloudwatch.put_dashboard(DashboardName=dashboard_name, DashboardBody=json.dumps(dashboard_body)) - -if __name__ == '__main__': - generate_dashboard() diff --git a/util/jenkins/update_celery_monitoring/requirements.txt b/util/jenkins/update_celery_monitoring/requirements.txt deleted file mode 100644 index f6cd1e4fbb1..00000000000 --- a/util/jenkins/update_celery_monitoring/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# make upgrade -# -awscli==1.14.32 # via -r requirements/celery.in -backoff==1.4.3 # via -r requirements/celery.in -boto3==1.5.4 # via -r requirements/celery.in -botocore==1.8.36 # via awscli, boto3, s3transfer -click==6.7 # via -r requirements/celery.in -colorama==0.3.7 # via awscli -docutils==0.17 # via awscli, botocore -jmespath==0.10.0 # via boto3, botocore -pyasn1==0.4.8 # via rsa -python-dateutil==2.8.1 # via botocore -pyyaml==3.12 # via awscli -redis==2.10.6 # via -r requirements/celery.in -rsa==3.4.2 # via awscli -s3transfer==0.1.13 # via awscli, boto3 -six==1.15.0 # via python-dateutil diff --git a/util/jenkins/update_celery_monitoring/update_celery_monitoring.py b/util/jenkins/update_celery_monitoring/update_celery_monitoring.py deleted file mode 100644 index 696d41b529b..00000000000 --- a/util/jenkins/update_celery_monitoring/update_celery_monitoring.py +++ /dev/null @@ -1,292 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -import re -import redis -import click -import boto3 -import botocore -import backoff -from pprint import pprint -from itertools import zip_longest -from collections import defaultdict - -MAX_TRIES = 5 - -# Queues that should be gone. Inclusion in this list will stop this script from -# zero filling them, but if they are >0 they will still get tracked -queue_blacklist = ['celery', 'ecommerce'] - - -class RedisWrapper(object): - def __init__(self, *args, **kwargs): - self.redis = redis.StrictRedis(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def keys(self): - return list(self.redis.keys()) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def type(self, key): - return self.redis.type(key) - - @backoff.on_exception(backoff.expo, - (redis.exceptions.TimeoutError, - redis.exceptions.ConnectionError), - max_tries=MAX_TRIES) - def llen(self, key): - return self.redis.llen(key) - - -class CwBotoWrapper(object): - def __init__(self, dev_test_mode=True): - self.dev_test_mode = dev_test_mode - self.client = boto3.client('cloudwatch') - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def list_metrics(self, *args, **kwargs): - return self.client.list_metrics(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def put_metric_data(self, *args, **kwargs): - if self.dev_test_mode: - print(("Test Mode: would have run put_metric_data({},{})".format(args, kwargs))) - else: - return self.client.put_metric_data(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def describe_alarms(self, *args, **kwargs): - return self.client.describe_alarms(*args, **kwargs) - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def put_metric_alarm(self, *args, **kwargs): - if self.dev_test_mode: - print(("Test Mode: would have run put_metric_alarm({},{})".format(args, kwargs))) - else: - return self.client.put_metric_alarm(*args, **kwargs) - - -class Ec2BotoWrapper(object): - def __init__(self): - self.client = boto3.client('ec2') - - @backoff.on_exception(backoff.expo, - (botocore.exceptions.ClientError), - max_tries=MAX_TRIES) - def describe_instances(self, *args, **kwargs): - return self.client.describe_instances(*args, **kwargs) - - -def count_workers(environment, deploy, cluster): - ec2 = Ec2BotoWrapper() - - counts_by_play = defaultdict(int) - - reservations = ec2.describe_instances( - Filters=[ - {'Name': 'tag:environment', 'Values': [environment]}, - {'Name': 'tag:deployment', 'Values': [deploy]}, - {'Name': 'tag:cluster', 'Values': [cluster]}, - {'Name': 'instance-state-name', 'Values': ['running']}, - ] - )['Reservations'] - - for reservation in reservations: - for instance in reservation["Instances"]: - tag_asg = None - for tag in instance['Tags']: - if tag.get('Key') == 'aws:autoscaling:groupName': - # Reduce number of metrics from 1000 to 10 by changing first 2 numbers of ASG version to stars - # This reduces the cloudwatch cost - tag_asg = re.sub('-v[0-9]{2}', '-v**', tag.get('Value')) - counts_by_play[tag_asg] += 1 - - metric_data = [] - - for play, num_workers in counts_by_play.items(): - metric_data.append({ - 'MetricName': 'count', - 'Dimensions': [{ - "Name": "workers", - "Value": play - }], - 'Value': num_workers - } - ) - - return metric_data - - -@click.command() -@click.option('--host', '-h', default='localhost', - help='Hostname of redis server') -@click.option('--port', '-p', default=6379, help='Port of redis server') -@click.option('--environment', '-e', required=True) -@click.option('--deploy', '-d', required=True, - help="Deployment (i.e. edx or edge)") -@click.option('--max-metrics', default=20, - help='Maximum number of CloudWatch metrics to publish') -@click.option('--threshold', default=50, - help='Default maximum queue length before alarm notification is' - + ' sent') -@click.option('--queue-threshold', type=(str, int), multiple=True, - help='Threshold per queue in format --queue-threshold' - + ' {queue_name} {threshold}. May be used multiple times') -@click.option('--sns-arn', '-s', help='ARN for SNS alert topic', required=True) -@click.option('--dev-test-mode', is_flag=True, help='Enable dev (no-op) mode') -def check_queues(host, port, environment, deploy, max_metrics, threshold, - queue_threshold, sns_arn, dev_test_mode): - - thresholds = dict(queue_threshold) - - timeout = 1 - redis_client = RedisWrapper(host=host, port=port, socket_timeout=timeout, - socket_connect_timeout=timeout) - cloudwatch = CwBotoWrapper(dev_test_mode=dev_test_mode) - namespace = "celery/{}-{}".format(environment, deploy) - metric_name = 'queue_length' - dimension = 'queue' - response = cloudwatch.list_metrics(Namespace=namespace, - MetricName=metric_name, - Dimensions=[{'Name': dimension}]) - existing_queues = [] - for m in response["Metrics"]: - existing_queues.extend( - [d['Value'] for d in m["Dimensions"] if ( - d['Name'] == dimension and - not d['Value'] in queue_blacklist and - not d['Value'].endswith(".pidbox") and - not d['Value'].startswith("_kombu"))]) - - redis_queues = set([k.decode() for k in redis_client.keys() - if (redis_client.type(k) == b'list' and - not k.decode().endswith(".pidbox") and - not k.decode().startswith("_kombu"))]) - - all_queues = existing_queues + list( - set(redis_queues).difference(existing_queues) - ) - - metric_data = [] - - for queue_name in all_queues: - metric_data.append({ - 'MetricName': metric_name, - 'Dimensions': [{ - "Name": dimension, - "Value": queue_name - }], - 'Value': redis_client.llen(queue_name), - 'Unit': 'Count', - }) - - if len(metric_data) > 0: - for metric_data_grouped in grouper(metric_data, max_metrics): - print("metric_data:") - pprint(metric_data, width=120) - cloudwatch.put_metric_data(Namespace=namespace, MetricData=metric_data) - - for queue in all_queues: - dimensions = [{'Name': dimension, 'Value': queue}] - queue_threshold = threshold - if queue in thresholds: - queue_threshold = thresholds[queue] - # Period is in seconds - period = 60 - evaluation_periods = 15 - comparison_operator = "GreaterThanThreshold" - treat_missing_data = "notBreaching" - statistic = "Maximum" - actions = [sns_arn] - alarm_name = "{}-{} {} queue length over threshold".format(environment, - deploy, - queue) - - existing_alarms = cloudwatch.describe_alarms(AlarmNames=[alarm_name])['MetricAlarms'] - do_put_alarm = False - if len(existing_alarms) > 1: - print(("WARNINING: found multiple existing alarms for {}".format(alarm_name))) - pprint(existing_alarms) - do_put_alarm = True - elif len(existing_alarms) == 1: - existing_alarm = existing_alarms[0] - - if (existing_alarm.get('Threshold') != queue_threshold or - existing_alarm.get('AlarmDescription') != alarm_name or - existing_alarm.get('Namespace') != namespace or - existing_alarm.get('MetricName') != metric_name or - existing_alarm.get('Dimensions') != dimensions or - existing_alarm.get('Period') != period or - existing_alarm.get('EvaluationPeriods') != evaluation_periods or - existing_alarm.get('TreatMissingData') != treat_missing_data or - existing_alarm.get('ComparisonOperator') != comparison_operator or - existing_alarm.get('Statistic') != statistic): - do_put_alarm = True - print("1") - elif not (len(existing_alarm.get('InsufficientDataActions')) == 1 and - existing_alarm.get('InsufficientDataActions')[0] == actions[0]): - do_put_alarm = True - elif not (len(existing_alarm.get('OKActions')) == 1 and - existing_alarm.get('OKActions')[0] == actions[0]): - do_put_alarm = True - elif not (len(existing_alarm.get('AlarmActions')) == 1 and - existing_alarm.get('AlarmActions')[0] == actions[0]): - do_put_alarm = True - if do_put_alarm: - print(('Updating existing alarm "{}"'.format(alarm_name))) - else: - do_put_alarm = True - print(('Creating new alarm "{}"'.format(alarm_name))) - if not do_put_alarm: - print(('Not updating alarm "{}", no changes'.format(alarm_name))) - else: - print(('put_alarm_metric: {}'.format(alarm_name))) - cloudwatch.put_metric_alarm(AlarmName=alarm_name, - AlarmDescription=alarm_name, - Namespace=namespace, - MetricName=metric_name, - Dimensions=dimensions, - Period=period, - EvaluationPeriods=evaluation_periods, - TreatMissingData=treat_missing_data, - Threshold=queue_threshold, - ComparisonOperator=comparison_operator, - Statistic=statistic, - InsufficientDataActions=actions, - OKActions=actions, - AlarmActions=actions) - - # Track number of worker instances so it can be graphed in CloudWatch - workers_metric_data = count_workers(environment, deploy, 'worker') - print("workers_metric_data:") - pprint(workers_metric_data, width=120) - cloudwatch.put_metric_data(Namespace=namespace, MetricData=workers_metric_data) - - -# Stolen right from the itertools recipes -# https://docs.python.org/3/library/itertools.html#itertools-recipes -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" - args = [iter(iterable)] * n - chunks = zip_longest(*args, fillvalue=fillvalue) - # Remove Nones in function - for chunk in chunks: - yield [v for v in chunk if v is not None] - - -if __name__ == '__main__': - check_queues() From f8d0f9fd459d0b3efd9d9c61f0b89167f8cf3cf0 Mon Sep 17 00:00:00 2001 From: DawoudSheraz Date: Fri, 1 Jan 2021 14:46:24 +0500 Subject: [PATCH 012/664] update pyyaml --- .../roles/aws/templates/requirements.txt.j2 | 59 +++++-- requirements.txt | 146 +++++++++++++----- requirements/aws.in | 3 +- requirements/base.in | 8 +- requirements/pingdom.in | 2 +- requirements/pip-tools.txt | 7 +- util/elasticsearch/requirements.txt | 18 ++- util/jenkins/requirements-cloudflare.txt | 15 +- util/jenkins/requirements.txt | 2 +- util/pingdom/requirements.txt | 24 ++- util/vpc-tools/requirements.txt | 21 ++- 11 files changed, 212 insertions(+), 93 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 918ec04b78e..9f9fe54cbcb 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,19 +4,46 @@ # # make upgrade # -awscli==1.18.129 # via -r requirements/aws.in -boto3==1.14.52 # via -r requirements/aws.in -boto==2.49.0 # via -r requirements/aws.in -botocore==1.17.52 # via awscli, boto3, s3transfer -colorama==0.4.3 # via awscli -docutils==0.15.2 # via awscli, botocore -jmespath==0.10.0 # via boto3, botocore -pyasn1==0.4.8 # via rsa -python-dateutil==2.8.1 # via botocore, s3cmd -python-magic==0.4.18 # via s3cmd -pyyaml==3.11 # via -r requirements/aws.in, awscli -rsa==4.5 # via awscli -s3cmd==2.1.0 # via -r requirements/aws.in -s3transfer==0.3.3 # via awscli, boto3 -six==1.15.0 # via python-dateutil -urllib3==1.25.10 # via botocore +awscli==1.18.207 + # via -r requirements/aws.in +boto3==1.16.47 + # via -r requirements/aws.in +boto==2.49.0 + # via -r requirements/aws.in +botocore==1.19.47 + # via + # awscli + # boto3 + # s3transfer +colorama==0.4.3 + # via awscli +docutils==0.15.2 + # via awscli +jmespath==0.10.0 + # via + # boto3 + # botocore +pyasn1==0.4.8 + # via rsa +python-dateutil==2.8.1 + # via + # botocore + # s3cmd +python-magic==0.4.18 + # via s3cmd +pyyaml==5.3.1 + # via + # -r requirements/aws.in + # awscli +rsa==4.5 + # via awscli +s3cmd==2.1.0 + # via -r requirements/aws.in +s3transfer==0.3.3 + # via + # awscli + # boto3 +six==1.15.0 + # via python-dateutil +urllib3==1.26.2 + # via botocore diff --git a/requirements.txt b/requirements.txt index 63aa26abba4..1806ca26b7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,43 +4,109 @@ # # make upgrade # -ansible==2.8.15 # via -r requirements/base.in -awscli==1.16.309 # via -r requirements/base.in -bcrypt==3.1.7 # via -c requirements/constraints.txt, paramiko -boto3==1.10.45 # via -r requirements/base.in -boto==2.48.0 # via -r requirements/base.in -botocore==1.13.45 # via awscli, boto3, s3transfer -certifi==2020.6.20 # via requests -cffi==1.14.2 # via bcrypt, cryptography, pynacl -chardet==3.0.4 # via requests -colorama==0.4.1 # via awscli -cryptography==3.1 # via ansible, paramiko -datadog==0.8.0 # via -r requirements/base.in -decorator==4.4.2 # via datadog, networkx -docopt==0.6.2 # via -r requirements/base.in -docutils==0.15.2 # via awscli, botocore -ecdsa==0.13.3 # via -r requirements/base.in -idna==2.7 # via requests -jinja2==2.8 # via -r requirements/base.in, ansible -jmespath==0.10.0 # via boto3, botocore -markupsafe==1.1.1 # via jinja2 -mysqlclient==1.4.6 # via -r requirements/base.in -networkx==1.11 # via -r requirements/base.in -paramiko==2.4.2 # via -r requirements/base.in -pathlib2==2.3.0 # via -r requirements/base.in -prettytable==0.7.2 # via -r requirements/base.in -pyasn1==0.4.8 # via paramiko, rsa -pycparser==2.20 # via cffi -pycrypto==2.6.1 # via -r requirements/base.in -pymongo==3.9.0 # via -r requirements/base.in -pynacl==1.4.0 # via paramiko -python-dateutil==2.8.1 # via botocore -pyyaml==5.2 # via -r requirements/base.in, ansible, awscli -requests==2.20.0 # via -r requirements/base.in, datadog -rsa==3.4.2 # via awscli -s3transfer==0.2.1 # via awscli, boto3 -six==1.15.0 # via bcrypt, cryptography, pathlib2, pynacl, python-dateutil -urllib3==1.24.3 # via botocore, requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +ansible==2.8.18 + # via -r requirements/base.in +awscli==1.18.207 + # via -r requirements/base.in +bcrypt==3.1.7 + # via + # -c requirements/constraints.txt + # paramiko +boto3==1.16.47 + # via -r requirements/base.in +boto==2.49.0 + # via -r requirements/base.in +botocore==1.19.47 + # via + # awscli + # boto3 + # s3transfer +certifi==2020.12.5 + # via requests +cffi==1.14.4 + # via + # bcrypt + # cryptography + # pynacl +chardet==4.0.0 + # via requests +colorama==0.4.3 + # via awscli +cryptography==3.2.1 + # via + # ansible + # paramiko +datadog==0.8.0 + # via -r requirements/base.in +decorator==4.4.2 + # via + # datadog + # networkx +docopt==0.6.2 + # via -r requirements/base.in +docutils==0.15.2 + # via awscli +ecdsa==0.13.3 + # via -r requirements/base.in +idna==2.10 + # via requests +jinja2==2.8 + # via + # -r requirements/base.in + # ansible +jmespath==0.10.0 + # via + # boto3 + # botocore +markupsafe==1.1.1 + # via jinja2 +mysqlclient==1.4.6 + # via -r requirements/base.in +networkx==1.11 + # via -r requirements/base.in +paramiko==2.4.2 + # via -r requirements/base.in +pathlib2==2.3.0 + # via -r requirements/base.in +prettytable==0.7.2 + # via -r requirements/base.in +pyasn1==0.4.8 + # via + # paramiko + # rsa +pycparser==2.20 + # via cffi +pycrypto==2.6.1 + # via -r requirements/base.in +pymongo==3.9.0 + # via -r requirements/base.in +pynacl==1.4.0 + # via paramiko +python-dateutil==2.8.1 + # via botocore +pyyaml==5.3.1 + # via + # -r requirements/base.in + # ansible + # awscli +requests==2.25.1 + # via + # -r requirements/base.in + # datadog +rsa==4.5 + # via awscli +s3transfer==0.3.3 + # via + # awscli + # boto3 +six==1.15.0 + # via + # bcrypt + # cryptography + # pathlib2 + # pynacl + # python-dateutil +urllib3==1.26.2 + # via + # botocore + # requests diff --git a/requirements/aws.in b/requirements/aws.in index b3bcd6a2ead..e457993d02a 100644 --- a/requirements/aws.in +++ b/requirements/aws.in @@ -5,5 +5,4 @@ boto boto3 futures ; python_version == "2.7" # via s3transfer s3cmd -# Lock pyYaml to what's installed on Ubuntu -pyyaml==3.11 +pyyaml diff --git a/requirements/base.in b/requirements/base.in index 36dcc17bd79..ab61bceb885 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,9 +2,9 @@ # Standard dependencies for Ansible runs ansible<2.9.0 -awscli==1.16.309 -boto==2.48.0 -boto3==1.10.45 +awscli +boto +boto3 datadog==0.8.0 docopt==0.6.2 ecdsa==0.13.3 @@ -17,4 +17,4 @@ prettytable==0.7.2 pycrypto==2.6.1 pymongo==3.9.0 # Needed for the mongo_* modules (playbooks/library/mongo_*) PyYAML -requests==2.20.0 +requests diff --git a/requirements/pingdom.in b/requirements/pingdom.in index 960ea707771..e9ff57f0095 100644 --- a/requirements/pingdom.in +++ b/requirements/pingdom.in @@ -1,6 +1,6 @@ # Requirements for util/pingdom/create_pingdom_alerts.py click==6.7 -PyYAML==3.12 +PyYAML requests==2.20.0 six==1.14.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 4b50eae7bd7..aa6ffb82da2 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,9 +4,10 @@ # # make upgrade # -click==7.1.2 # via pip-tools -pip-tools==5.3.1 # via -r requirements/pip-tools.in -six==1.15.0 # via pip-tools +click==7.1.2 + # via pip-tools +pip-tools==5.5.0 + # via -r requirements/pip-tools.in # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index a77725c6ffe..3db189fd47e 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -4,9 +4,15 @@ # # make upgrade # -deepdiff==3.1.0 # via -r requirements/elasticsearch.in -elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -importlib-metadata==1.7.0 # via jsonpickle -jsonpickle==1.4.1 # via deepdiff -urllib3==1.25.10 # via elasticsearch -zipp==1.2.0 # via importlib-metadata +deepdiff==3.1.0 + # via -r requirements/elasticsearch.in +elasticsearch==0.4.5 + # via -r requirements/elasticsearch.in +importlib-metadata==2.1.1 + # via jsonpickle +jsonpickle==1.4.2 + # via deepdiff +urllib3==1.26.2 + # via elasticsearch +zipp==1.2.0 + # via importlib-metadata diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index d11418c97c4..47272aa7795 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,8 +4,13 @@ # # make upgrade # -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -idna==2.7 # via requests -requests==2.20.0 # via -r requirements/cloudflare.in -urllib3==1.24.3 # via requests +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via requests +idna==2.7 + # via requests +requests==2.20.0 + # via -r requirements/cloudflare.in +urllib3==1.24.3 + # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 51ff9793bfc..38b66423f4f 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -26,7 +26,7 @@ pyasn1==0.4.8 # via rsa PyMySQL==0.9.3 python-dateutil==2.8.1 # via botocore, opsgenie-sdk pytz==2019.3 # via celery, opsgenie-sdk -pyyaml==3.12 # via awscli +pyyaml==5.3.1 # via awscli redis==2.10.6 requests==2.22.0 # via opsgenie-sdk rsa==3.4.2 # via awscli diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 97401952c7a..2751140fcdb 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,11 +4,19 @@ # # make upgrade # -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -click==6.7 # via -r requirements/pingdom.in -idna==2.7 # via requests -pyyaml==3.12 # via -r requirements/pingdom.in -requests==2.20.0 # via -r requirements/pingdom.in -six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.24.3 # via requests +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via requests +click==6.7 + # via -r requirements/pingdom.in +idna==2.7 + # via requests +pyyaml==5.3.1 + # via -r requirements/pingdom.in +requests==2.20.0 + # via -r requirements/pingdom.in +six==1.14.0 + # via -r requirements/pingdom.in +urllib3==1.24.3 + # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 0ebab3137d8..c76892b8b2f 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -4,10 +4,17 @@ # # make upgrade # -boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==2.10 # via requests -requests==2.24.0 # via -r requirements/vpc-tools.in -urllib3==1.25.10 # via requests +boto==2.49.0 + # via -r requirements/vpc-tools.in +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +docopt==0.6.2 + # via -r requirements/vpc-tools.in +idna==2.10 + # via requests +requests==2.25.1 + # via -r requirements/vpc-tools.in +urllib3==1.26.2 + # via requests From 985b3cd9b0d6652dea4f6ff4319bd5dd978581b1 Mon Sep 17 00:00:00 2001 From: DawoudSheraz Date: Tue, 16 Mar 2021 17:20:43 +0500 Subject: [PATCH 013/664] re-run make upgrade --- .../roles/aws/templates/requirements.txt.j2 | 14 +++++++------- requirements.txt | 19 +++++++++---------- requirements/pip-tools.txt | 6 +++++- util/elasticsearch/requirements.txt | 8 ++------ util/pingdom/requirements.txt | 2 +- util/vpc-tools/requirements.txt | 2 +- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 9f9fe54cbcb..b3ab13fd20d 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.18.207 +awscli==1.19.28 # via -r requirements/aws.in -boto3==1.16.47 +boto3==1.17.28 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -botocore==1.19.47 +botocore==1.20.28 # via # awscli # boto3 @@ -29,9 +29,9 @@ python-dateutil==2.8.1 # via # botocore # s3cmd -python-magic==0.4.18 +python-magic==0.4.22 # via s3cmd -pyyaml==5.3.1 +pyyaml==5.4.1 # via # -r requirements/aws.in # awscli @@ -39,11 +39,11 @@ rsa==4.5 # via awscli s3cmd==2.1.0 # via -r requirements/aws.in -s3transfer==0.3.3 +s3transfer==0.3.4 # via # awscli # boto3 six==1.15.0 # via python-dateutil -urllib3==1.26.2 +urllib3==1.26.4 # via botocore diff --git a/requirements.txt b/requirements.txt index 1806ca26b7f..50e9d166aa1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,26 +4,26 @@ # # make upgrade # -ansible==2.8.18 +ansible==2.8.19 # via -r requirements/base.in -awscli==1.18.207 +awscli==1.19.28 # via -r requirements/base.in bcrypt==3.1.7 # via # -c requirements/constraints.txt # paramiko -boto3==1.16.47 +boto3==1.17.28 # via -r requirements/base.in boto==2.49.0 # via -r requirements/base.in -botocore==1.19.47 +botocore==1.20.28 # via # awscli # boto3 # s3transfer certifi==2020.12.5 # via requests -cffi==1.14.4 +cffi==1.14.5 # via # bcrypt # cryptography @@ -32,7 +32,7 @@ chardet==4.0.0 # via requests colorama==0.4.3 # via awscli -cryptography==3.2.1 +cryptography==3.4.6 # via # ansible # paramiko @@ -84,7 +84,7 @@ pynacl==1.4.0 # via paramiko python-dateutil==2.8.1 # via botocore -pyyaml==5.3.1 +pyyaml==5.4.1 # via # -r requirements/base.in # ansible @@ -95,18 +95,17 @@ requests==2.25.1 # datadog rsa==4.5 # via awscli -s3transfer==0.3.3 +s3transfer==0.3.4 # via # awscli # boto3 six==1.15.0 # via # bcrypt - # cryptography # pathlib2 # pynacl # python-dateutil -urllib3==1.26.2 +urllib3==1.26.4 # via # botocore # requests diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index aa6ffb82da2..5957365f77e 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -6,8 +6,12 @@ # click==7.1.2 # via pip-tools -pip-tools==5.5.0 +pep517==0.10.0 + # via pip-tools +pip-tools==6.0.1 # via -r requirements/pip-tools.in +toml==0.10.2 + # via pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 3db189fd47e..b9b47e8ab9e 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -8,11 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -importlib-metadata==2.1.1 - # via jsonpickle -jsonpickle==1.4.2 +jsonpickle==2.0.0 # via deepdiff -urllib3==1.26.2 +urllib3==1.26.4 # via elasticsearch -zipp==1.2.0 - # via importlib-metadata diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 2751140fcdb..761793f7930 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -12,7 +12,7 @@ click==6.7 # via -r requirements/pingdom.in idna==2.7 # via requests -pyyaml==5.3.1 +pyyaml==5.4.1 # via -r requirements/pingdom.in requests==2.20.0 # via -r requirements/pingdom.in diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index c76892b8b2f..e5b55226d8b 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -16,5 +16,5 @@ idna==2.10 # via requests requests==2.25.1 # via -r requirements/vpc-tools.in -urllib3==1.26.2 +urllib3==1.26.4 # via requests From 395626be7b82562cf0cf4313bf407e45d8995d35 Mon Sep 17 00:00:00 2001 From: DawoudSheraz Date: Tue, 16 Mar 2021 17:32:08 +0500 Subject: [PATCH 014/664] python35 make upgrade --- playbooks/roles/aws/templates/requirements.txt.j2 | 8 ++++---- requirements.txt | 11 ++++++----- requirements/pip-tools.txt | 6 +----- util/elasticsearch/requirements.txt | 4 ++++ util/pingdom/requirements.txt | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index b3ab13fd20d..cae4e7d7a76 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.19.28 +awscli==1.18.223 # via -r requirements/aws.in -boto3==1.17.28 +boto3==1.16.63 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -botocore==1.20.28 +botocore==1.19.63 # via # awscli # boto3 @@ -31,7 +31,7 @@ python-dateutil==2.8.1 # s3cmd python-magic==0.4.22 # via s3cmd -pyyaml==5.4.1 +pyyaml==5.3.1 # via # -r requirements/aws.in # awscli diff --git a/requirements.txt b/requirements.txt index 50e9d166aa1..9f45941645e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,17 +6,17 @@ # ansible==2.8.19 # via -r requirements/base.in -awscli==1.19.28 +awscli==1.18.223 # via -r requirements/base.in bcrypt==3.1.7 # via # -c requirements/constraints.txt # paramiko -boto3==1.17.28 +boto3==1.16.63 # via -r requirements/base.in boto==2.49.0 # via -r requirements/base.in -botocore==1.20.28 +botocore==1.19.63 # via # awscli # boto3 @@ -32,7 +32,7 @@ chardet==4.0.0 # via requests colorama==0.4.3 # via awscli -cryptography==3.4.6 +cryptography==3.2.1 # via # ansible # paramiko @@ -84,7 +84,7 @@ pynacl==1.4.0 # via paramiko python-dateutil==2.8.1 # via botocore -pyyaml==5.4.1 +pyyaml==5.3.1 # via # -r requirements/base.in # ansible @@ -102,6 +102,7 @@ s3transfer==0.3.4 six==1.15.0 # via # bcrypt + # cryptography # pathlib2 # pynacl # python-dateutil diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 5957365f77e..aa6ffb82da2 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -6,12 +6,8 @@ # click==7.1.2 # via pip-tools -pep517==0.10.0 - # via pip-tools -pip-tools==6.0.1 +pip-tools==5.5.0 # via -r requirements/pip-tools.in -toml==0.10.2 - # via pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index b9b47e8ab9e..754191b2b89 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -8,7 +8,11 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in +importlib-metadata==2.1.1 + # via jsonpickle jsonpickle==2.0.0 # via deepdiff urllib3==1.26.4 # via elasticsearch +zipp==1.2.0 + # via importlib-metadata diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 761793f7930..2751140fcdb 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -12,7 +12,7 @@ click==6.7 # via -r requirements/pingdom.in idna==2.7 # via requests -pyyaml==5.4.1 +pyyaml==5.3.1 # via -r requirements/pingdom.in requests==2.20.0 # via -r requirements/pingdom.in From f731eeea6c448e50b27950bc195332b1db0826cd Mon Sep 17 00:00:00 2001 From: Fred Smith Date: Tue, 13 Apr 2021 15:11:35 -0400 Subject: [PATCH 015/664] Revert "update pyyaml " --- .../roles/aws/templates/requirements.txt.j2 | 59 ++----- requirements.txt | 146 +++++------------- requirements/aws.in | 3 +- requirements/base.in | 8 +- requirements/pingdom.in | 2 +- requirements/pip-tools.txt | 7 +- util/elasticsearch/requirements.txt | 18 +-- util/jenkins/requirements-cloudflare.txt | 15 +- util/jenkins/requirements.txt | 2 +- util/pingdom/requirements.txt | 24 +-- util/vpc-tools/requirements.txt | 21 +-- 11 files changed, 93 insertions(+), 212 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index cae4e7d7a76..918ec04b78e 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,46 +4,19 @@ # # make upgrade # -awscli==1.18.223 - # via -r requirements/aws.in -boto3==1.16.63 - # via -r requirements/aws.in -boto==2.49.0 - # via -r requirements/aws.in -botocore==1.19.63 - # via - # awscli - # boto3 - # s3transfer -colorama==0.4.3 - # via awscli -docutils==0.15.2 - # via awscli -jmespath==0.10.0 - # via - # boto3 - # botocore -pyasn1==0.4.8 - # via rsa -python-dateutil==2.8.1 - # via - # botocore - # s3cmd -python-magic==0.4.22 - # via s3cmd -pyyaml==5.3.1 - # via - # -r requirements/aws.in - # awscli -rsa==4.5 - # via awscli -s3cmd==2.1.0 - # via -r requirements/aws.in -s3transfer==0.3.4 - # via - # awscli - # boto3 -six==1.15.0 - # via python-dateutil -urllib3==1.26.4 - # via botocore +awscli==1.18.129 # via -r requirements/aws.in +boto3==1.14.52 # via -r requirements/aws.in +boto==2.49.0 # via -r requirements/aws.in +botocore==1.17.52 # via awscli, boto3, s3transfer +colorama==0.4.3 # via awscli +docutils==0.15.2 # via awscli, botocore +jmespath==0.10.0 # via boto3, botocore +pyasn1==0.4.8 # via rsa +python-dateutil==2.8.1 # via botocore, s3cmd +python-magic==0.4.18 # via s3cmd +pyyaml==3.11 # via -r requirements/aws.in, awscli +rsa==4.5 # via awscli +s3cmd==2.1.0 # via -r requirements/aws.in +s3transfer==0.3.3 # via awscli, boto3 +six==1.15.0 # via python-dateutil +urllib3==1.25.10 # via botocore diff --git a/requirements.txt b/requirements.txt index 9f45941645e..63aa26abba4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,109 +4,43 @@ # # make upgrade # -ansible==2.8.19 - # via -r requirements/base.in -awscli==1.18.223 - # via -r requirements/base.in -bcrypt==3.1.7 - # via - # -c requirements/constraints.txt - # paramiko -boto3==1.16.63 - # via -r requirements/base.in -boto==2.49.0 - # via -r requirements/base.in -botocore==1.19.63 - # via - # awscli - # boto3 - # s3transfer -certifi==2020.12.5 - # via requests -cffi==1.14.5 - # via - # bcrypt - # cryptography - # pynacl -chardet==4.0.0 - # via requests -colorama==0.4.3 - # via awscli -cryptography==3.2.1 - # via - # ansible - # paramiko -datadog==0.8.0 - # via -r requirements/base.in -decorator==4.4.2 - # via - # datadog - # networkx -docopt==0.6.2 - # via -r requirements/base.in -docutils==0.15.2 - # via awscli -ecdsa==0.13.3 - # via -r requirements/base.in -idna==2.10 - # via requests -jinja2==2.8 - # via - # -r requirements/base.in - # ansible -jmespath==0.10.0 - # via - # boto3 - # botocore -markupsafe==1.1.1 - # via jinja2 -mysqlclient==1.4.6 - # via -r requirements/base.in -networkx==1.11 - # via -r requirements/base.in -paramiko==2.4.2 - # via -r requirements/base.in -pathlib2==2.3.0 - # via -r requirements/base.in -prettytable==0.7.2 - # via -r requirements/base.in -pyasn1==0.4.8 - # via - # paramiko - # rsa -pycparser==2.20 - # via cffi -pycrypto==2.6.1 - # via -r requirements/base.in -pymongo==3.9.0 - # via -r requirements/base.in -pynacl==1.4.0 - # via paramiko -python-dateutil==2.8.1 - # via botocore -pyyaml==5.3.1 - # via - # -r requirements/base.in - # ansible - # awscli -requests==2.25.1 - # via - # -r requirements/base.in - # datadog -rsa==4.5 - # via awscli -s3transfer==0.3.4 - # via - # awscli - # boto3 -six==1.15.0 - # via - # bcrypt - # cryptography - # pathlib2 - # pynacl - # python-dateutil -urllib3==1.26.4 - # via - # botocore - # requests +ansible==2.8.15 # via -r requirements/base.in +awscli==1.16.309 # via -r requirements/base.in +bcrypt==3.1.7 # via -c requirements/constraints.txt, paramiko +boto3==1.10.45 # via -r requirements/base.in +boto==2.48.0 # via -r requirements/base.in +botocore==1.13.45 # via awscli, boto3, s3transfer +certifi==2020.6.20 # via requests +cffi==1.14.2 # via bcrypt, cryptography, pynacl +chardet==3.0.4 # via requests +colorama==0.4.1 # via awscli +cryptography==3.1 # via ansible, paramiko +datadog==0.8.0 # via -r requirements/base.in +decorator==4.4.2 # via datadog, networkx +docopt==0.6.2 # via -r requirements/base.in +docutils==0.15.2 # via awscli, botocore +ecdsa==0.13.3 # via -r requirements/base.in +idna==2.7 # via requests +jinja2==2.8 # via -r requirements/base.in, ansible +jmespath==0.10.0 # via boto3, botocore +markupsafe==1.1.1 # via jinja2 +mysqlclient==1.4.6 # via -r requirements/base.in +networkx==1.11 # via -r requirements/base.in +paramiko==2.4.2 # via -r requirements/base.in +pathlib2==2.3.0 # via -r requirements/base.in +prettytable==0.7.2 # via -r requirements/base.in +pyasn1==0.4.8 # via paramiko, rsa +pycparser==2.20 # via cffi +pycrypto==2.6.1 # via -r requirements/base.in +pymongo==3.9.0 # via -r requirements/base.in +pynacl==1.4.0 # via paramiko +python-dateutil==2.8.1 # via botocore +pyyaml==5.2 # via -r requirements/base.in, ansible, awscli +requests==2.20.0 # via -r requirements/base.in, datadog +rsa==3.4.2 # via awscli +s3transfer==0.2.1 # via awscli, boto3 +six==1.15.0 # via bcrypt, cryptography, pathlib2, pynacl, python-dateutil +urllib3==1.24.3 # via botocore, requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/aws.in b/requirements/aws.in index e457993d02a..b3bcd6a2ead 100644 --- a/requirements/aws.in +++ b/requirements/aws.in @@ -5,4 +5,5 @@ boto boto3 futures ; python_version == "2.7" # via s3transfer s3cmd -pyyaml +# Lock pyYaml to what's installed on Ubuntu +pyyaml==3.11 diff --git a/requirements/base.in b/requirements/base.in index ab61bceb885..36dcc17bd79 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,9 +2,9 @@ # Standard dependencies for Ansible runs ansible<2.9.0 -awscli -boto -boto3 +awscli==1.16.309 +boto==2.48.0 +boto3==1.10.45 datadog==0.8.0 docopt==0.6.2 ecdsa==0.13.3 @@ -17,4 +17,4 @@ prettytable==0.7.2 pycrypto==2.6.1 pymongo==3.9.0 # Needed for the mongo_* modules (playbooks/library/mongo_*) PyYAML -requests +requests==2.20.0 diff --git a/requirements/pingdom.in b/requirements/pingdom.in index e9ff57f0095..960ea707771 100644 --- a/requirements/pingdom.in +++ b/requirements/pingdom.in @@ -1,6 +1,6 @@ # Requirements for util/pingdom/create_pingdom_alerts.py click==6.7 -PyYAML +PyYAML==3.12 requests==2.20.0 six==1.14.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index aa6ffb82da2..4b50eae7bd7 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,10 +4,9 @@ # # make upgrade # -click==7.1.2 - # via pip-tools -pip-tools==5.5.0 - # via -r requirements/pip-tools.in +click==7.1.2 # via pip-tools +pip-tools==5.3.1 # via -r requirements/pip-tools.in +six==1.15.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 754191b2b89..a77725c6ffe 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -4,15 +4,9 @@ # # make upgrade # -deepdiff==3.1.0 - # via -r requirements/elasticsearch.in -elasticsearch==0.4.5 - # via -r requirements/elasticsearch.in -importlib-metadata==2.1.1 - # via jsonpickle -jsonpickle==2.0.0 - # via deepdiff -urllib3==1.26.4 - # via elasticsearch -zipp==1.2.0 - # via importlib-metadata +deepdiff==3.1.0 # via -r requirements/elasticsearch.in +elasticsearch==0.4.5 # via -r requirements/elasticsearch.in +importlib-metadata==1.7.0 # via jsonpickle +jsonpickle==1.4.1 # via deepdiff +urllib3==1.25.10 # via elasticsearch +zipp==1.2.0 # via importlib-metadata diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 47272aa7795..d11418c97c4 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,13 +4,8 @@ # # make upgrade # -certifi==2020.12.5 - # via requests -chardet==3.0.4 - # via requests -idna==2.7 - # via requests -requests==2.20.0 - # via -r requirements/cloudflare.in -urllib3==1.24.3 - # via requests +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +idna==2.7 # via requests +requests==2.20.0 # via -r requirements/cloudflare.in +urllib3==1.24.3 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 38b66423f4f..51ff9793bfc 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -26,7 +26,7 @@ pyasn1==0.4.8 # via rsa PyMySQL==0.9.3 python-dateutil==2.8.1 # via botocore, opsgenie-sdk pytz==2019.3 # via celery, opsgenie-sdk -pyyaml==5.3.1 # via awscli +pyyaml==3.12 # via awscli redis==2.10.6 requests==2.22.0 # via opsgenie-sdk rsa==3.4.2 # via awscli diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 2751140fcdb..97401952c7a 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,19 +4,11 @@ # # make upgrade # -certifi==2020.12.5 - # via requests -chardet==3.0.4 - # via requests -click==6.7 - # via -r requirements/pingdom.in -idna==2.7 - # via requests -pyyaml==5.3.1 - # via -r requirements/pingdom.in -requests==2.20.0 - # via -r requirements/pingdom.in -six==1.14.0 - # via -r requirements/pingdom.in -urllib3==1.24.3 - # via requests +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +click==6.7 # via -r requirements/pingdom.in +idna==2.7 # via requests +pyyaml==3.12 # via -r requirements/pingdom.in +requests==2.20.0 # via -r requirements/pingdom.in +six==1.14.0 # via -r requirements/pingdom.in +urllib3==1.24.3 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index e5b55226d8b..0ebab3137d8 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -4,17 +4,10 @@ # # make upgrade # -boto==2.49.0 - # via -r requirements/vpc-tools.in -certifi==2020.12.5 - # via requests -chardet==4.0.0 - # via requests -docopt==0.6.2 - # via -r requirements/vpc-tools.in -idna==2.10 - # via requests -requests==2.25.1 - # via -r requirements/vpc-tools.in -urllib3==1.26.4 - # via requests +boto==2.49.0 # via -r requirements/vpc-tools.in +certifi==2020.6.20 # via requests +chardet==3.0.4 # via requests +docopt==0.6.2 # via -r requirements/vpc-tools.in +idna==2.10 # via requests +requests==2.24.0 # via -r requirements/vpc-tools.in +urllib3==1.25.10 # via requests From 8848d85e664d9ff2e21a88ada25288745eedb04d Mon Sep 17 00:00:00 2001 From: Fred Smith Date: Tue, 13 Apr 2021 15:15:05 -0400 Subject: [PATCH 016/664] Revert "Revert "update pyyaml "" --- .../roles/aws/templates/requirements.txt.j2 | 59 +++++-- requirements.txt | 146 +++++++++++++----- requirements/aws.in | 3 +- requirements/base.in | 8 +- requirements/pingdom.in | 2 +- requirements/pip-tools.txt | 7 +- util/elasticsearch/requirements.txt | 18 ++- util/jenkins/requirements-cloudflare.txt | 15 +- util/jenkins/requirements.txt | 2 +- util/pingdom/requirements.txt | 24 ++- util/vpc-tools/requirements.txt | 21 ++- 11 files changed, 212 insertions(+), 93 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 918ec04b78e..cae4e7d7a76 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,19 +4,46 @@ # # make upgrade # -awscli==1.18.129 # via -r requirements/aws.in -boto3==1.14.52 # via -r requirements/aws.in -boto==2.49.0 # via -r requirements/aws.in -botocore==1.17.52 # via awscli, boto3, s3transfer -colorama==0.4.3 # via awscli -docutils==0.15.2 # via awscli, botocore -jmespath==0.10.0 # via boto3, botocore -pyasn1==0.4.8 # via rsa -python-dateutil==2.8.1 # via botocore, s3cmd -python-magic==0.4.18 # via s3cmd -pyyaml==3.11 # via -r requirements/aws.in, awscli -rsa==4.5 # via awscli -s3cmd==2.1.0 # via -r requirements/aws.in -s3transfer==0.3.3 # via awscli, boto3 -six==1.15.0 # via python-dateutil -urllib3==1.25.10 # via botocore +awscli==1.18.223 + # via -r requirements/aws.in +boto3==1.16.63 + # via -r requirements/aws.in +boto==2.49.0 + # via -r requirements/aws.in +botocore==1.19.63 + # via + # awscli + # boto3 + # s3transfer +colorama==0.4.3 + # via awscli +docutils==0.15.2 + # via awscli +jmespath==0.10.0 + # via + # boto3 + # botocore +pyasn1==0.4.8 + # via rsa +python-dateutil==2.8.1 + # via + # botocore + # s3cmd +python-magic==0.4.22 + # via s3cmd +pyyaml==5.3.1 + # via + # -r requirements/aws.in + # awscli +rsa==4.5 + # via awscli +s3cmd==2.1.0 + # via -r requirements/aws.in +s3transfer==0.3.4 + # via + # awscli + # boto3 +six==1.15.0 + # via python-dateutil +urllib3==1.26.4 + # via botocore diff --git a/requirements.txt b/requirements.txt index 63aa26abba4..9f45941645e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,43 +4,109 @@ # # make upgrade # -ansible==2.8.15 # via -r requirements/base.in -awscli==1.16.309 # via -r requirements/base.in -bcrypt==3.1.7 # via -c requirements/constraints.txt, paramiko -boto3==1.10.45 # via -r requirements/base.in -boto==2.48.0 # via -r requirements/base.in -botocore==1.13.45 # via awscli, boto3, s3transfer -certifi==2020.6.20 # via requests -cffi==1.14.2 # via bcrypt, cryptography, pynacl -chardet==3.0.4 # via requests -colorama==0.4.1 # via awscli -cryptography==3.1 # via ansible, paramiko -datadog==0.8.0 # via -r requirements/base.in -decorator==4.4.2 # via datadog, networkx -docopt==0.6.2 # via -r requirements/base.in -docutils==0.15.2 # via awscli, botocore -ecdsa==0.13.3 # via -r requirements/base.in -idna==2.7 # via requests -jinja2==2.8 # via -r requirements/base.in, ansible -jmespath==0.10.0 # via boto3, botocore -markupsafe==1.1.1 # via jinja2 -mysqlclient==1.4.6 # via -r requirements/base.in -networkx==1.11 # via -r requirements/base.in -paramiko==2.4.2 # via -r requirements/base.in -pathlib2==2.3.0 # via -r requirements/base.in -prettytable==0.7.2 # via -r requirements/base.in -pyasn1==0.4.8 # via paramiko, rsa -pycparser==2.20 # via cffi -pycrypto==2.6.1 # via -r requirements/base.in -pymongo==3.9.0 # via -r requirements/base.in -pynacl==1.4.0 # via paramiko -python-dateutil==2.8.1 # via botocore -pyyaml==5.2 # via -r requirements/base.in, ansible, awscli -requests==2.20.0 # via -r requirements/base.in, datadog -rsa==3.4.2 # via awscli -s3transfer==0.2.1 # via awscli, boto3 -six==1.15.0 # via bcrypt, cryptography, pathlib2, pynacl, python-dateutil -urllib3==1.24.3 # via botocore, requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +ansible==2.8.19 + # via -r requirements/base.in +awscli==1.18.223 + # via -r requirements/base.in +bcrypt==3.1.7 + # via + # -c requirements/constraints.txt + # paramiko +boto3==1.16.63 + # via -r requirements/base.in +boto==2.49.0 + # via -r requirements/base.in +botocore==1.19.63 + # via + # awscli + # boto3 + # s3transfer +certifi==2020.12.5 + # via requests +cffi==1.14.5 + # via + # bcrypt + # cryptography + # pynacl +chardet==4.0.0 + # via requests +colorama==0.4.3 + # via awscli +cryptography==3.2.1 + # via + # ansible + # paramiko +datadog==0.8.0 + # via -r requirements/base.in +decorator==4.4.2 + # via + # datadog + # networkx +docopt==0.6.2 + # via -r requirements/base.in +docutils==0.15.2 + # via awscli +ecdsa==0.13.3 + # via -r requirements/base.in +idna==2.10 + # via requests +jinja2==2.8 + # via + # -r requirements/base.in + # ansible +jmespath==0.10.0 + # via + # boto3 + # botocore +markupsafe==1.1.1 + # via jinja2 +mysqlclient==1.4.6 + # via -r requirements/base.in +networkx==1.11 + # via -r requirements/base.in +paramiko==2.4.2 + # via -r requirements/base.in +pathlib2==2.3.0 + # via -r requirements/base.in +prettytable==0.7.2 + # via -r requirements/base.in +pyasn1==0.4.8 + # via + # paramiko + # rsa +pycparser==2.20 + # via cffi +pycrypto==2.6.1 + # via -r requirements/base.in +pymongo==3.9.0 + # via -r requirements/base.in +pynacl==1.4.0 + # via paramiko +python-dateutil==2.8.1 + # via botocore +pyyaml==5.3.1 + # via + # -r requirements/base.in + # ansible + # awscli +requests==2.25.1 + # via + # -r requirements/base.in + # datadog +rsa==4.5 + # via awscli +s3transfer==0.3.4 + # via + # awscli + # boto3 +six==1.15.0 + # via + # bcrypt + # cryptography + # pathlib2 + # pynacl + # python-dateutil +urllib3==1.26.4 + # via + # botocore + # requests diff --git a/requirements/aws.in b/requirements/aws.in index b3bcd6a2ead..e457993d02a 100644 --- a/requirements/aws.in +++ b/requirements/aws.in @@ -5,5 +5,4 @@ boto boto3 futures ; python_version == "2.7" # via s3transfer s3cmd -# Lock pyYaml to what's installed on Ubuntu -pyyaml==3.11 +pyyaml diff --git a/requirements/base.in b/requirements/base.in index 36dcc17bd79..ab61bceb885 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,9 +2,9 @@ # Standard dependencies for Ansible runs ansible<2.9.0 -awscli==1.16.309 -boto==2.48.0 -boto3==1.10.45 +awscli +boto +boto3 datadog==0.8.0 docopt==0.6.2 ecdsa==0.13.3 @@ -17,4 +17,4 @@ prettytable==0.7.2 pycrypto==2.6.1 pymongo==3.9.0 # Needed for the mongo_* modules (playbooks/library/mongo_*) PyYAML -requests==2.20.0 +requests diff --git a/requirements/pingdom.in b/requirements/pingdom.in index 960ea707771..e9ff57f0095 100644 --- a/requirements/pingdom.in +++ b/requirements/pingdom.in @@ -1,6 +1,6 @@ # Requirements for util/pingdom/create_pingdom_alerts.py click==6.7 -PyYAML==3.12 +PyYAML requests==2.20.0 six==1.14.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 4b50eae7bd7..aa6ffb82da2 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,9 +4,10 @@ # # make upgrade # -click==7.1.2 # via pip-tools -pip-tools==5.3.1 # via -r requirements/pip-tools.in -six==1.15.0 # via pip-tools +click==7.1.2 + # via pip-tools +pip-tools==5.5.0 + # via -r requirements/pip-tools.in # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index a77725c6ffe..754191b2b89 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -4,9 +4,15 @@ # # make upgrade # -deepdiff==3.1.0 # via -r requirements/elasticsearch.in -elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -importlib-metadata==1.7.0 # via jsonpickle -jsonpickle==1.4.1 # via deepdiff -urllib3==1.25.10 # via elasticsearch -zipp==1.2.0 # via importlib-metadata +deepdiff==3.1.0 + # via -r requirements/elasticsearch.in +elasticsearch==0.4.5 + # via -r requirements/elasticsearch.in +importlib-metadata==2.1.1 + # via jsonpickle +jsonpickle==2.0.0 + # via deepdiff +urllib3==1.26.4 + # via elasticsearch +zipp==1.2.0 + # via importlib-metadata diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index d11418c97c4..47272aa7795 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,8 +4,13 @@ # # make upgrade # -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -idna==2.7 # via requests -requests==2.20.0 # via -r requirements/cloudflare.in -urllib3==1.24.3 # via requests +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via requests +idna==2.7 + # via requests +requests==2.20.0 + # via -r requirements/cloudflare.in +urllib3==1.24.3 + # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 51ff9793bfc..38b66423f4f 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -26,7 +26,7 @@ pyasn1==0.4.8 # via rsa PyMySQL==0.9.3 python-dateutil==2.8.1 # via botocore, opsgenie-sdk pytz==2019.3 # via celery, opsgenie-sdk -pyyaml==3.12 # via awscli +pyyaml==5.3.1 # via awscli redis==2.10.6 requests==2.22.0 # via opsgenie-sdk rsa==3.4.2 # via awscli diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 97401952c7a..2751140fcdb 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,11 +4,19 @@ # # make upgrade # -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -click==6.7 # via -r requirements/pingdom.in -idna==2.7 # via requests -pyyaml==3.12 # via -r requirements/pingdom.in -requests==2.20.0 # via -r requirements/pingdom.in -six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.24.3 # via requests +certifi==2020.12.5 + # via requests +chardet==3.0.4 + # via requests +click==6.7 + # via -r requirements/pingdom.in +idna==2.7 + # via requests +pyyaml==5.3.1 + # via -r requirements/pingdom.in +requests==2.20.0 + # via -r requirements/pingdom.in +six==1.14.0 + # via -r requirements/pingdom.in +urllib3==1.24.3 + # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 0ebab3137d8..e5b55226d8b 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -4,10 +4,17 @@ # # make upgrade # -boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2020.6.20 # via requests -chardet==3.0.4 # via requests -docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==2.10 # via requests -requests==2.24.0 # via -r requirements/vpc-tools.in -urllib3==1.25.10 # via requests +boto==2.49.0 + # via -r requirements/vpc-tools.in +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +docopt==0.6.2 + # via -r requirements/vpc-tools.in +idna==2.10 + # via requests +requests==2.25.1 + # via -r requirements/vpc-tools.in +urllib3==1.26.4 + # via requests From 98c314a50d1a537271a83fa46e4579642663357e Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 14 Apr 2021 11:32:32 +0500 Subject: [PATCH 017/664] fix pyyaml in jenkins requirement --- util/jenkins/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 38b66423f4f..51ff9793bfc 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -26,7 +26,7 @@ pyasn1==0.4.8 # via rsa PyMySQL==0.9.3 python-dateutil==2.8.1 # via botocore, opsgenie-sdk pytz==2019.3 # via celery, opsgenie-sdk -pyyaml==5.3.1 # via awscli +pyyaml==3.12 # via awscli redis==2.10.6 requests==2.22.0 # via opsgenie-sdk rsa==3.4.2 # via awscli From 0723826cb06a82da7a8149d9bb3eea396299f3b5 Mon Sep 17 00:00:00 2001 From: Waheed Ahmed Date: Wed, 14 Apr 2021 16:51:21 +0500 Subject: [PATCH 018/664] Set site name for authn MFE. --- playbooks/roles/authn/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/authn/tasks/main.yml b/playbooks/roles/authn/tasks/main.yml index e40c77658a2..b5ea186d721 100644 --- a/playbooks/roles/authn/tasks/main.yml +++ b/playbooks/roles/authn/tasks/main.yml @@ -4,3 +4,4 @@ vars: MFE_NAME: authn MFE_VERSION: '{{ AUTHN_MFE_VERSION }}' + MFE_SITE_NAME: 'Your Platform Name Here' From 3832eaae224e04e75f92282db7e097083ecaa436 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Fri, 9 Apr 2021 17:45:24 -0300 Subject: [PATCH 019/664] fix: allow mfe_deployer to run The `mfe` role requires `COMMON_ECOMMERCE_BASE_URL` to be set, but that variable is not defined in `common_vars`, `ecommerce`, or anywhere else. Instead of introducing possibly breaking changes in the `ecommerce` or `mfe` roles, simply set it to the default `ECOMMERCE_ECOMMERCE_URL_ROOT` here so that the native playbook can complete successfully. Cherry-picked from commit 9991f92b397b7c2d19a3900e1837aad28e0fabe5 --- playbooks/openedx_native.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index bf6740bcee9..4dd5bac10a0 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -39,6 +39,8 @@ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false EDXAPP_ENABLE_MEMCACHE: true EDXAPP_ENABLE_ELASTIC_SEARCH: true + # For the mfe role. + COMMON_ECOMMERCE_BASE_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}' roles: - role: swapfile SWAPFILE_SIZE: 4GB From d9289b8767d7cab8028c341fa240e6bd5386a698 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Thu, 15 Apr 2021 12:38:51 -0400 Subject: [PATCH 020/664] feat: Add timezones to DE jenkins console log timestamps This has frustrated me for literally years. Timestamps should either be in UTC, or clearly show the timezone, plain and simple. --- playbooks/roles/jenkins_data_engineering/defaults/main.yml | 3 +++ playbooks/roles/jenkins_data_engineering/meta/main.yml | 1 + playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 3 +++ playbooks/roles/jenkins_data_engineering_new/meta/main.yml | 1 + 4 files changed, 8 insertions(+) diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index 6b5df57d6f6..860fbf9c378 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -249,3 +249,6 @@ jenkins_connection_delay: 1 jenkins_private_keyfile: "{{ jenkins_user_home }}/.ssh/id_rsa" jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" + +# Be clear about which time zone the console log timestamps are in!!! +de_jenkins_timestamper_system_time: "''HH:mm:ssXX' '" diff --git a/playbooks/roles/jenkins_data_engineering/meta/main.yml b/playbooks/roles/jenkins_data_engineering/meta/main.yml index 4599ab36feb..72f1aee7772 100644 --- a/playbooks/roles/jenkins_data_engineering/meta/main.yml +++ b/playbooks/roles/jenkins_data_engineering/meta/main.yml @@ -81,5 +81,6 @@ dependencies: - hudson.scm.SCM.Tag USERS: '{{ JENKINS_DATA_ENGINEERING_AUTH_ADMINISTRATORS }}' jenkins_common_main_env_vars: '{{ jenkins_base_environment_variables }} + {{ jenkins_additional_environment_variables }}' + jenkins_common_timestamper_system_clock_format: '{{ de_jenkins_timestamper_system_time }}' - role: mongo_client diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 36e5d73048c..02ed9e9220c 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -286,3 +286,6 @@ jenkins_connection_delay: 1 jenkins_private_keyfile: "{{ jenkins_user_home }}/.ssh/id_rsa" jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" + +# Be clear about which time zone the console log timestamps are in!!! +de_jenkins_timestamper_system_time: "''HH:mm:ssXX' '" diff --git a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml index 6dd370fea23..8de3b56e1ef 100644 --- a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml @@ -50,3 +50,4 @@ dependencies: jenkins_common_python_versions: '{{ de_jenkins_python_versions }}' jenkins_common_python_installations: '{{ de_jenkins_python_installations }}' jenkins_common_snap_pkgs: '{{ de_jenkins_snap_pkgs }}' + jenkins_common_timestamper_system_clock_format: '{{ de_jenkins_timestamper_system_time }}' From e0f937132b419d2ef7c7767685e5cc1381a93618 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 16 Apr 2021 17:39:10 +0500 Subject: [PATCH 021/664] add util/jenkins/requirements.txt to make upgrade --- Makefile | 4 +++- requirements/jenkins.in | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 requirements/jenkins.in diff --git a/Makefile b/Makefile index 3d89474c8be..c642ea22cd8 100755 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ upgrade: ## update the pip requirements files to use the latest releases satisfy pip-compile --upgrade -o util/jenkins/requirements-cloudflare.txt requirements/cloudflare.in pip-compile --upgrade -o util/pingdom/requirements.txt requirements/pingdom.in pip-compile --upgrade -o util/vpc-tools/requirements.txt requirements/vpc-tools.in + pip-compile --upgrade -o util/jenkins/requirements.txt requirements/jenkins.in # Post process all of the files generated above to work around open pip-tools issues util/post-pip-compile.sh \ requirements/pip-tools.txt \ @@ -40,6 +41,7 @@ upgrade: ## update the pip requirements files to use the latest releases satisfy util/elasticsearch/requirements.txt \ util/jenkins/requirements-cloudflare.txt \ util/pingdom/requirements.txt \ - util/vpc-tools/requirements.txt + util/vpc-tools/requirements.txt \ + util/jenkins/requirements.txt include *.mk diff --git a/requirements/jenkins.in b/requirements/jenkins.in new file mode 100644 index 00000000000..cee5a2e27c6 --- /dev/null +++ b/requirements/jenkins.in @@ -0,0 +1,15 @@ +# Python dependencies for the util/jenkins/requirements.txt + +awscli +boto +boto3 +futures ; python_version == "2.7" # via s3transfer +s3cmd +pyyaml +backoff==1.4.3 +celery==3.1.25 +click==6.7 +opsgenie-sdk==0.3.1 +PyMySQL==0.9.3 +redis==2.10.6 +splunk-sdk==1.6.6 From 58e898a7d1173929d7d1b00a4b90b04297100127 Mon Sep 17 00:00:00 2001 From: Waheed Ahmed Date: Mon, 19 Apr 2021 13:30:44 +0500 Subject: [PATCH 022/664] Authn MFE: add npm overrides --- playbooks/roles/authn/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/authn/tasks/main.yml b/playbooks/roles/authn/tasks/main.yml index b5ea186d721..4a1aeb518fd 100644 --- a/playbooks/roles/authn/tasks/main.yml +++ b/playbooks/roles/authn/tasks/main.yml @@ -5,3 +5,4 @@ MFE_NAME: authn MFE_VERSION: '{{ AUTHN_MFE_VERSION }}' MFE_SITE_NAME: 'Your Platform Name Here' + MFE_NPM_OVERRIDES: '{{ AUTHN_MFE_NPM_OVERRIDES | default(MFE_DEPLOY_NPM_OVERRIDES) }}' From 61dedaa048f8d9430edbaef6005fad802f75003f Mon Sep 17 00:00:00 2001 From: Max Sokolski Date: Wed, 21 Apr 2021 13:02:43 +0300 Subject: [PATCH 023/664] fix: mysql 5.7 issue (#6376) mysql-client and mysql-server packages have been replaced with 5.7.34-1ubuntu18.04 version and should be updated for the mysql role --- playbooks/roles/mysql/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 7d1ac8c57b6..10a69f54b60 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -13,9 +13,9 @@ mysql_release_specific_debian_pkgs: mysql_debian_pkgs: "{{ mysql_debian_pkgs_default + mysql_release_specific_debian_pkgs[ansible_distribution_release] }}" mysql_server_pkg: "{{ 'mysql-server-5.7' if mysql_server_version_5_7 is defined and (mysql_server_version_5_7 | bool) else 'mysql-server-5.6' }}" -mysql_server_5_7_pkg: "mysql-server=5.7.33-1ubuntu18.04" -mysql_client_5_7_pkg: "mysql-client=5.7.33-1ubuntu18.04" -mysql_community_server_5_7_pkg: "mysql-server=5.7.33-1ubuntu18.04" +mysql_server_5_7_pkg: "mysql-server=5.7.34-1ubuntu18.04" +mysql_client_5_7_pkg: "mysql-client=5.7.34-1ubuntu18.04" +mysql_community_server_5_7_pkg: "mysql-server=5.7.34-1ubuntu18.04" mysql_dir: /etc/mysql From 8fd1d122cbee777d4cd49b526165095509907542 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 22 Apr 2021 14:27:00 +0500 Subject: [PATCH 024/664] requirements update in util/jenkins --- util/jenkins/requirements.txt | 105 ++++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 23 deletions(-) diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 51ff9793bfc..644e294b69c 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -4,36 +4,95 @@ # # make upgrade # -amqp==1.4.9 # via kombu -anyjson==0.3.3 # via kombu -awscli==1.14.32 +amqp==1.4.9 + # via kombu +anyjson==0.3.3 + # via kombu +awscli==1.19.55 + # via -r requirements/jenkins.in backoff==1.4.3 -billiard==3.3.0.23 # via celery -boto3==1.5.4 -botocore==1.8.36 # via awscli, boto3, s3transfer + # via -r requirements/jenkins.in +billiard==3.3.0.23 + # via celery +boto3==1.17.55 + # via -r requirements/jenkins.in +boto==2.49.0 + # via -r requirements/jenkins.in +botocore==1.20.55 + # via + # awscli + # boto3 + # s3transfer celery==3.1.25 -certifi==2019.11.28 # via opsgenie-sdk, requests -chardet==3.0.4 # via requests + # via -r requirements/jenkins.in +certifi==2020.12.5 + # via + # opsgenie-sdk + # requests +chardet==4.0.0 + # via requests click==6.7 -colorama==0.3.7 # via awscli -docutils==0.15.2 # via awscli, botocore + # via -r requirements/jenkins.in +colorama==0.4.3 + # via awscli +docutils==0.15.2 + # via awscli futures==3.3.0 ; python_version == "2.7" -idna==2.8 # via requests -jmespath==0.9.4 # via boto3, botocore -kombu==3.0.37 # via celery + # via + # -r requirements/jenkins.in + # s3transfer +idna==2.10 + # via requests +jmespath==0.10.0 + # via + # boto3 + # botocore +kombu==3.0.37 + # via celery opsgenie-sdk==0.3.1 -pyasn1==0.4.8 # via rsa -PyMySQL==0.9.3 -python-dateutil==2.8.1 # via botocore, opsgenie-sdk -pytz==2019.3 # via celery, opsgenie-sdk -pyyaml==3.12 # via awscli + # via -r requirements/jenkins.in +pyasn1==0.4.8 + # via rsa +pymysql==0.9.3 + # via -r requirements/jenkins.in +python-dateutil==2.8.1 + # via + # botocore + # opsgenie-sdk + # s3cmd +python-magic==0.4.22 + # via s3cmd +pytz==2021.1 + # via + # celery + # opsgenie-sdk +pyyaml==5.4.1 + # via + # -r requirements/jenkins.in + # awscli redis==2.10.6 -requests==2.22.0 # via opsgenie-sdk -rsa==3.4.2 # via awscli + # via -r requirements/jenkins.in +requests==2.25.1 + # via opsgenie-sdk +rsa==4.5 + # via awscli +s3cmd==2.1.0 + # via -r requirements/jenkins.in +s3transfer==0.4.1 + # via + # awscli + # boto3 +six==1.15.0 + # via + # opsgenie-sdk + # python-dateutil splunk-sdk==1.6.6 -s3transfer==0.1.13 # via awscli, boto3 -six==1.13.0 # via opsgenie-sdk, python-dateutil -urllib3==1.25.7 # via opsgenie-sdk, requests + # via -r requirements/jenkins.in +urllib3==1.26.4 + # via + # botocore + # opsgenie-sdk + # requests # The following packages are considered to be unsafe in a requirements file: # setuptools From 1aca75e3b661f99cda608f0d4cdbc1c878e9e967 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Mon, 26 Apr 2021 12:28:43 -0400 Subject: [PATCH 025/664] fix: change old jenkins timestamp patterns to use ZZ Old jenkins uses an older version of Java's timestamp pattern format. X or XX was introduced later (available in new jenkins). I'm not really sure if this is a Jenkins limitation or a Java JRE limitation. --- playbooks/roles/jenkins_data_engineering/defaults/main.yml | 4 +++- .../roles/jenkins_data_engineering_new/defaults/main.yml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index 860fbf9c378..1e2225a0284 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -251,4 +251,6 @@ jenkins_private_keyfile: "{{ jenkins_user_home }}/.ssh/id_rsa" jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" # Be clear about which time zone the console log timestamps are in!!! -de_jenkins_timestamper_system_time: "''HH:mm:ssXX' '" +# use ZZ for Jenkins < 2.222.x +# use XX for Jenkins >= 2.222.x +de_jenkins_timestamper_system_time: "''HH:mm:ssZZ' '" diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 02ed9e9220c..30e8ad8fad1 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -288,4 +288,6 @@ jenkins_private_keyfile: "{{ jenkins_user_home }}/.ssh/id_rsa" jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" # Be clear about which time zone the console log timestamps are in!!! +# use ZZ for Jenkins < 2.222.x +# use XX for Jenkins >= 2.222.x de_jenkins_timestamper_system_time: "''HH:mm:ssXX' '" From cfd8b4e211925ee43714a58fe1fc384bc1414d73 Mon Sep 17 00:00:00 2001 From: Luis Moreno Date: Mon, 26 Apr 2021 10:53:33 -0400 Subject: [PATCH 026/664] fix: exit with ansible exit code. (cherry picked from commit d78d2f642a4eb2c63806495f888f55430ef67321) --- util/install/native.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/install/native.sh b/util/install/native.sh index f80feca4381..7e816e8d351 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -189,4 +189,5 @@ if [[ $ansible_status -ne 0 ]]; then echo " Your environment:" env | egrep -i 'version|release' | sed -e 's/^/ /' echo "============================================================" + exit ${ansible_status} fi From d83f249916fd3a9aa6c62f4c244377eb07626200 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 28 Apr 2021 10:00:19 +0500 Subject: [PATCH 027/664] add license manager ansible config for sandbox --- playbooks/edx_continuous_integration.yml | 2 + playbooks/license_manager.yml | 22 +++ playbooks/roles/edxapp/defaults/main.yml | 7 + playbooks/roles/edxlocal/defaults/main.yml | 6 + .../roles/license_manager/defaults/main.yml | 168 ++++++++++++++++++ playbooks/roles/license_manager/meta/main.yml | 56 ++++++ .../roles/license_manager/tasks/main.yml | 23 +++ .../nginx/sites-available/license_manager.j2 | 49 ----- .../oauth_client_setup/defaults/main.yml | 10 ++ util/jenkins/ansible-provision.sh | 51 ++---- 10 files changed, 312 insertions(+), 82 deletions(-) create mode 100644 playbooks/license_manager.yml create mode 100644 playbooks/roles/license_manager/defaults/main.yml create mode 100644 playbooks/roles/license_manager/meta/main.yml create mode 100644 playbooks/roles/license_manager/tasks/main.yml delete mode 100644 playbooks/roles/nginx/templates/edx/app/nginx/sites-available/license_manager.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index af0facaef81..6eb5e2294a4 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -38,6 +38,8 @@ - discovery - role: registrar when: REGISTRAR_ENABLED + - role: license_manager + when: LICENSE_MANAGER_ENABLED - role: learner_portal when: LEARNER_PORTAL_ENABLED - role: program_console diff --git a/playbooks/license_manager.yml b/playbooks/license_manager.yml new file mode 100644 index 00000000000..691ef7ccbe4 --- /dev/null +++ b/playbooks/license_manager.yml @@ -0,0 +1,22 @@ +- name: Deploy edX License Manager + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: True + CLUSTER_NAME: 'license_manager' + REGISTRAR_ENABLED: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - role: nginx + nginx_default_sites: + - license_manager + - license_manager + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: hermes + HERMES_TARGET_SERVICE: 'license_manager' + when: REGISTRAR_HERMES_ENABLED diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 62e89efb46c..32c526a8ee2 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -957,6 +957,9 @@ REGISTRAR_SERVICE_USER_NAME: "registrar_worker" DESIGNER_SERVICE_USER_EMAIL: "designer_worker@example.com" DESIGNER_SERVICE_USER_NAME: "designer_worker" +# Configuration needed for LMS to communicate with the License Manager service +LICENSE_MANAGER_SERVICE_USER_EMAIL: "license_manager_worker@example.com" +LICENSE_MANAGER_SERVICE_USER_NAME: "license_manager_worker" # Configuration settings needed for the LMS to communicate with the Enterprise service. EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/enterprise/api/v1" @@ -1774,6 +1777,10 @@ SERVICE_WORKER_USERS: username: "{{ REGISTRAR_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ LICENSE_MANAGER_SERVICE_USER_EMAIL }}" + username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ EDXAPP_RETIREMENT_SERVICE_USER_EMAIL }}" username: "{{ EDXAPP_RETIREMENT_SERVICE_USER_NAME }}" is_staff: true diff --git a/playbooks/roles/edxlocal/defaults/main.yml b/playbooks/roles/edxlocal/defaults/main.yml index d83b6c12dad..b7830245548 100644 --- a/playbooks/roles/edxlocal/defaults/main.yml +++ b/playbooks/roles/edxlocal/defaults/main.yml @@ -17,6 +17,7 @@ edxlocal_databases: - "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}" - "{{ VEDA_WEB_FRONTEND_DEFAULT_DB_NAME | default(None) }}" - "{{ REGISTRAR_DEFAULT_DB_NAME | default(None) }}" + - "{{ LICENSE_MANAGER_DEFAULT_DB_NAME | default(None) }}" edxlocal_database_users: - { @@ -79,3 +80,8 @@ edxlocal_database_users: user: "{{ REGISTRAR_MYSQL_USER | default(None) }}", pass: "{{ REGISTRAR_MYSQL_PASSWORD | default(None) }}" } + - { + db: "{{ LICENSE_MANAGER_DEFAULT_DB_NAME | default(None) }}", + user: "{{ LICENSE_MANAGER_MYSQL_USER | default(None) }}", + pass: "{{ LICENSE_MANAGER_MYSQL_PASSWORD | default(None) }}" + } diff --git a/playbooks/roles/license_manager/defaults/main.yml b/playbooks/roles/license_manager/defaults/main.yml new file mode 100644 index 00000000000..59986cb8dc4 --- /dev/null +++ b/playbooks/roles/license_manager/defaults/main.yml @@ -0,0 +1,168 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +## +# Defaults for role license_manager +# + +LICENSE_MANAGER_GIT_IDENTITY: !!null + +LICENSE_MANAGER_ENABLED: True + +# +# vars are namespace with the module name. +# +license_manager_service_name: 'license_manager' + +license_manager_user: "{{ license_manager_service_name }}" +license_manager_home: "{{ COMMON_APP_DIR }}/{{ license_manager_service_name }}" +license_manager_app_dir: "{{ COMMON_APP_DIR }}/{{ license_manager_service_name }}" +license_manager_code_dir: "{{ license_manager_app_dir }}/{{ license_manager_service_name }}" +license_manager_venvs_dir: "{{ license_manager_app_dir }}/venvs" +license_manager_venv_dir: "{{ license_manager_venvs_dir }}/license_manager" +license_manager_celery_default_queue: 'license_manager.default' +license_manager_hostname: "license-manager" + +LICENSE_MANAGER_USE_PYTHON38: True + +LICENSE_MANAGER_CELERY_ALWAYS_EAGER: false +LICENSE_MANAGER_CELERY_BROKER_TRANSPORT: '' +LICENSE_MANAGER_CELERY_BROKER_USER: '' +LICENSE_MANAGER_CELERY_BROKER_PASSWORD: '' +LICENSE_MANAGER_CELERY_BROKER_HOSTNAME: '' +LICENSE_MANAGER_CELERY_BROKER_VHOST: '' + +license_manager_environment: + LICENSE_MANAGER_CFG: '{{ COMMON_CFG_DIR }}/{{ license_manager_service_name }}.yml' + +license_manager_gunicorn_port: 18170 + +license_manager_debian_pkgs: [] + +LICENSE_MANAGER_REPOS: + - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' + DOMAIN: '{{ COMMON_GIT_MIRROR }}' + PATH: '{{ COMMON_GIT_PATH }}' + REPO: 'license-manager.git' + VERSION: '{{ LICENSE_MANAGER_VERSION }}' + DESTINATION: "{{ license_manager_code_dir }}" + SSH_KEY: '{{ LICENSE_MANAGER_GIT_IDENTITY }}' + +LICENSE_MANAGER_NGINX_PORT: '1{{ license_manager_gunicorn_port }}' +LICENSE_MANAGER_SSL_NGINX_PORT: '4{{ license_manager_gunicorn_port }}' + +LICENSE_MANAGER_DEFAULT_DB_NAME: 'license-manager' +LICENSE_MANAGER_MYSQL_HOST: 'localhost' +# MySQL usernames are limited to 16 characters +LICENSE_MANAGER_MYSQL_USER: 'license-manager001' +LICENSE_MANAGER_MYSQL_PASSWORD: 'password' +LICENSE_MANAGER_MYSQL_CONN_MAX_AGE: 60 + +LICENSE_MANAGER_MEMCACHE: [ 'memcache' ] + +LICENSE_MANAGER_DJANGO_SETTINGS_MODULE: 'license_manager.settings.production' +LICENSE_MANAGER_DOMAIN: 'localhost' +LICENSE_MANAGER_URL_ROOT: 'http://{{ LICENSE_MANAGER_DOMAIN }}:{{ LICENSE_MANAGER_NGINX_PORT }}' +LICENSE_MANAGER_API_ROOT: '{{ LICENSE_MANAGER_URL_ROOT }}/api' +LICENSE_MANAGER_LOGOUT_URL: '{{ LICENSE_MANAGER_URL_ROOT }}/logout/' + +LICENSE_MANAGER_LANG: 'en_US.UTF-8' +LICENSE_MANAGER_LANGUAGE_CODE: 'en' +LICENSE_MANAGER_LANGUAGE_COOKIE_NAME: 'openedx-language-preference' + +LICENSE_MANAGER_SERVICE_USER: 'license_manager_service_user' + +LICENSE_MANAGER_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ license_manager_service_name }}' +LICENSE_MANAGER_MEDIA_ROOT: '{{ LICENSE_MANAGER_DATA_DIR }}/media' +LICENSE_MANAGER_MEDIA_URL: '/api/media/' + +LICENSE_MANAGER_MEDIA_STORAGE_BACKEND: + DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage' + MEDIA_ROOT: '{{ LICENSE_MANAGER_MEDIA_ROOT }}' + MEDIA_URL: '{{ LICENSE_MANAGER_MEDIA_URL }}' + +# TODO: Let edx_django_service manage LICENSE_MANAGER_STATIC_ROOT in phase 2. +LICENSE_MANAGER_STATIC_ROOT: '{{ LICENSE_MANAGER_DATA_DIR }}/staticfiles' +LICENSE_MANAGER_STATIC_URL: '/static/' + +LICENSE_MANAGER_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage' + +LICENSE_MANAGER_CORS_ORIGIN_ALLOW_ALL: false +LICENSE_MANAGER_CORS_ORIGIN_WHITELIST: [] + +LICENSE_MANAGER_CSRF_COOKIE_SECURE: false +LICENSE_MANAGER_CSRF_TRUSTED_ORIGINS: [] + +LICENSE_MANAGER_VERSION: 'master' + +LICENSE_MANAGER_GUNICORN_EXTRA: '' + +LICENSE_MANAGER_EXTRA_APPS: [] + +LICENSE_MANAGER_SESSION_EXPIRE_AT_BROWSER_CLOSE: false + +LICENSE_MANAGER_CERTIFICATE_LANGUAGES: + 'en': 'English' + 'es_419': 'Spanish' + +# Used to automatically configure OAuth2 Client +LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_KEY: 'license_manager-sso-key' +LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_SECRET: 'license_manager-sso-secret' +LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_KEY: 'license_manager-backend-service-key' +LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_SECRET: 'license_manager-backend-service-secret' +LICENSE_MANAGER_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false + +# API key for segment.io +LICENSE_MANAGER_SEGMENT_KEY: !!null + +LICENSE_MANAGER_DISCOVERY_BASE_URL: !!null +LICENSE_MANAGER_LMS_BASE_URL: !!null + +license_manager_service_config_overrides: + CERTIFICATE_LANGUAGES: '{{ LICENSE_MANAGER_CERTIFICATE_LANGUAGES }}' + LICENSE_MANAGER_SERVICE_USER: '{{ LICENSE_MANAGER_SERVICE_USER }}' + LANGUAGE_COOKIE_NAME: '{{ LICENSE_MANAGER_LANGUAGE_COOKIE_NAME }}' + SEGMENT_KEY: "{{ LICENSE_MANAGER_SEGMENT_KEY }}" + DISCOVERY_BASE_URL: "{{ LICENSE_MANAGER_DISCOVERY_BASE_URL }}" + LMS_BASE_URL: "{{ LICENSE_MANAGER_LMS_BASE_URL }}" + CORS_ORIGIN_WHITELIST: "{{ LICENSE_MANAGER_CORS_ORIGIN_WHITELIST }}" + CSRF_TRUSTED_ORIGINS: "{{ LICENSE_MANAGER_CSRF_TRUSTED_ORIGINS }}" + CSRF_COOKIE_SECURE: "{{ LICENSE_MANAGER_CSRF_COOKIE_SECURE }}" + CELERY_ALWAYS_EAGER: '{{ LICENSE_MANAGER_CELERY_ALWAYS_EAGER }}' + CELERY_BROKER_TRANSPORT: '{{ LICENSE_MANAGER_CELERY_BROKER_TRANSPORT }}' + CELERY_BROKER_USER: '{{ LICENSE_MANAGER_CELERY_BROKER_USER }}' + CELERY_BROKER_PASSWORD: '{{ LICENSE_MANAGER_CELERY_BROKER_PASSWORD }}' + CELERY_BROKER_HOSTNAME: '{{ LICENSE_MANAGER_CELERY_BROKER_HOSTNAME }}' + CELERY_BROKER_VHOST: '{{ LICENSE_MANAGER_CELERY_BROKER_VHOST }}' + CELERY_DEFAULT_EXCHANGE: 'license_manager' + CELERY_DEFAULT_ROUTING_KEY: 'license_manager' + CELERY_DEFAULT_QUEUE: '{{ license_manager_celery_default_queue }}' + +# See edx_django_service_automated_users for an example of what this should be +LICENSE_MANAGER_AUTOMATED_USERS: {} + +# NOTE: These variables are only needed to create the demo site (e.g. for sandboxes) + +LICENSE_MANAGER_ENABLE_NEWRELIC_DISTRIBUTED_TRACING: false + +# Remote config +LICENSE_MANAGER_HERMES_ENABLED: "{{ COMMON_HERMES_ENABLED }}" + +LICENSE_MANAGER_DECRYPT_CONFIG_ENABLED: "{{ COMMON_DECRYPT_CONFIG_ENABLED }}" +LICENSE_MANAGER_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" + +# Worker settings +worker_django_settings_module: "{{ LICENSE_MANAGER_DJANGO_SETTINGS_MODULE }}" +LICENSE_MANAGER_CELERY_WORKERS: + - queue: '{{ license_manager_celery_default_queue }}' + concurrency: 1 + monitor: True +license_manager_workers: "{{ LICENSE_MANAGER_CELERY_WORKERS }}" + +license_manager_post_migrate_commands: [] diff --git a/playbooks/roles/license_manager/meta/main.yml b/playbooks/roles/license_manager/meta/main.yml new file mode 100644 index 00000000000..375e37c93c3 --- /dev/null +++ b/playbooks/roles/license_manager/meta/main.yml @@ -0,0 +1,56 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +## +# Role includes for role license_manager +# +dependencies: + - role: edx_django_service + edx_django_service_use_python38: '{{ LICENSE_MANAGER_USE_PYTHON38 }}' + edx_django_service_version: '{{ LICENSE_MANAGER_VERSION }}' + edx_django_service_name: '{{ license_manager_service_name }}' + edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ license_manager_service_name }}' + edx_django_service_user: '{{ license_manager_user }}' + edx_django_service_config_overrides: '{{ license_manager_service_config_overrides }}' + edx_django_service_debian_pkgs_extra: '{{ license_manager_debian_pkgs }}' + edx_django_service_gunicorn_port: '{{ license_manager_gunicorn_port }}' + edx_django_service_django_settings_module: '{{ LICENSE_MANAGER_DJANGO_SETTINGS_MODULE }}' + edx_django_service_environment_extra: '{{ license_manager_environment }}' + edx_django_service_gunicorn_extra: '{{ LICENSE_MANAGER_GUNICORN_EXTRA }}' + edx_django_service_nginx_port: '{{ LICENSE_MANAGER_NGINX_PORT }}' + edx_django_service_ssl_nginx_port: '{{ LICENSE_MANAGER_SSL_NGINX_PORT }}' + edx_django_service_language_code: '{{ LICENSE_MANAGER_LANGUAGE_CODE }}' + edx_django_service_secret_key: '{{ LICENSE_MANAGER_SECRET_KEY }}' + edx_django_service_media_storage_backend: '{{ LICENSE_MANAGER_MEDIA_STORAGE_BACKEND }}' + edx_django_service_staticfiles_storage: '{{ LICENSE_MANAGER_STATICFILES_STORAGE }}' + edx_django_service_memcache: '{{ LICENSE_MANAGER_MEMCACHE }}' + edx_django_service_default_db_host: '{{ LICENSE_MANAGER_MYSQL_HOST }}' + edx_django_service_default_db_name: '{{ LICENSE_MANAGER_DEFAULT_DB_NAME }}' + edx_django_service_default_db_atomic_requests: false + edx_django_service_db_user: '{{ LICENSE_MANAGER_MYSQL_USER }}' + edx_django_service_db_password: '{{ LICENSE_MANAGER_MYSQL_PASSWORD }}' + edx_django_service_default_db_conn_max_age: '{{ LICENSE_MANAGER_MYSQL_CONN_MAX_AGE }}' + edx_django_service_extra_apps: '{{ LICENSE_MANAGER_EXTRA_APPS }}' + edx_django_service_session_expire_at_browser_close: '{{ LICENSE_MANAGER_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' + edx_django_service_social_auth_edx_oauth2_key: '{{ LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_KEY }}' + edx_django_service_social_auth_edx_oauth2_secret: '{{ LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_SECRET }}' + edx_django_service_backend_service_edx_oauth2_key: '{{ LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_KEY }}' + edx_django_service_backend_service_edx_oauth2_secret: '{{ LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_SECRET }}' + edx_django_service_automated_users: '{{ LICENSE_MANAGER_AUTOMATED_USERS }}' + edx_django_service_cors_whitelist: '{{ LICENSE_MANAGER_CORS_ORIGIN_WHITELIST }}' + edx_django_service_post_migrate_commands: '{{ license_manager_post_migrate_commands }}' + edx_django_service_enable_newrelic_distributed_tracing: '{{ LICENSE_MANAGER_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}' + edx_django_service_api_root: '{{ LICENSE_MANAGER_API_ROOT }}' + edx_django_service_decrypt_config_enabled: '{{ LICENSE_MANAGER_DECRYPT_CONFIG_ENABLED }}' + edx_django_service_copy_config_enabled: '{{ LICENSE_MANAGER_COPY_CONFIG_ENABLED }}' + edx_django_service_migration_check_services: '{{ license_manager_service_name }},{{ license_manager_service_name }}-workers' + edx_django_service_enable_celery_workers: true + edx_django_service_workers: '{{ license_manager_workers }}' + edx_django_service_repos: '{{ LICENSE_MANAGER_REPOS }}' + edx_django_service_hostname: '~^((stage|prod)-)?{{ license_manager_hostname }}.*' diff --git a/playbooks/roles/license_manager/tasks/main.yml b/playbooks/roles/license_manager/tasks/main.yml new file mode 100644 index 00000000000..c37f37ef6e7 --- /dev/null +++ b/playbooks/roles/license_manager/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +# +# +# Tasks for role license_manager +# +# Overview: This role's tasks come from edx_django_service. +# +# +# Dependencies: +# +# +# Example play: +# +# + diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/license_manager.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/license_manager.j2 deleted file mode 100644 index db527a06db7..00000000000 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/license_manager.j2 +++ /dev/null @@ -1,49 +0,0 @@ -upstream {{ license_manager_service_name }}_app_server { - server {{ minikube_host_ip }}:{{ license_manager_node_port }} fail_timeout=0; -} - -server { - listen {{ license_manager_nginx_port }}; - - {% if NGINX_ENABLE_SSL %} - - listen {{ license_manager_ssl_nginx_port }} ssl; - - ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; - ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; - {% endif %} - - {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %} - # request the browser to use SSL for all connections - add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}; includeSubDomains"; - {% endif %} - - {% include "common-settings.j2" %} - - # Prevent invalid display courseware in IE 10+ with high privacy settings - add_header P3P '{{ NGINX_P3P_MESSAGE }}'; - - {% include "handle-tls-redirect-and-ip-disclosure.j2" %} - - server_name '~^((stage|prod)-)?{{ license_manager_service_name }}.*'; - - location / { - try_files $uri @proxy_to_app; - } - - {% include "robots.j2" %} - -location @proxy_to_app { - proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; - proxy_set_header X-Forwarded-Port $http_x_forwarded_port; - proxy_set_header X-Forwarded-For $http_x_forwarded_for; - - # newrelic-specific header records the time when nginx handles a request. - proxy_set_header X-Queue-Start "t=${msec}"; - - proxy_set_header Host $http_host; - - proxy_redirect off; - proxy_pass http://{{ license_manager_service_name }}_app_server; - } -} diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 9dad5cb6e9c..937f6c3c496 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -92,6 +92,16 @@ oauth_client_setup_oauth2_clients: backend_service_secret: "{{ RETIREMENT_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", username: "{{ EDXAPP_RETIREMENT_SERVICE_USER_NAME | default('None') }}", } + - { + name: "{{ license_manager_service_name | default('None') }}", + url_root: "{{ LICENSE_MANAGER_URL_ROOT | default('None') }}", + sso_id: "{{ LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_KEY | default('None') }}", + sso_secret: "{{ LICENSE_MANAGER_SOCIAL_AUTH_EDX_OAUTH2_SECRET | default('None') }}", + backend_service_id: "{{ LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_KEY | default('None') }}", + backend_service_secret: "{{ LICENSE_MANAGER_BACKEND_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", + logout_uri: "{{ LICENSE_MANAGER_LOGOUT_URL | default('None') }}", + username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME | default('None') }}", + } # # OS packages # diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index fc2fb537886..25d80804d0a 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -184,6 +184,14 @@ if [[ -z $registrar_version ]]; then REGISTRAR_VERSION="master" fi +if [[ -z $license_manager ]]; then + license_manager="false" +fi + +if [[ -z $license_manager_version ]]; then + LICENSE_MANAGER_VERSION="master" +fi + if [[ -z $learner_portal ]]; then learner_portal="false" fi @@ -302,6 +310,13 @@ PAYMENT_SANDBOX_BUILD: True VIDEO_PIPELINE_BASE_NGINX_PORT: 80 VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: 443 +LICENSE_MANAGER_NGINX_PORT: 80 +LICENSE_MANAGER_SSL_NGINX_PORT: 443 +LICENSE_MANAGER_VERSION: $license_manager_version +LICENSE_MANAGER_ENABLED: $license_manager +LICENSE_MANAGER_DECRYPT_CONFIG_ENABLED: true +LICENSE_MANAGER_COPY_CONFIG_ENABLED: true + DISCOVERY_NGINX_PORT: 80 DISCOVERY_SSL_NGINX_PORT: 443 DISCOVERY_VERSION: $discovery_version @@ -319,15 +334,6 @@ COMMON_ECOMMERCE_BASE_URL: https://ecommerce-${deploy_host} nginx_default_sites: - lms -license_manager_service_name: "license-manager" -license_manager_nginx_port: 80 -license_manager_ssl_nginx_port: 443 -license_manager_gunicorn_port: 18170 -license_manager_node_port: 32100 - -edx_notes_api_gunicorn_port: 8120 -edx_notes_api_node_port: 32101 - mysql_server_version_5_7: True # User provided extra vars @@ -448,10 +454,12 @@ VEDA_WEB_FRONTEND_VERSION: ${video_pipeline_version:-master} VEDA_PIPELINE_WORKER_VERSION: ${video_pipeline_version:-master} VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master} +LICENSE_MANAGER_URL_ROOT: "https://license-manager-${deploy_host}" + EOF fi -encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api insights discovery credentials registrar edx_notes_api) +encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api insights discovery credentials registrar edx_notes_api license_manager) for app in ${encrypted_config_apps[@]}; do eval app_decrypt_and_copy_config_enabled=\${${app}_decrypt_and_copy_config_enabled} @@ -582,28 +590,5 @@ if [[ $enable_newrelic == "true" ]]; then run_ansible run_role.yml -i "${deploy_host}," -e role=newrelic_infrastructure $extra_var_arg --user ubuntu fi -if [[ $license_manager == "true" ]]; then - k8s_django_apps="license-manager" -fi -if [[ $edx_notes_api == "true" ]]; then - k8s_django_apps+=" edx-notes-api" -fi -if [[ ! -z $k8s_django_apps ]]; then - cat << EOF >> $extra_vars_file -K8S_DJANGO_APPS: $k8s_django_apps -EOF - - manifest_dir="k8s" - ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/configuration-internal/$manifest_dir dest=/var/tmp/" -u ubuntu -b - run_ansible run_role.yml -i "${deploy_host}," -e role=minikube $extra_var_arg --user ubuntu - minikube_host=$(ansible -c ssh -i "${deploy_host}," $deploy_host -m shell -a "su -c 'minikube ip' minikube" -u ubuntu -b) - minikube_host_ip=`echo $minikube_host | awk '{print $NF}'` - cat << EOF >> $extra_vars_file -minikube_host_ip: $minikube_host_ip -nginx_sites: ['license_manager'] -EOF - run_ansible run_role.yml -i "${deploy_host}," -e role=nginx $extra_var_arg --user ubuntu -fi - rm -f "$extra_vars_file" rm -f ${extra_vars_file}_clean From 4a2b040c1c0add51f1432fc80740219361910fe3 Mon Sep 17 00:00:00 2001 From: Diane Kaplan Date: Tue, 27 Apr 2021 16:36:15 -0400 Subject: [PATCH 028/664] add frontend-app-learning for sandboxes --- playbooks/edx_continuous_integration.yml | 3 +++ playbooks/learning.yml | 16 ++++++++++++++++ playbooks/roles/edx_ansible/templates/update.j2 | 1 + playbooks/roles/launch_ec2/tasks/main.yml | 3 ++- util/jenkins/ansible-provision.sh | 14 ++++++++++++++ 5 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 playbooks/learning.yml diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index af0facaef81..a51c1958808 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -49,6 +49,9 @@ - role: payment MFE_NAME: payment when: PAYMENT_MFE_ENABLED + - role: mfe + MFE_NAME: learning + when: LEARNING_MFE_ENABLED - oauth_client_setup - role: datadog when: COMMON_ENABLE_DATADOG diff --git a/playbooks/learning.yml b/playbooks/learning.yml new file mode 100644 index 00000000000..f939159cef9 --- /dev/null +++ b/playbooks/learning.yml @@ -0,0 +1,16 @@ +- name: Deploy learning MFE Frontend + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'learning' + MYMFE_ENABLED: True + MYMFE_SANDBOX_BUILD: False + roles: + - role: mfe + MFE_NAME: learning + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index 5d7fd4685b1..02514bbde10 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -81,6 +81,7 @@ repos_to_cmd["program_console"]="$edx_ansible_cmd program_console.yml -e 'PROGRA repos_to_cmd["prospectus"]="$edx_ansible_cmd prospectus.yml -e 'PROSPECTUS_VERSION=$2'" repos_to_cmd["authn"]="$edx_ansible_cmd authn_frontend.yml -e 'AUTHN_MFE_VERSION=$2'" repos_to_cmd["payment"]="$edx_ansible_cmd payment.yml -e 'PAYMENT_MFE_VERSION=$2'" +repos_to_cmd["learning"]="$edx_ansible_cmd learning.yml -e 'LEARNING_MFE_VERSION=$2'" if [[ -z $1 || -z $2 ]]; then echo diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index ac558aa1d2b..c3100c193ed 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -89,7 +89,8 @@ delay: 30 with_nested: - "{{ ec2.instances }}" - - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager'] + - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning'] - name: Add new instance to host group local_action: diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index fc2fb537886..0fef8094b3b 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -220,6 +220,14 @@ if [[ -z $payment_version ]]; then PAYMENT_MFE_VERSION="master" fi +if [[ -z $learning ]]; then + learning="false" +fi + +if [[ -z $learning_version ]]; then + LEARNING_MFE_VERSION="master" +fi + # Lowercase the dns name to deal with an ansible bug dns_name="${dns_name,,}" @@ -325,6 +333,12 @@ license_manager_ssl_nginx_port: 443 license_manager_gunicorn_port: 18170 license_manager_node_port: 32100 +LEARNING_NGINX_PORT: 80 +LEARNING_SSL_NGINX_PORT: 443 +LEARNING_MFE_VERSION: $learning_version +LEARNING_MFE_ENABLED: $learning +LEARNING_SANDBOX_BUILD: True + edx_notes_api_gunicorn_port: 8120 edx_notes_api_node_port: 32101 From 7fc73f1b7982fa3e706a83271cb8c6d3942d3a7d Mon Sep 17 00:00:00 2001 From: Jazib Humayun Date: Thu, 29 Apr 2021 22:07:53 +0500 Subject: [PATCH 029/664] feat: Install python 3.8 & 3.9 on New DE Jenkins Python3.8 is not installed in New DE Jenkins, I have encountered some packages are not availble in < python3.8 in order to deploy prefect flows, so installing python3.8 and python3.9 --- .../roles/jenkins_data_engineering_new/defaults/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 30e8ad8fad1..c49bf628434 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -36,6 +36,8 @@ de_jenkins_snap_pkgs: de_jenkins_python_versions: - python3.5 - python3.7 + - python3.8 + - python3.9 # Jenkins aliases to installed Python binaries de_jenkins_python_installations: @@ -51,6 +53,9 @@ de_jenkins_python_installations: - PYTHON_ALIAS: 'PYTHON_3.8' PYTHON_PATH: '/usr/bin/python3.8' PYTHON_PROPERTIES: [] + - PYTHON_ALIAS: 'PYTHON_3.9' + PYTHON_PATH: '/usr/bin/python3.9' + PYTHON_PROPERTIES: [] jenkins_base_environment_variables: - NAME: 'AUTOMATION_USER' From 983cb796be69278d8b0e71690a614f3e0ca19325 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 30 Apr 2021 15:05:39 +0500 Subject: [PATCH 030/664] add enterprise catalog sandbox config --- playbooks/edx_continuous_integration.yml | 2 ++ playbooks/roles/edxapp/defaults/main.yml | 7 +++++++ playbooks/roles/edxlocal/defaults/main.yml | 6 ++++++ .../roles/enterprise_catalog/defaults/main.yml | 2 ++ playbooks/roles/enterprise_catalog/meta/main.yml | 1 + playbooks/roles/launch_ec2/tasks/main.yml | 2 +- .../roles/oauth_client_setup/defaults/main.yml | 10 ++++++++++ util/jenkins/ansible-provision.sh | 13 +++++++++++++ 8 files changed, 42 insertions(+), 1 deletion(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 05b9bd99a1b..13e76e137ea 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -40,6 +40,8 @@ when: REGISTRAR_ENABLED - role: license_manager when: LICENSE_MANAGER_ENABLED + - role: enterprise_catalog + when: ENTERPRISE_CATALOG_ENABLED - role: learner_portal when: LEARNER_PORTAL_ENABLED - role: program_console diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 32c526a8ee2..057d299f09b 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -961,6 +961,9 @@ DESIGNER_SERVICE_USER_NAME: "designer_worker" LICENSE_MANAGER_SERVICE_USER_EMAIL: "license_manager_worker@example.com" LICENSE_MANAGER_SERVICE_USER_NAME: "license_manager_worker" +ENTERPRISE_CATALOG_SERVICE_USER_EMAIL: "enterprise_catalog_worker@example.com" +ENTERPRISE_CATALOG_SERVICE_USER_NAME: "enterprise_catalog_worker" + # Configuration settings needed for the LMS to communicate with the Enterprise service. EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/enterprise/api/v1" @@ -1781,6 +1784,10 @@ SERVICE_WORKER_USERS: username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ ENTERPRISE_CATALOG_SERVICE_USER_EMAIL }}" + username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ EDXAPP_RETIREMENT_SERVICE_USER_EMAIL }}" username: "{{ EDXAPP_RETIREMENT_SERVICE_USER_NAME }}" is_staff: true diff --git a/playbooks/roles/edxlocal/defaults/main.yml b/playbooks/roles/edxlocal/defaults/main.yml index b7830245548..509ff28d6ce 100644 --- a/playbooks/roles/edxlocal/defaults/main.yml +++ b/playbooks/roles/edxlocal/defaults/main.yml @@ -18,6 +18,7 @@ edxlocal_databases: - "{{ VEDA_WEB_FRONTEND_DEFAULT_DB_NAME | default(None) }}" - "{{ REGISTRAR_DEFAULT_DB_NAME | default(None) }}" - "{{ LICENSE_MANAGER_DEFAULT_DB_NAME | default(None) }}" + - "{{ ENTERPRISE_CATALOG_DEFAULT_DB_NAME | default(None) }}" edxlocal_database_users: - { @@ -85,3 +86,8 @@ edxlocal_database_users: user: "{{ LICENSE_MANAGER_MYSQL_USER | default(None) }}", pass: "{{ LICENSE_MANAGER_MYSQL_PASSWORD | default(None) }}" } + - { + db: "{{ ENTERPRISE_CATALOG_DEFAULT_DB_NAME | default(None) }}", + user: "{{ ENTERPRISE_CATALOG_MYSQL_USER | default(None) }}", + pass: "{{ ENTERPRISE_CATALOG_MYSQL_PASSWORD | default(None) }}" + } diff --git a/playbooks/roles/enterprise_catalog/defaults/main.yml b/playbooks/roles/enterprise_catalog/defaults/main.yml index 869eeb3ab8e..248f9e20ea9 100644 --- a/playbooks/roles/enterprise_catalog/defaults/main.yml +++ b/playbooks/roles/enterprise_catalog/defaults/main.yml @@ -26,6 +26,8 @@ enterprise_catalog_venv_dir: "{{ enterprise_catalog_venvs_dir }}/enterprise_cata enterprise_catalog_celery_default_queue: 'enterprise_catalog.default' +enterprise_catalog_hostname: 'enterprise-catalog' + ENTERPRISE_CATALOG_CELERY_ALWAYS_EAGER: false ENTERPRISE_CATALOG_CELERY_BROKER_TRANSPORT: '' ENTERPRISE_CATALOG_CELERY_BROKER_USER: '' diff --git a/playbooks/roles/enterprise_catalog/meta/main.yml b/playbooks/roles/enterprise_catalog/meta/main.yml index f4f3fcf2da2..af48090ee00 100644 --- a/playbooks/roles/enterprise_catalog/meta/main.yml +++ b/playbooks/roles/enterprise_catalog/meta/main.yml @@ -53,5 +53,6 @@ dependencies: # Need this override because the existing AWS ASGs have a services tag with a name that doesn't match the convention edx_django_service_workers_supervisor_conf: 'enterprise_catalog_worker.conf' edx_django_service_docker_image_name: 'openedx/enterprise-catalog' + edx_django_service_hostname: '~^((stage|prod)-)?{{ enterprise_catalog_hostname }}.*' EDX_DJANGO_SERVICE_ENABLE_ADMIN_URLS_RESTRICTION: '{{ ENTERPRISE_CATALOG_ENABLE_ADMIN_URLS_RESTRICTION }}' EDX_DJANGO_SERVICE_ADMIN_URLS: '{{ ENTERPRISE_CATALOG_ADMIN_URLS }}' diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index c3100c193ed..c3edb114af1 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,7 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog'] - name: Add new instance to host group local_action: diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 937f6c3c496..8bedb458abb 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -102,6 +102,16 @@ oauth_client_setup_oauth2_clients: logout_uri: "{{ LICENSE_MANAGER_LOGOUT_URL | default('None') }}", username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME | default('None') }}", } + - { + name: "{{ enterprise_catalog_service_name | default('None') }}", + url_root: "{{ ENTERPRISE_CATALOG_URL_ROOT | default('None') }}", + sso_id: "{{ ENTERPRISE_CATALOG_SOCIAL_AUTH_EDX_OAUTH2_KEY | default('None') }}", + sso_secret: "{{ ENTERPRISE_CATALOG_SOCIAL_AUTH_EDX_OAUTH2_SECRET | default('None') }}", + backend_service_id: "{{ ENTERPRISE_CATALOG_BACKEND_SERVICE_EDX_OAUTH2_KEY | default('None') }}", + backend_service_secret: "{{ ENTERPRISE_CATALOG_BACKEND_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", + logout_uri: "{{ ENTERPRISE_CATALOG_LOGOUT_URL | default('None') }}", + username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME | default('None') }}", + } # # OS packages # diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index d8d1103dc58..85daba30be3 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -192,6 +192,10 @@ if [[ -z $license_manager_version ]]; then LICENSE_MANAGER_VERSION="master" fi +if [[ -z $enterprise_catalog_version ]]; then + ENTERPRISE_CATALOG_VERSION="master" +fi + if [[ -z $learner_portal ]]; then learner_portal="false" fi @@ -325,6 +329,13 @@ LICENSE_MANAGER_ENABLED: $license_manager LICENSE_MANAGER_DECRYPT_CONFIG_ENABLED: true LICENSE_MANAGER_COPY_CONFIG_ENABLED: true +ENTERPRISE_CATALOG_NGINX_PORT: 80 +ENTERPRISE_CATALOG_SSL_NGINX_PORT: 443 +ENTERPRISE_CATALOG_VERSION: $enterprise_catalog_version +ENTERPRISE_CATALOG_ENABLED: $enterprise_catalog +ENTERPRISE_CATALOG_DECRYPT_CONFIG_ENABLED: true +ENTERPRISE_CATALOG_COPY_CONFIG_ENABLED: true + DISCOVERY_NGINX_PORT: 80 DISCOVERY_SSL_NGINX_PORT: 443 DISCOVERY_VERSION: $discovery_version @@ -470,6 +481,8 @@ VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master} LICENSE_MANAGER_URL_ROOT: "https://license-manager-${deploy_host}" +ENTERPRISE_CATALOG_URL_ROOT: "https://enterprise-catalog-${deploy_host}" + EOF fi From d8ab54b6ff3d249b6f9731c6a9c4b21d27d22215 Mon Sep 17 00:00:00 2001 From: "M. Zulqarnain" Date: Tue, 4 May 2021 16:38:58 +0500 Subject: [PATCH 031/664] feat: upgrade pip to 20.1.1 (#6315) --- docker/build/analytics_pipeline/Dockerfile | 2 +- playbooks/roles/common_vars/defaults/main.yml | 2 +- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- pre-requirements.txt | 2 +- util/install/ansible-bootstrap.sh | 2 +- util/install/native.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/build/analytics_pipeline/Dockerfile b/docker/build/analytics_pipeline/Dockerfile index 5c611f347ab..bcdc6be0e1c 100644 --- a/docker/build/analytics_pipeline/Dockerfile +++ b/docker/build/analytics_pipeline/Dockerfile @@ -29,7 +29,7 @@ ENV BOTO_CONFIG=/dev/null \ ANALYTICS_PIPELINE_VENV=/edx/app/analytics_pipeline/venvs \ BOOTSTRAP=/etc/bootstrap.sh \ COMMON_BASE_DIR=/edx \ - COMMON_PIP_PACKAGES_PIP='pip==20.0.2' \ + COMMON_PIP_PACKAGES_PIP='pip==20.1.1' \ COMMON_PIP_PACKAGES_SETUPTOOLS='setuptools==44.1.0' \ COMMON_PIP_PACKAGES_VIRTUALENV='virtualenv==20.1.0' \ COMMON_MYSQL_READ_ONLY_USER='read_only' \ diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 11d0e3431cc..5d6515e43e5 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -149,7 +149,7 @@ old_python_debian_pkgs: - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1" -COMMON_PIP_VERSION: '20.0.2' +COMMON_PIP_VERSION: '20.1.1' common_pip_pkgs: - pip=={{ COMMON_PIP_VERSION }} diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 35017a88b7d..a68375f54e3 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -275,7 +275,7 @@ - name: Pin pip to a specific version. # Not pinning to the same version as everything else because sandboxes are still python 2.7 - command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==20.0.2" + command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==20.1.1" args: chdir: "{{ edxapp_code_dir }}" become_user: "{{ edxapp_sandbox_user }}" diff --git a/pre-requirements.txt b/pre-requirements.txt index 9e1ff379964..23dcf46d410 100644 --- a/pre-requirements.txt +++ b/pre-requirements.txt @@ -1 +1 @@ -pip==20.0.2 +pip==20.1.1 diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index a4c6bb40eb0..5d1f04be146 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -33,7 +33,7 @@ fi # Bootstrapping constants # VIRTUAL_ENV_VERSION="16.7.10" -PIP_VERSION="20.0.2" +PIP_VERSION="20.1.1" SETUPTOOLS_VERSION="44.1.0" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" diff --git a/util/install/native.sh b/util/install/native.sh index 7e816e8d351..0b978ce535e 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -101,7 +101,7 @@ sudo apt-get upgrade -y sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python3-pip libmysqlclient-dev python3-apt python3-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ # ansible-bootstrap installs yaml that pip 19 can't uninstall. sudo apt-get remove -y python-yaml -sudo pip3 install --upgrade pip==20.0.2 +sudo pip3 install --upgrade pip==20.1.1 sudo pip3 install --upgrade setuptools==44.1.0 sudo -H pip3 install --upgrade virtualenv==20.2.0 From e1a63f6a7c3a40f5c5353935fb463c6bc7c743f6 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Wed, 5 May 2021 11:38:29 +0500 Subject: [PATCH 032/664] build: update xqwatcher to use Python38 environment --- playbooks/roles/xqwatcher/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/xqwatcher/defaults/main.yml b/playbooks/roles/xqwatcher/defaults/main.yml index c9af89aedf5..dc47bbe229a 100644 --- a/playbooks/roles/xqwatcher/defaults/main.yml +++ b/playbooks/roles/xqwatcher/defaults/main.yml @@ -87,7 +87,7 @@ xqwatcher_code_dir: "{{ xqwatcher_app_dir }}/src" xqwatcher_repo_name: xqueue-watcher.git -xqwatcher_python_version: "python3.5" +xqwatcher_python_version: "python3.8" #TODO: change this to /edx/etc after pulling xqwatcher.json out xqwatcher_conf_dir: "{{ xqwatcher_app_dir }}" From 73e65cf0a799dd31e24b953fb2522be28e96a8d9 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 6 May 2021 11:08:26 +0500 Subject: [PATCH 033/664] cleanup minikube role --- playbooks/roles/minikube/defaults/main.yml | 19 ------- playbooks/roles/minikube/meta/main.yml | 4 -- playbooks/roles/minikube/tasks/main.yml | 55 ------------------- .../templates/k8s_manifest_script.sh.j2 | 36 ------------ 4 files changed, 114 deletions(-) delete mode 100644 playbooks/roles/minikube/defaults/main.yml delete mode 100644 playbooks/roles/minikube/meta/main.yml delete mode 100644 playbooks/roles/minikube/tasks/main.yml delete mode 100644 playbooks/roles/minikube/templates/k8s_manifest_script.sh.j2 diff --git a/playbooks/roles/minikube/defaults/main.yml b/playbooks/roles/minikube/defaults/main.yml deleted file mode 100644 index dac909d2499..00000000000 --- a/playbooks/roles/minikube/defaults/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -minikube_user: "minikube" - -minikube_kubernetes_version: "1.18.9" - -minikube_kubectl_version: "v1.18.9" -minikube_version: "latest" -minikube_install_dir: "/usr/local/bin" - -minikube_helm_url: "https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz" - -minikube_packages: - - url: "https://storage.googleapis.com/kubernetes-release/release/{{ minikube_kubectl_version }}/bin/linux/amd64/kubectl" - dest: "{{ minikube_install_dir }}/kubectl" - - url: "https://storage.googleapis.com/minikube/releases/{{ minikube_version }}/minikube-linux-amd64" - dest: "{{ minikube_install_dir }}/minikube" - - url: "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.2.1/kustomize_kustomize.v3.2.1_linux_amd64" - dest: "{{ minikube_install_dir }}/kustomize" diff --git a/playbooks/roles/minikube/meta/main.yml b/playbooks/roles/minikube/meta/main.yml deleted file mode 100644 index 71f8b01bccd..00000000000 --- a/playbooks/roles/minikube/meta/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -dependencies: - - docker-tools diff --git a/playbooks/roles/minikube/tasks/main.yml b/playbooks/roles/minikube/tasks/main.yml deleted file mode 100644 index a9cad715a47..00000000000 --- a/playbooks/roles/minikube/tasks/main.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- - -- name: Add minikube user - user: - name: "{{ minikube_user }}" - shell: /bin/bash - -- name: add minikube user to docker group - user: - name: "{{ minikube_user }}" - groups: "{{ docker_group }}" - append: yes - -- name: get required binaries and put to executables path - get_url: - url: "{{ item.url }}" - dest: "{{ item.dest }}" - mode: "0755" - with_items: "{{ minikube_packages }}" - -- name: download helm package - get_url: - url: "{{ minikube_helm_url }}" - dest: "/tmp/helm.tar.gz" - mode: 0644 - -- name: extract helm archive - unarchive: - src: "/tmp/helm.tar.gz" - dest: "/tmp/" - copy: false - -- name: copy helm binary to executable path - copy: - src: /tmp/linux-amd64/helm - dest: /usr/local/bin/helm - remote_src: yes - mode: 0755 - -- name: start minikube - shell: - minikube start \ - --vm-driver=docker --memory=4000 \ - --kubernetes-version={{ minikube_kubernetes_version }} - become_user: "{{ minikube_user }}" - -- name: copy manifest script - template: - src: "k8s_manifest_script.sh.j2" - dest: "/tmp/k8s_manifest_script.sh" - mode: 0755 - -- name: execute manifest script to install k8s apps - shell: bash /tmp/k8s_manifest_script.sh - become_user: "{{ minikube_user }}" diff --git a/playbooks/roles/minikube/templates/k8s_manifest_script.sh.j2 b/playbooks/roles/minikube/templates/k8s_manifest_script.sh.j2 deleted file mode 100644 index a1fdfbf27ee..00000000000 --- a/playbooks/roles/minikube/templates/k8s_manifest_script.sh.j2 +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -manifest_dir="/var/tmp/k8s/applications/" - -cd $manifest_dir - - -{% if K8S_DJANGO_APPS|length %} - k8s_apps="vault mysql redis elasticsearch {{ K8S_DJANGO_APPS }}" -{% else %} - k8s_apps="vault mysql redis elasticsearch" -{% endif %} - -for k8s_app in $k8s_apps; do - - kubectl create namespace $k8s_app - - if [[ $k8s_app == "vault" ]]; then - helm install $k8s_app --namespace $k8s_app $k8s_app/ -f $k8s_app/development.yaml - kustomize build vault-development-bootstrapper/ | kubectl apply -f - - kustomize build vault-acl-job/development/ | kubectl apply -f - - elif [[ $k8s_app == "edx-notes-api" || $k8s_app == "license-manager" ]]; then - if [[ $k8s_app == "license-manager" ]]; then - app_port={{ license_manager_gunicorn_port }} - node_port={{ license_manager_node_port }} - elif [[ $k8s_app == "edx-notes-api" ]]; then - app_port={{ edx_notes_api_gunicorn_port }} - node_port={{ edx_notes_api_node_port }} - fi - helm install $k8s_app --namespace $k8s_app -f $k8s_app/development-currently-deployed-image.yaml -f $k8s_app/development-config.yaml $k8s_app/ - kubectl expose deployment $k8s_app -n $k8s_app --name=$k8s_app-nodeport --port=$app_port \ - --type=NodePort --overrides '{ "apiVersion": "v1","spec":{"ports":[{"port":'$app_port',"protocol":"TCP","nodePort":'$node_port'}]}}' - else - helm install $k8s_app --namespace $k8s_app $k8s_app/ -f $k8s_app/development.yaml - fi -done From e0b7fac62c6c4c832b4a1569ced2bc57a189cb18 Mon Sep 17 00:00:00 2001 From: "Albert (AJ) St. Aubin" Date: Thu, 13 May 2021 12:46:53 -0400 Subject: [PATCH 034/664] refactor: Removing the cert_whitelist command from this role [MICROBA-1052] The cert_whitelist is being deprecated. We are removing it use here in this role because it provides no value in the way we currently generate certificates. --- playbooks/roles/demo/tasks/deploy.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/playbooks/roles/demo/tasks/deploy.yml b/playbooks/roles/demo/tasks/deploy.yml index 2754b761cda..0c427c3d8d8 100644 --- a/playbooks/roles/demo/tasks/deploy.yml +++ b/playbooks/roles/demo/tasks/deploy.yml @@ -47,13 +47,6 @@ - "{{ demo_test_and_staff_users }}" when: demo_checkout.changed -- name: add test users to the certificate whitelist - shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} --service-variant lms cert_whitelist -a {{ item.email }} -c {{ demo_course_id }}" - args: - chdir: "{{ demo_edxapp_code_dir }}" - with_items: "{{ demo_test_users }}" - when: demo_checkout.changed - - name: seed the forums for the demo course shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} seed_permissions_roles {{ demo_course_id }}" args: From f731c07c83191d7fcc822a9e605744ed4ce9d318 Mon Sep 17 00:00:00 2001 From: Demid Date: Fri, 14 May 2021 02:10:34 +0300 Subject: [PATCH 035/664] [OSPR-5773][BB-3669] Ansible task to install extra edX Django Service requirements (#6400) * feat: ansible task to install extra edx django service requirements * docs: update changelog --- CHANGELOG.md | 8 ++++++++ playbooks/roles/discovery/defaults/main.yml | 11 +++++++++++ playbooks/roles/discovery/meta/main.yml | 1 + playbooks/roles/ecommerce/defaults/main.yml | 11 +++++++++++ playbooks/roles/ecommerce/meta/main.yml | 1 + .../roles/edx_django_service/defaults/main.yml | 11 +++++++++++ playbooks/roles/edx_django_service/tasks/main.yml | 13 +++++++++++++ 7 files changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be1259159f..f9dbbae44dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-05-13 + - Role: edx_django_service + - Added task that installs extra python packages specified in `edx_django_service_extra_requirements`. + - Role: discovery + - Installs extra python packages specified in `DISCOVERY_EXTRA_REQUIREMENTS`. + - Role: ecommerce + - Installs extra python packages specified in `ECOMMERCE_EXTRA_REQUIREMENTS`. + - 2021-03-08 - Remove instruction from ansile-bootstrap.sh that instructed people to activate the virtualenv. This was incorrect for community installations. diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 0af003b7b76..6c4387c66ee 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -139,6 +139,17 @@ DISCOVERY_REPOS: DESTINATION: "{{ discovery_code_dir }}" SSH_KEY: "{{ DISCOVERY_GIT_IDENTITY }}" +# List of additional python packages that should be installed into the +# discovery virtual environment. +# `name` (required), `version` (optional), and `extra_args` (optional) +# are supported and correspond to the options of ansible's pip module. +# Example: +# DISCOVERY_EXTRA_REQUIREMENTS: +# - name: mypackage +# version: 1.0.1 +# - name: git+https://git.myproject.org/MyProject#egg=MyProject +DISCOVERY_EXTRA_REQUIREMENTS: [] + discovery_service_config_overrides: ELASTICSEARCH_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}' ELASTICSEARCH_INDEX_NAME: '{{ DISCOVERY_ELASTICSEARCH_INDEX_NAME }}' diff --git a/playbooks/roles/discovery/meta/main.yml b/playbooks/roles/discovery/meta/main.yml index 328131dc83e..f0b0446f819 100644 --- a/playbooks/roles/discovery/meta/main.yml +++ b/playbooks/roles/discovery/meta/main.yml @@ -27,6 +27,7 @@ dependencies: edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ discovery_service_name }}' edx_django_service_config_overrides: '{{ discovery_service_config_overrides }}' edx_django_service_debian_pkgs_extra: '{{ discovery_debian_pkgs }}' + edx_django_service_extra_requirements: '{{ DISCOVERY_EXTRA_REQUIREMENTS }}' edx_django_service_gunicorn_port: '{{ discovery_gunicorn_port }}' edx_django_service_django_settings_module: '{{ DISCOVERY_DJANGO_SETTINGS_MODULE }}' edx_django_service_environment_extra: '{{ discovery_environment }}' diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 27c47b42a3f..747acc43890 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -22,6 +22,17 @@ ECOMMERCE_REPOS: DESTINATION: "{{ ecommerce_code_dir }}" SSH_KEY: "{{ ECOMMERCE_GIT_IDENTITY }}" +# List of additional python packages that should be installed into the +# ecommerce virtual environment. +# `name` (required), `version` (optional), and `extra_args` (optional) +# are supported and correspond to the options of ansible's pip module. +# Example: +# ECOMMERCE_EXTRA_REQUIREMENTS: +# - name: mypackage +# version: 1.0.1 +# - name: git+https://git.myproject.org/MyProject#egg=MyProject +ECOMMERCE_EXTRA_REQUIREMENTS: [] + # depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC # and a key being provided via NEWRELIC_LICENSE_KEY ECOMMERCE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ ecommerce_service_name }}" diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index 35a4a1fac13..8b4fceed69b 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -21,6 +21,7 @@ dependencies: edx_django_service_config_overrides: '{{ ecommerce_service_config_overrides }}' edx_django_service_debian_pkgs_extra: '{{ ecommerce_debian_pkgs + ecommerce_release_specific_debian_pkgs[ansible_distribution_release] }}' edx_django_service_django_settings_module: '{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}' + edx_django_service_extra_requirements: '{{ ECOMMERCE_EXTRA_REQUIREMENTS }}' edx_django_service_repos: '{{ ECOMMERCE_REPOS }}' edx_django_service_environment_extra: '{{ ecommerce_environment }}' edx_django_service_gunicorn_extra: '{{ ECOMMERCE_GUNICORN_EXTRA }}' diff --git a/playbooks/roles/edx_django_service/defaults/main.yml b/playbooks/roles/edx_django_service/defaults/main.yml index 55f4d8eea15..cde52c7581f 100644 --- a/playbooks/roles/edx_django_service/defaults/main.yml +++ b/playbooks/roles/edx_django_service/defaults/main.yml @@ -57,6 +57,17 @@ edx_django_service_debian_pkgs_default: edx_django_service_debian_pkgs_extra: [] edx_django_service_debian_pkgs: '{{ edx_django_service_debian_pkgs_default + edx_django_service_debian_pkgs_extra }}' +# List of additional python packages that should be installed into the +# service virtual environment. +# `name` (required), `version` (optional), and `extra_args` (optional) +# are supported and correspond to the options of ansible's pip module. +# Example: +# edx_django_service_extra_requirements: +# - name: mypackage +# version: 1.0.1 +# - name: git+https://git.myproject.org/MyProject#egg=MyProject +edx_django_service_extra_requirements: [] + edx_django_service_gunicorn_extra: '' edx_django_service_gunicorn_extra_conf: '' edx_django_service_gunicorn_host: '127.0.0.1' diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 26260fbdc3c..5055f762694 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -180,6 +180,19 @@ - devstack - devstack:install +- name: install extra requirements + pip: + name: "{{ item.name }}" + version: "{{ item.version|default(omit) }}" + extra_args: "--exists-action w {{ item.extra_args|default('') }}" + virtualenv: "{{ edx_django_service_venv_dir }}" + state: present + with_items: "{{ edx_django_service_extra_requirements }}" + become_user: "{{ edx_django_service_user }}" + tags: + - install + - install:app-requirements + - name: Check for existing make_migrate container command: "docker ps -aq --filter name='{{ edx_django_service_name }}.make_migrate'" register: edx_django_service_make_migrate_container From 7a4fcabb5e1db12e08fe2112e0e3f408d8cb81ef Mon Sep 17 00:00:00 2001 From: Jhony Avella Date: Thu, 6 May 2021 14:48:01 -0500 Subject: [PATCH 036/664] feat: Account MFE support in Native Installation (#6381) Add required vars for Account MFE to work in native installation. Add LANGUAGE_PREFERENCE_COOKIE_NAME default environment var for MFEs. Update var that controls MFEs repo version when using mfe_deployer. Co-authored-by: Eric Herrera --- playbooks/openedx_native.yml | 12 ++++++++++++ playbooks/roles/mfe/defaults/main.yml | 2 ++ util/install/native.sh | 4 +--- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 4dd5bac10a0..924b5dd6a98 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -8,9 +8,20 @@ gather_facts: True vars: migrate_db: "yes" + MFE_DEPLOY_NGINX_PORT: 19010 + MFE_BASE: "{{ EDXAPP_LMS_BASE }}:{{ MFE_DEPLOY_NGINX_PORT }}" + MFE_DEPLOY_COMMON_HOSTNAME: "{{ EDXAPP_LMS_BASE }}" EDXAPP_PREVIEW_LMS_BASE: 'preview.{{ EDXAPP_LMS_BASE }}' EDXAPP_LOGIN_REDIRECT_WHITELIST: [ "{{ EDXAPP_CMS_BASE }}", + "{{ MFE_BASE }}", + ] + EDXAPP_CORS_ORIGIN_WHITELIST: [ + "{{ MFE_BASE }}", + ] + + EDXAPP_CSRF_TRUSTED_ORIGINS: [ + "{{ MFE_BASE }}", ] EDXAPP_ENABLE_CORS_HEADERS: true EDXAPP_ENABLE_CROSS_DOMAIN_CSRF_COOKIE: true @@ -18,6 +29,7 @@ EDXAPP_CROSS_DOMAIN_CSRF_COOKIE_NAME: 'native-csrf-cookie' EDXAPP_LMS_BASE_SCHEME: http COMMON_LMS_BASE_URL: "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ EDXAPP_LMS_BASE }}" + EDXAPP_ACCOUNT_MICROFRONTEND_URL: "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}/account" EDXAPP_LMS_NGINX_PORT: '80' EDX_PLATFORM_VERSION: 'master' # Set to false if deployed behind another proxy/load balancer. diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 8f46cff0f7f..5f1e8c019f2 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -66,6 +66,7 @@ MFE_OPEN_SOURCE_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}/opensource" MFE_TERMS_OF_SERVICE_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}/tos" MFE_PRIVACY_POLICY_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}/privacy" MFE_SEARCH_CATALOG_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}/courses" +MFE_LANGUAGE_PREFERENCE_COOKIE_NAME: "openedx-language-preference" MFE_LOGO_URL: "{{ COMMON_LOGO_URL }}" MFE_LOGO_TRADEMARK_URL: "{{ COMMON_LOGO_TRADEMARK_URL }}" @@ -132,6 +133,7 @@ MFE_ENVIRONMENT_DEFAULT: LOGO_TRADEMARK_URL: "{{ MFE_LOGO_TRADEMARK_URL }}" LOGO_WHITE_URL: "{{ MFE_LOGO_WHITE_URL }}" FAVICON_URL: "{{ MFE_FAVICON_URL }}" + LANGUAGE_PREFERENCE_COOKIE_NAME: "{{ MFE_LANGUAGE_PREFERENCE_COOKIE_NAME }}" MFE_STANDALONE_NGINX: true diff --git a/util/install/native.sh b/util/install/native.sh index 0b978ce535e..4d47db6327f 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -122,9 +122,7 @@ VERSION_VARS=( ECOMMERCE_WORKER_VERSION DISCOVERY_VERSION THEMES_VERSION - ACCOUNT_MFE_VERSION - GRADEBOOK_MFE_VERSION - PROFILE_MFE_VERSION + MFE_DEPLOY_VERSION ) for var in ${VERSION_VARS[@]}; do From 71a4df0bc8f6b6a65cb2a4c7e8e0ac819bd3120a Mon Sep 17 00:00:00 2001 From: Jhony Avella Date: Fri, 14 May 2021 15:02:50 -0500 Subject: [PATCH 037/664] feat: Add mfe_flags_setup role (#6406) New role creates waffle flags required to redirect to MFEs. Co-authored-by: Eric Herrera --- playbooks/mfe_flags_setup.yml | 11 ++++++ playbooks/openedx_native.yml | 1 + .../roles/mfe_flags_setup/defaults/main.yml | 4 ++ .../roles/mfe_flags_setup/tasks/main.yml | 37 +++++++++++++++++++ 4 files changed, 53 insertions(+) create mode 100644 playbooks/mfe_flags_setup.yml create mode 100644 playbooks/roles/mfe_flags_setup/defaults/main.yml create mode 100644 playbooks/roles/mfe_flags_setup/tasks/main.yml diff --git a/playbooks/mfe_flags_setup.yml b/playbooks/mfe_flags_setup.yml new file mode 100644 index 00000000000..36825eaf24b --- /dev/null +++ b/playbooks/mfe_flags_setup.yml @@ -0,0 +1,11 @@ +--- + +- name: Setup required MFE waffle flags + hosts: all + become: True + gather_facts: True + vars_files: + - "roles/common_vars/defaults/main.yml" + - "roles/edxapp/defaults/main.yml" + roles: + - role: mfe_flags_setup diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 924b5dd6a98..1457b5ac8ce 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -121,3 +121,4 @@ - role: user_retirement_pipeline when: COMMON_RETIREMENT_SERVICE_SETUP - role: mfe_deployer + - role: mfe_flags_setup diff --git a/playbooks/roles/mfe_flags_setup/defaults/main.yml b/playbooks/roles/mfe_flags_setup/defaults/main.yml new file mode 100644 index 00000000000..82ff5c998ec --- /dev/null +++ b/playbooks/roles/mfe_flags_setup/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +MFE_FLAGS_SETUP_FLAGS_LIST: + - account.redirect_to_microfrontend diff --git a/playbooks/roles/mfe_flags_setup/tasks/main.yml b/playbooks/roles/mfe_flags_setup/tasks/main.yml new file mode 100644 index 00000000000..450d74d0652 --- /dev/null +++ b/playbooks/roles/mfe_flags_setup/tasks/main.yml @@ -0,0 +1,37 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +# +# +# Tasks for role mfe_flags_setup +# +# Overview: +# +# +# Dependencies: +# +# +# Example play: +# +# + +- name: Get edxapp waffle flags list + shell: > + {{ edxapp_venv_bin }}/python {{ COMMON_BIN_DIR }}/manage.edxapp lms waffle_flag -l --settings={{ COMMON_EDXAPP_SETTINGS }} + become_user: "{{ edxapp_user }}" + environment: "{{ edxapp_environment }}" + register: edxapp_waffle_flags_list + +- name: Create MFE waffle flag if it does not exist + shell: > + {{ edxapp_venv_bin }}/python {{ COMMON_BIN_DIR }}/manage.edxapp lms waffle_flag {{ item }} --everyone --create --settings={{ COMMON_EDXAPP_SETTINGS }} + become_user: "{{ edxapp_user }}" + environment: "{{ edxapp_environment }}" + when: item not in edxapp_waffle_flags_list.stdout + loop: "{{ MFE_FLAGS_SETUP_FLAGS_LIST }}" From 5f4e1372ea0dacb1bae86f599363e3e557f1c5a6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 18 May 2021 16:54:15 -0400 Subject: [PATCH 038/664] fix: tubular should be installed at particular version Installations of Juniper were failing because tubular was always installing master, rather than the Juniper branch. This change will check out a particular version, to keep everything working. --- CHANGELOG.md | 8 ++++++-- .../roles/user_retirement_pipeline/defaults/main.yml | 4 +++- util/install/native.sh | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9dbbae44dd..68216faea28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,17 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-05-18 + - The version of tubular is controlled by RETIREMENT_SERVICE_VERSION. + Previously it was always "master", which broke older Open edX re-installations. + - 2021-05-13 - Role: edx_django_service - Added task that installs extra python packages specified in `edx_django_service_extra_requirements`. - Role: discovery - - Installs extra python packages specified in `DISCOVERY_EXTRA_REQUIREMENTS`. + - Installs extra python packages specified in `DISCOVERY_EXTRA_REQUIREMENTS`. - Role: ecommerce - - Installs extra python packages specified in `ECOMMERCE_EXTRA_REQUIREMENTS`. + - Installs extra python packages specified in `ECOMMERCE_EXTRA_REQUIREMENTS`. - 2021-03-08 - Remove instruction from ansile-bootstrap.sh that instructed people to activate diff --git a/playbooks/roles/user_retirement_pipeline/defaults/main.yml b/playbooks/roles/user_retirement_pipeline/defaults/main.yml index 73266838edf..d648d69d569 100644 --- a/playbooks/roles/user_retirement_pipeline/defaults/main.yml +++ b/playbooks/roles/user_retirement_pipeline/defaults/main.yml @@ -31,6 +31,8 @@ retirement_service_pip_version: "19.0.3" retirement_service_environment: PATH: '{{ retirement_service_venv_dir }}/bin:{{ ansible_env.PATH }}' +RETIREMENT_SERVICE_VERSION: "master" + # Set up git repos RETIREMENT_SERVICE_GIT_IDENTITY: !!null RETIREMENT_SERVICE_GIT_REPOS: @@ -38,7 +40,7 @@ RETIREMENT_SERVICE_GIT_REPOS: DOMAIN: "{{ COMMON_GIT_MIRROR }}" PATH: "{{ COMMON_GIT_PATH }}" REPO: "tubular.git" - VERSION: "master" + VERSION: "{{ RETIREMENT_SERVICE_VERSION }}" DESTINATION: "{{ retirement_service_app_dir }}" SSH_KEY: "{{ RETIREMENT_SERVICE_GIT_IDENTITY }}" diff --git a/util/install/native.sh b/util/install/native.sh index 4d47db6327f..277c1ece3c1 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -123,6 +123,7 @@ VERSION_VARS=( DISCOVERY_VERSION THEMES_VERSION MFE_DEPLOY_VERSION + RETIREMENT_SERVICE_VERSION ) for var in ${VERSION_VARS[@]}; do From 1543913ee8895c9f577badefbdadaad5277decf6 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 11 May 2021 10:56:55 -0400 Subject: [PATCH 039/664] fix: better decoding of ansible errors Some failed log lines say "FAILED", but some don't, depending on whether the task is loop or not. Sometimes an earlier FAILED message is actually ignored, but is found by the old check. Look for the leading "failed:" or "fatal:" lable instead to always get the actual error that stopped the playbook. --- util/install/native.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/util/install/native.sh b/util/install/native.sh index 4d47db6327f..e6c8bb47750 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -172,10 +172,8 @@ if [[ $ansible_status -ne 0 ]]; then echo "------------------------------------------------------------" echo " " echo "Decoded error:" - # Find the FAILED line before the "NO MORE HOSTS" line, and decode it. - # The plusses in the regex are because if I run this with -x, the awk line - # will be added to the log, and the regex would find itself if it didn't have plusses. - awk '/NO +MORE +HOSTS/{if (bad) print bad} /FAILED/{bad=$0}' $log_file | python3 /var/tmp/configuration/util/ansible_msg.py + # Find the last "failed" or "fatal" line and decode it. + awk '/^(failed|fatal):/{bad=$0} END {if (bad) print bad}' $log_file | python3 /var/tmp/configuration/util/ansible_msg.py echo " " echo "============================================================" echo "Installation failed!" From 18b8105eff446daedcd92e252565148717282db8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 20 May 2021 05:01:04 +0000 Subject: [PATCH 040/664] Update Plugins --- playbooks/roles/jenkins_build/defaults/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 897d59edbd0..38de36ed1c2 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -73,7 +73,7 @@ build_jenkins_plugins_list: version: '1.44' group: 'org.jenkins-ci.plugins' - name: 'credentials' - version: '2.3.15' + version: '2.4.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' version: '1.23' @@ -249,6 +249,9 @@ build_jenkins_plugins_list: - name: 'ssh-slaves' version: '1.28.1' group: 'org.jenkins-ci.plugins' + - name: 'structs' + version: '1.23' + group: 'org.jenkins-ci.plugins' - name: 'subversion' version: '2.13.2' group: 'org.jenkins-ci.plugins' From d7c5665b4fba8bda401dce2ca6aef31df3206bbc Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 24 May 2021 16:50:39 -0400 Subject: [PATCH 041/664] feat: split apart lms and cms django group management (#6410) In ``manage_edxapp_users_and_groups.yml`` playbook, allow LMS and CMS groups to be managed separately via ``manage-groups-lms`` and ``manage-groups-cms`` tags. These replace the ``manage-groups`` tag, which will be interpreted as ``manage-groups-lms`` until it is removed. TNL-8274 --- CHANGELOG.md | 7 ++ playbooks/manage_edxapp_users_and_groups.yml | 91 +++++++++++++++----- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68216faea28..d0702e0dbc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,15 @@ + # Changelog All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-05-24 + - In ``manage_edxapp_users_and_groups.yml`` playbook, allow LMS and CMS + groups to be managed separately via ``manage-groups-lms`` and + ``manage-groups-cms`` tags. These replace the ``manage-groups`` tag, + which will be interepreted as ``manage-groups-lms`` until it is removed. + - 2021-05-18 - The version of tubular is controlled by RETIREMENT_SERVICE_VERSION. Previously it was always "master", which broke older Open edX re-installations. diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 39b9dbb4940..170421e99a9 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -6,8 +6,8 @@ # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions # license: https://github.com/edx/configuration/blob/master/LICENSE.TXT # -# Usage: ansible-playbook -i lms-host-1, -e@/path/to/group/configfile -e@/path/to/user/configfile -# +# Usage: ansible-playbook -i edxapp-host-1, -e@/path/to/configfile-of-users-andor-groups +# -e 'group_environment=prod-edge' # Overview: # This playbook ensures that the specified users and groups exist in the targeted # edxapp cluster. @@ -15,25 +15,31 @@ # Users have the following properties: # - username (required, str) # - email (required, str) -# - groups (optional, list[str]) -# - superuser (optional, bool) -# - staff (optional, bool) +# - initial_password_hash (optional, str) # - remove (optional, bool): ensures the user does not exist +# - staff (optional, bool) +# - superuser (optional, bool) # - unusable_password (optional, bool): ensures the password is unusable +# - groups (optional, list[str]) +# - _comment (optional, str): ignored # # Groups can have the following properties: # - name (required, str) -# - permissions (optional, list[str]) +# - permissions (required, list[str]) # - remove (optional, bool): ensures the group does not exist +# - _comment (optional, str): ignored # # Example: # -# users: +# django_users: +# # - username: bobby # email: bobby@droptabl.es -# groups: [group1, group2] -# superuser: true # staff: true +# superuser: true +# groups: +# - group1 +# - group2 # # - username: fred # email: fred@smith @@ -41,7 +47,11 @@ # # - username: smitty # email: smitty@werbenmanjens.en -# groups: [group1] +# groups: +# - group1 +# _comment: | +# he was +# number one! # # - username: frank # email: frank@bigcorp.com @@ -54,19 +64,44 @@ # email: zoe@example.com # initial_password_hash: 'pbkdf2_sha256$20000$levJ6jdVYCsu$gdBLGf2DNPqfaKdcETXtFocRU8Kk+sMsIvKkmw1dKbY=' # -# groups: -# - name: group3 -# remove: true +# django_groups: # # - name: group1 # permissions: # - permission1 # - permission2 # -# - name: group2 -# permissions: [permission3] +# - name: group3 +# remove: true +# permissions: [] +# _comment: | +# group3 is the best group +# yada yada +# +# Note: +# +# LMS and CMS do still share the edxapp database, and therefore share a +# User table and Group table. So, edxapp users are created in an LMS context +# but exist in CMS as well. +# +# However, some edxapp Django apps are only installed into CMS (but not LMS), +# and vice versa. In order to create a group and grant it permissions, +# the permissions must be from apps that are installed into the running +# service. So, to create groups for LMS-only apps, we must create groups +# in an LMS context, and to create groups for CMS-only apps, we must create +# groups in a CMS context. Thus, while users are managed jointly for LMS/CMS, +# groups are managed separately. +# +# That being said, note that the groups created in one service variant should be +# disjoint with those created the other, as the underlying Group table is shared. +# That is, each group name should be defined for LMS *or* CMS, not both. +# Otherwise, whichever group is created second will override the first one. # -# NB: to get a list of all available permissions, run the following code: +# Of course, the jointly-managed LMS/CMS users can be assigned to any combination +# of both LMS and CMS groups. Assigning users to CMS groups does in fact work through +# an LMS context, since the actual CMS permissions are not being referenced. +# +# Note: to get a list of all available permissions, run the following code within a Django shell: # # from django.contrib.auth.models import Permission # for perm in Permission.objects.all(): @@ -79,22 +114,38 @@ manage_path: /edx/bin/manage.edxapp ignore_user_creation_errors: no deployment_settings: "{{ EDXAPP_SETTINGS | default('production') }}" + group_environment: "" # By default, create groups for all envs (for backwards compatibility). vars_files: - roles/common_vars/defaults/main.yml tasks: - - name: Manage groups + - name: Manage LMS groups tags: - - manage-groups + - manage-groups-lms + - manage-groups # Old tag pre-lms/cms-group-split, can be removed after TNL-8274. shell: > . {{env_path}} && {{ python_path }} {{ manage_path }} lms --settings={{ deployment_settings }} manage_group {{ item.name | quote }} {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" + when: (not group_environment) or group_environment in item.environments + become: true + become_user: "{{ common_web_user }}" + + - name: Manage CMS groups + tags: + - manage-groups-cms + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} cms --settings={{ deployment_settings }} + manage_group {{ item.name | quote }} + {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + with_items: "{{ django_groups }}" + when: (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" - - name: Manage active users + - name: Manage active LMS/CMS users tags: - manage-active-users shell: > @@ -115,7 +166,7 @@ become: true become_user: "{{ common_web_user }}" - - name: Manage inactive users + - name: Manage inactive LMS/CMS users tags: - manage-inactive-users shell: > From 6edfb20661aa9332c13de3f8274d17f2f39f547f Mon Sep 17 00:00:00 2001 From: Saleem Latif Date: Fri, 7 May 2021 14:46:14 +0500 Subject: [PATCH 042/664] Added playbook, role and docker file to setup/test tableau server. --- docker/build/tableau/Dockerfile | 27 +++++ docker/build/tableau/ansible_overrides.yml | 11 ++ docker/plays/tableau.yml | 10 ++ playbooks/roles/tableau/defaults/main.yml | 52 +++++++++ playbooks/roles/tableau/meta/main.yml | 3 + playbooks/roles/tableau/tasks/main.yml | 105 ++++++++++++++++++ .../tableau/templates/registration.json.j2 | 1 + playbooks/roles/tableau/templates/secrets.j2 | 21 ++++ playbooks/tableau.yml | 10 ++ 9 files changed, 240 insertions(+) create mode 100644 docker/build/tableau/Dockerfile create mode 100644 docker/build/tableau/ansible_overrides.yml create mode 100644 docker/plays/tableau.yml create mode 100644 playbooks/roles/tableau/defaults/main.yml create mode 100644 playbooks/roles/tableau/meta/main.yml create mode 100644 playbooks/roles/tableau/tasks/main.yml create mode 100644 playbooks/roles/tableau/templates/registration.json.j2 create mode 100644 playbooks/roles/tableau/templates/secrets.j2 create mode 100644 playbooks/tableau.yml diff --git a/docker/build/tableau/Dockerfile b/docker/build/tableau/Dockerfile new file mode 100644 index 00000000000..93b433f108a --- /dev/null +++ b/docker/build/tableau/Dockerfile @@ -0,0 +1,27 @@ +# To build this Dockerfile: +# +# From the root of configuration: +# +# docker build -f docker/build/tableau/Dockerfile . +# +# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible +# with the currently checked-out configuration repo. + +ARG BASE_IMAGE_TAG=latest +FROM edxops/focal-common:${BASE_IMAGE_TAG} +LABEL maintainer="edxops" +USER root + +ADD . /edx/app/edx_ansible/edx_ansible +WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays + +COPY docker/build/edxapp/ansible_overrides.yml / + +ARG OPENEDX_RELEASE=master +ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} +RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook tableau.yml \ + -c local -i '127.0.0.1,' \ + -t 'install,assets,devstack' \ + --extra-vars="@/ansible_overrides.yml" + +EXPOSE 8850 80 diff --git a/docker/build/tableau/ansible_overrides.yml b/docker/build/tableau/ansible_overrides.yml new file mode 100644 index 00000000000..582ed531c9a --- /dev/null +++ b/docker/build/tableau/ansible_overrides.yml @@ -0,0 +1,11 @@ +--- +TABLEAU_ADMIN_USER: tableau +TABLEAU_ADMIN_PASSWORD: 'password' +TABLEAU_SERVER_ADMIN_USER: "tableau-admin" +TABLEAU_SERVER_ADMIN_PASSWORD: "password" + +TABLEAU_REGISTRATION_CONFIG_USER_FIRST_NAME: "Saleem" +TABLEAU_REGISTRATION_CONFIG_USER_LAST_NAME: "Latif" +TABLEAU_REGISTRATION_CONFIG_USER_TITLE: "Software Engineer" +TABLEAU_REGISTRATION_CONFIG_USER_EMAIL: "saleem@edx.org" +TABLEAU_REGISTRATION_CONFIG_USER_PHONE: "" diff --git a/docker/plays/tableau.yml b/docker/plays/tableau.yml new file mode 100644 index 00000000000..bb828367e03 --- /dev/null +++ b/docker/plays/tableau.yml @@ -0,0 +1,10 @@ +- name: Deploy tableau + hosts: all + become: True + gather_facts: True + vars: + serial_count: 1 + CLUSTER_NAME: 'tableau' + serial: "{{ serial_count }}" + roles: + - tableau diff --git a/playbooks/roles/tableau/defaults/main.yml b/playbooks/roles/tableau/defaults/main.yml new file mode 100644 index 00000000000..d243ce25d73 --- /dev/null +++ b/playbooks/roles/tableau/defaults/main.yml @@ -0,0 +1,52 @@ +# variables common to the tableau role, automatically loaded +# when the role is included +--- + +# Variables in all caps are environment specific +# Lowercase variables are internal to the role +# +# Defaults specified here should not contain +# any secrets or host identifying information. +# + +tableau_data_dir: "{{ COMMON_DATA_DIR }}/tableau" +tableau_app_dir: "{{ COMMON_APP_DIR }}/tableau" +tableau_log_dir: "{{ COMMON_LOG_DIR }}/tableau" +tableau_code_dir: "{{ tableau_app_dir }}/tableau" +tableau_installer_dir: "{{ tableau_code_dir }}/linux/automated-installer/" + +TABLEAU_ADMIN_USER: tableau +TABLEAU_ADMIN_PASSWORD: 'tableau001-pass' +tableau_user: "{{ TABLEAU_ADMIN_USER }}" +tableau_user_createhome: 'yes' +tableau_user_shell: '/bin/false' + +tableau_repo: "https://github.com/tableau/server-install-script-samples.git" +tableau_version: 'master' + +tableau_server_deb_pkg: 'https://downloads.tableau.com/esdalt/2020.4.5/tableau-server-2020-4-5_amd64.deb' +tableau_server_deb_pkg_name: 'tableau-server-amd64.deb' + +TABLEAU_SERVER_ADMIN_USER: "tableau001" +TABLEAU_SERVER_ADMIN_PASSWORD: "tableau001-pass" + +TABLEAU_REGISTRATION_CONFIG_USER_FIRST_NAME: "Mike" +TABLEAU_REGISTRATION_CONFIG_USER_LAST_NAME: "OConnell" +TABLEAU_REGISTRATION_CONFIG_USER_TITLE: "Engineering Manager" +TABLEAU_REGISTRATION_CONFIG_USER_EMAIL: "moconnell@edx.org" +TABLEAU_REGISTRATION_CONFIG_USER_PHONE: "" + +tableau_registration_config: + zip: "02141" + country: "USA" + city: "Cambridge" + last_name: "{{ TABLEAU_REGISTRATION_CONFIG_USER_LAST_NAME }}" + industry: "Engineering" + eula: "Yes" + title: "{{ TABLEAU_REGISTRATION_CONFIG_USER_TITLE }}" + phone: "{{ TABLEAU_REGISTRATION_CONFIG_USER_PHONE }}" + company: "edX, Inc" + state: "MA" + department: "Enterprise" + first_name: "{{ TABLEAU_REGISTRATION_CONFIG_USER_FIRST_NAME }}" + email: "{{ TABLEAU_REGISTRATION_CONFIG_USER_EMAIL }}" diff --git a/playbooks/roles/tableau/meta/main.yml b/playbooks/roles/tableau/meta/main.yml new file mode 100644 index 00000000000..2083f0e1251 --- /dev/null +++ b/playbooks/roles/tableau/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - common diff --git a/playbooks/roles/tableau/tasks/main.yml b/playbooks/roles/tableau/tasks/main.yml new file mode 100644 index 00000000000..5b93f0aca3a --- /dev/null +++ b/playbooks/roles/tableau/tasks/main.yml @@ -0,0 +1,105 @@ +# requires: +# - group_vars/all +# - common/tasks/main.yml +--- +- name: create application user + user: + name: "{{ tableau_user }}" + home: "{{ tableau_app_dir }}" + createhome: "{{ tableau_user_createhome }}" + shell: "{{ tableau_user_shell }}" + tags: + - install + - install:base + +- name: create tableau user dirs + file: + path: "{{ item.path }}" + state: directory + owner: "{{ tableau_user }}" + group: "{{ common_web_group }}" + mode: "{{ item.mode | default(0755) }}" + with_items: + - { path: "{{ tableau_app_dir }}" } + # needed for the ansible 1.5 git module + - { path: "{{ tableau_app_dir }}/.ssh" } + - { path: "{{ tableau_data_dir }}" } + tags: + - install + - install:base + +- name: create tableau log dir + file: + path: "{{ tableau_log_dir }}" + state: directory + owner: "{{ common_log_user }}" + group: "{{ common_log_user }}" + tags: + - install + - install:base + +- name: set git fetch.prune to ignore deleted remote refs + shell: git config --global fetch.prune true + become_user: "{{ tableau_user }}" + tags: + - install + - install:base + +# Clone tableau repo containing install scripts. +- name: checkout tableau repo into {{ tableau_code_dir }} + git: + dest: "{{ tableau_code_dir }}" + repo: "{{ tableau_repo }}" + version: "{{ tableau_version }}" + accept_hostkey: yes + become_user: "{{ tableau_user }}" + register: tableau_checkout + tags: + - install + - install:code + +- name: git clean after checking out tableau repo + shell: cd {{ tableau_code_dir }} && git clean -xdf + become_user: "{{ tableau_user }}" + tags: + - install + - install:code + +- name: Download tableau server + get_url: + url: "{{ tableau_server_deb_pkg }}" + dest: "{{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }}" + become_user: "{{ tableau_user }}" + tags: + - install + - install:tableau-server + +- name: copy the template to the desired location + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + with_items: + - { src: 'secrets.j2', dest: '{{ tableau_installer_dir }}/secrets', owner: '{{ tableau_user }}', group: '{{ common_web_user }}', mode: '0644' } + - { src: 'registration.json.j2', dest: '{{ tableau_installer_dir }}/registration.json', owner: '{{ tableau_user }}', group: '{{ common_web_user }}', mode: '0644' } + tags: + - install + - install:base + +- name: Make automated installer executable + file: + dest: "{{ tableau_installer_dir }}/automated-installer" + mode: "a+x" + become_user: "{{ tableau_user }}" + tags: + - install + - install:code + +- name: Run automated installer to actually setup the tableau server + shell: "{{ tableau_installer_dir }}/automated-installer -s {{ tableau_installer_dir }}/secrets -f {{ tableau_installer_dir }}/config.json -r {{ tableau_installer_dir }}/registration.json --accepteula {{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }}" + sudo: true + tags: + - install + - install:code diff --git a/playbooks/roles/tableau/templates/registration.json.j2 b/playbooks/roles/tableau/templates/registration.json.j2 new file mode 100644 index 00000000000..ad432eb7910 --- /dev/null +++ b/playbooks/roles/tableau/templates/registration.json.j2 @@ -0,0 +1 @@ +{{ tableau_registration_config | to_nice_json }} diff --git a/playbooks/roles/tableau/templates/secrets.j2 b/playbooks/roles/tableau/templates/secrets.j2 new file mode 100644 index 00000000000..83c1e2f4cd1 --- /dev/null +++ b/playbooks/roles/tableau/templates/secrets.j2 @@ -0,0 +1,21 @@ +# Note: If you do not enter the tsm_admin_pass or the +# tableau_server_admin_pass in this file, you will be prompted to enter this +# information during installation. However, you must enter the account names +# for tsm_admin_user and tableau_server_admin_user. +# Credentials for the account that is running the automated-installer script. +# This account will be added to the 'tsmadmin' group. The 'tsmadmin' group is +# created during the Tableau installation process. Members of the 'tsmadmin' +# group can run TSM commands. +# +tsm_admin_user="{{ TABLEAU_ADMIN_USER }}" +tsm_admin_pass="{{ TABLEAU_ADMIN_PASSWORD }}" + +# Enter a username and password to create the initial Tableau administrator +# account. This account will be created in Tableau Server by the installation +# process and will have Tableau Server administrator rights. The user account +# will be local to Tableau Server and will not be a Linux OS account. If you +# are using LDAP or AD for authentication, then the account you specify for +# the Tableau administrator must be a valid account from the directory service. +# +tableau_server_admin_user="{{ TABLEAU_SERVER_ADMIN_USER }}" +tableau_server_admin_pass="{{ TABLEAU_SERVER_ADMIN_PASSWORD }}" diff --git a/playbooks/tableau.yml b/playbooks/tableau.yml new file mode 100644 index 00000000000..bb828367e03 --- /dev/null +++ b/playbooks/tableau.yml @@ -0,0 +1,10 @@ +- name: Deploy tableau + hosts: all + become: True + gather_facts: True + vars: + serial_count: 1 + CLUSTER_NAME: 'tableau' + serial: "{{ serial_count }}" + roles: + - tableau From 163253b4f8f8feafcc86a761d2aba45efb2ba502 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 26 May 2021 13:15:50 -0400 Subject: [PATCH 043/664] WS-1825 remove pip from prospectus build (#6424) --- playbooks/roles/prospectus/tasks/main.yml | 40 ++++------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 015943f0c83..a0b1499bbc2 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -74,33 +74,6 @@ - install - install:system-requirements -- name: build virtualenv with python2.7 - command: "virtualenv --python=python2.7 {{ prospectus_venv_dir }}" - args: - creates: "{{ prospectus_venv_dir }}/bin/pip" - become_user: "{{ prospectus_user }}" - when: not prospectus_use_python3 - tags: - - install - - install:system-requirements - -- name: build virtualenv with python3.8 - command: "virtualenv --python=python3.8 {{ prospectus_venv_dir }}" - args: - creates: "{{ prospectus_venv_dir }}/bin/pip" - become_user: "{{ prospectus_user }}" - when: prospectus_use_python3 - tags: - - install - - install:system-requirements - -- name: Pin pip to a specific version. - command: "{{ prospectus_venv_dir }}/bin/pip install pip=={{ COMMON_PIP_VERSION }}" - become_user: "{{ prospectus_user }}" - tags: - - install - - install:system-requirements - - name: Add prospectus configuration file template: src: ".env.environment.j2" @@ -109,10 +82,9 @@ owner: "{{ prospectus_user }}" group: "{{ prospectus_user }}" -# NOTE (CCB): Ideally we should use the pip Ansible command, -# but that doesn't seem to work with the Python 3.x virtualenv. -- name: install nodenv - command: pip install nodeenv +- name: Install nodeenv + apt: + name: nodeenv become_user: "{{ prospectus_user }}" environment: "{{ prospectus_env_vars }}" tags: @@ -120,8 +92,10 @@ - install:system-requirements # Install node -- name: create nodeenv - shell: "{{ prospectus_venv_dir }}/bin/nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" +- name: Create nodeenv + shell: "nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" + become_user: "{{ prospectus_user }}" + environment: "{{ prospectus_env_vars }}" tags: - install - install:system-requirements From f54e62fc8157468381061e027da05b73772f6913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Sun, 16 May 2021 10:01:40 +0200 Subject: [PATCH 044/664] feat: add celerybeat configuration --- playbooks/roles/edxapp/defaults/main.yml | 3 +++ .../edx/app/supervisor/conf.d.available/workers.conf.j2 | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 057d299f09b..b0ed6b13b9f 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -173,6 +173,8 @@ EDXAPP_CELERY_BROKER_VHOST: "" EDXAPP_CELERY_BROKER_USE_SSL: false EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" +# PersistentScheduler is the default celery beat scheduler +EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" EDXAPP_BRANCH_IO_KEY: "" EDXAPP_AUTH_USE_OPENID_PROVIDER: true @@ -1417,6 +1419,7 @@ generic_env_config: &edxapp_generic_env LOCATION: "{{ EDXAPP_CACHE_COURSE_STRUCTURE_MEMCACHE }}" # Default to two hours TIMEOUT: "7200" + CELERYBEAT_SCHEDULER: "{{ EDXAPP_CELERYBEAT_SCHEDULER }}" CELERY_BROKER_TRANSPORT: "{{ EDXAPP_CELERY_BROKER_TRANSPORT }}" CELERY_BROKER_HOSTNAME: "{{ EDXAPP_CELERY_BROKER_HOSTNAME }}" COMMENTS_SERVICE_URL: "{{ EDXAPP_COMMENTS_SERVICE_URL }}" diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 1d96f0785ef..8ac7fa57b75 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -7,7 +7,7 @@ directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} +command={{ edxapp_app_dir }}/worker.sh --beat --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }} ; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit From ca3b4f99d8bfab02a41711960c4dd654a5aab875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Thu, 20 May 2021 15:00:04 +0200 Subject: [PATCH 045/664] chore: remove comment --- playbooks/roles/edxapp/defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index b0ed6b13b9f..9f5a0e13e19 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -173,7 +173,6 @@ EDXAPP_CELERY_BROKER_VHOST: "" EDXAPP_CELERY_BROKER_USE_SSL: false EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" -# PersistentScheduler is the default celery beat scheduler EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" EDXAPP_BRANCH_IO_KEY: "" From 1f01ce9f55f693c582250a598149317ff37954fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Thu, 27 May 2021 08:06:00 +0200 Subject: [PATCH 046/664] refactor: use boolean flag to enable or disable celery beat Co-authored-by: Joseph Mulloy --- playbooks/roles/edxapp/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 9f5a0e13e19..01bb8b87d12 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -174,6 +174,7 @@ EDXAPP_CELERY_BROKER_USE_SSL: false EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" +EDXAPP_ENABLE_CELERY_BEAT: false EDXAPP_BRANCH_IO_KEY: "" EDXAPP_AUTH_USE_OPENID_PROVIDER: true From 7821749bf0425a0c3229b67b54ceabd1c95d74e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Thu, 27 May 2021 08:06:06 +0200 Subject: [PATCH 047/664] refactor: use boolean flag to enable or disable celery beat Co-authored-by: Joseph Mulloy --- .../edx/app/supervisor/conf.d.available/workers.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 8ac7fa57b75..13a68d58bce 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -7,7 +7,7 @@ directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/worker.sh --beat --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} +command={{ edxapp_app_dir }}/worker.sh {{ '--beat' if EDXAPP_ENABLE_CELERY_BEAT|bool else '' }} --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }} ; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit From ad7c796ba74982709428c0494a133d123cd2abd9 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Wed, 2 Jun 2021 05:08:11 +0200 Subject: [PATCH 048/664] Switch to mongo_4_0 role in openedx_native.yml (#6416) --- playbooks/openedx_native.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 1457b5ac8ce..9498e0970e0 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -71,7 +71,7 @@ when: EDXAPP_MYSQL_HOST == 'localhost' - role: memcache when: EDXAPP_ENABLE_MEMCACHE - - role: mongo_3_6 + - role: mongo_4_0 when: "'localhost' in EDXAPP_MONGO_HOSTS" - role: redis when: SANDBOX_ENABLE_REDIS From 41435afe3164d8be92a0386fac4ce9e7cb9a1cc1 Mon Sep 17 00:00:00 2001 From: "M. Zulqarnain" Date: Wed, 2 Jun 2021 15:27:45 +0500 Subject: [PATCH 049/664] feat: upgrade pip to 20.3.4 (#6412) --- docker/build/analytics_pipeline/Dockerfile | 2 +- playbooks/roles/common_vars/defaults/main.yml | 2 +- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- pre-requirements.txt | 2 +- util/install/ansible-bootstrap.sh | 2 +- util/install/native.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/build/analytics_pipeline/Dockerfile b/docker/build/analytics_pipeline/Dockerfile index bcdc6be0e1c..8dbd4ccb094 100644 --- a/docker/build/analytics_pipeline/Dockerfile +++ b/docker/build/analytics_pipeline/Dockerfile @@ -29,7 +29,7 @@ ENV BOTO_CONFIG=/dev/null \ ANALYTICS_PIPELINE_VENV=/edx/app/analytics_pipeline/venvs \ BOOTSTRAP=/etc/bootstrap.sh \ COMMON_BASE_DIR=/edx \ - COMMON_PIP_PACKAGES_PIP='pip==20.1.1' \ + COMMON_PIP_PACKAGES_PIP='pip==20.3.4' \ COMMON_PIP_PACKAGES_SETUPTOOLS='setuptools==44.1.0' \ COMMON_PIP_PACKAGES_VIRTUALENV='virtualenv==20.1.0' \ COMMON_MYSQL_READ_ONLY_USER='read_only' \ diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 5d6515e43e5..4949012d8aa 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -149,7 +149,7 @@ old_python_debian_pkgs: - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1" -COMMON_PIP_VERSION: '20.1.1' +COMMON_PIP_VERSION: '20.3.4' common_pip_pkgs: - pip=={{ COMMON_PIP_VERSION }} diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index a68375f54e3..ffef1ba5a15 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -275,7 +275,7 @@ - name: Pin pip to a specific version. # Not pinning to the same version as everything else because sandboxes are still python 2.7 - command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==20.1.1" + command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==20.3.4" args: chdir: "{{ edxapp_code_dir }}" become_user: "{{ edxapp_sandbox_user }}" diff --git a/pre-requirements.txt b/pre-requirements.txt index 23dcf46d410..a9a3cb756e0 100644 --- a/pre-requirements.txt +++ b/pre-requirements.txt @@ -1 +1 @@ -pip==20.1.1 +pip==20.3.4 diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 5d1f04be146..5e0ca53020a 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -33,7 +33,7 @@ fi # Bootstrapping constants # VIRTUAL_ENV_VERSION="16.7.10" -PIP_VERSION="20.1.1" +PIP_VERSION="20.3.4" SETUPTOOLS_VERSION="44.1.0" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" diff --git a/util/install/native.sh b/util/install/native.sh index 5e6608af5e4..cd93446eea8 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -101,7 +101,7 @@ sudo apt-get upgrade -y sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python3-pip libmysqlclient-dev python3-apt python3-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ # ansible-bootstrap installs yaml that pip 19 can't uninstall. sudo apt-get remove -y python-yaml -sudo pip3 install --upgrade pip==20.1.1 +sudo pip3 install --upgrade pip==20.3.4 sudo pip3 install --upgrade setuptools==44.1.0 sudo -H pip3 install --upgrade virtualenv==20.2.0 From 851f7692ab81ccf678874c47299d6bbb24193a91 Mon Sep 17 00:00:00 2001 From: "M. Zulqarnain" Date: Wed, 2 Jun 2021 18:23:03 +0500 Subject: [PATCH 050/664] Fix: xblock-vectordraw & xblock-activetable name fix (#6437) * fix: xblock-vectordraw & xblock-activetable name fix --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 01bb8b87d12..95cb8edca0f 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -549,9 +549,9 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/ubc/ubcpi.git@1.0.0#egg=ubcpi-xblock extra_args: -e # Vector Drawing and ActiveTable XBlocks (Davidson) - - name: git+https://github.com/open-craft/xblock-vectordraw.git@76976425356dfc7f13570f354c0c438db84c2840#egg=xblock-vectordraw==0.3.0 + - name: git+https://github.com/open-craft/xblock-vectordraw.git@76976425356dfc7f13570f354c0c438db84c2840#egg=vectordraw-xblock==0.3.0 extra_args: -e - - name: git+https://github.com/open-craft/xblock-activetable.git@013003aa3ce28f0ae03b8227dc3a6daa4e19997d#egg=xblock-activetable + - name: git+https://github.com/open-craft/xblock-activetable.git@013003aa3ce28f0ae03b8227dc3a6daa4e19997d#egg=activetable-xblock extra_args: -e - name: edx-zoom==1.8 # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) From d3233873b95a207289262fe6dca0382d0245be4b Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Thu, 3 Jun 2021 09:13:44 +0930 Subject: [PATCH 051/664] [FAL-1792] fix: required jenkins package no longer supported (#6422) * fix: required jenkins package no longer supported The Jenkins packages available from pkg.jenkins.io no longer support the command-line scripts run by jenkins_analytics. Need to download the old Jenkins package from the archives mirror site instead. Co-authored-by: Joseph Mulloy --- playbooks/roles/jenkins_master/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_master/defaults/main.yml b/playbooks/roles/jenkins_master/defaults/main.yml index 1298a2f1cf1..e92ed248ace 100644 --- a/playbooks/roles/jenkins_master/defaults/main.yml +++ b/playbooks/roles/jenkins_master/defaults/main.yml @@ -8,7 +8,7 @@ jenkins_protocol_https: true jenkins_job_venv_dir: "/edx/var/jenkins/jobvenvs/" JENKINS_VERSION: '1.651.3' -jenkins_deb_url: "https://pkg.jenkins.io/debian-stable/binary/jenkins_{{ JENKINS_VERSION }}_all.deb" +jenkins_deb_url: "https://archives.jenkins-ci.org/debian-stable/jenkins_{{ JENKINS_VERSION }}_all.deb" jenkins_deb: "jenkins_{{ JENKINS_VERSION }}_all.deb" # Jenkins jvm args are set when starting the Jenkins service, e.g., "-Xmx1024m" jenkins_jvm_args: "" From 9689224f0676decca76b3acebb07e1c8699e73b3 Mon Sep 17 00:00:00 2001 From: Peter Pinch <430126+pdpinch@users.noreply.github.com> Date: Wed, 26 May 2021 08:06:49 -0400 Subject: [PATCH 052/664] refactor: remove obsolete setting ENABLE_INSTRUCTOR_ANALYTICS ENABLE_INSTRUCTOR_ANALYTICS was removed from edx-platform in 2015. --- CHANGELOG.md | 3 +++ docker/build/edxapp/lms.yml | 1 - docker/build/edxapp/studio.yml | 1 - playbooks/roles/edxapp/defaults/main.yml | 2 -- playbooks/sample_vars/server_vars.yml | 1 - 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0702e0dbc5..87e4e61b89a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-06-05 + - Remove ENABLE_INSTRUCTOR_ANALYTICS setting, which was removed from edx-platform in 2015 + - 2021-05-24 - In ``manage_edxapp_users_and_groups.yml`` playbook, allow LMS and CMS groups to be managed separately via ``manage-groups-lms`` and diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index d8ae9e5bf27..9e0a085ac23 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -284,7 +284,6 @@ FEATURES: ENABLE_EDXNOTES: true ENABLE_ENROLLMENT_RESET: false ENABLE_GRADE_DOWNLOADS: true - ENABLE_INSTRUCTOR_ANALYTICS: false ENABLE_MKTG_SITE: false ENABLE_MOBILE_REST_API: false ENABLE_OAUTH2_PROVIDER: false diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index fae9165f9ad..f8136ee5364 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -250,7 +250,6 @@ FEATURES: ENABLE_EDXNOTES: true ENABLE_ENROLLMENT_RESET: false ENABLE_GRADE_DOWNLOADS: true - ENABLE_INSTRUCTOR_ANALYTICS: false ENABLE_MKTG_SITE: false ENABLE_MOBILE_REST_API: false ENABLE_OAUTH2_PROVIDER: false diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 95cb8edca0f..3e1ac091713 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -185,7 +185,6 @@ EDXAPP_ENABLE_CROSS_DOMAIN_CSRF_COOKIE: false EDXAPP_ENABLE_DISCUSSION_HOME_PANEL: true EDXAPP_ENABLE_DISCUSSION_SERVICE: true EDXAPP_ENABLE_GRADE_DOWNLOADS: true -EDXAPP_ENABLE_INSTRUCTOR_ANALYTICS: false EDXAPP_ENABLE_SPECIAL_EXAMS: false EDXAPP_ENABLE_VIDEO_UPLOAD_PIPELINE: false @@ -315,7 +314,6 @@ EDXAPP_ENABLE_ENROLLMENT_RESET: false EDXAPP_FEATURES_DEFAULT: AUTH_USE_OPENID_PROVIDER: "{{ EDXAPP_AUTH_USE_OPENID_PROVIDER }}" ENABLE_DISCUSSION_SERVICE: "{{ EDXAPP_ENABLE_DISCUSSION_SERVICE }}" - ENABLE_INSTRUCTOR_ANALYTICS: "{{ EDXAPP_ENABLE_INSTRUCTOR_ANALYTICS }}" PREVIEW_LMS_BASE: "{{ EDXAPP_PREVIEW_LMS_BASE }}" ENABLE_GRADE_DOWNLOADS: "{{ EDXAPP_ENABLE_GRADE_DOWNLOADS }}" ENABLE_MKTG_SITE: "{{ EDXAPP_ENABLE_MKTG_SITE }}" diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index 624933d8aef..e01ddc83f95 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -91,7 +91,6 @@ #EDXAPP_FEATURES: # ENABLE_DISCUSSION_SERVICE: true # ENABLE_DISCUSSION_HOME_PANEL: true -# ENABLE_INSTRUCTOR_ANALYTICS: false # SUBDOMAIN_BRANDING: false # SUBDOMAIN_COURSE_LISTINGS: false # PREVIEW_LMS_BASE: "{{ EDXAPP_PREVIEW_LMS_BASE }}" From 77a0b37aa6df562087ba6e2f298dad33f46132c4 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 3 Jun 2021 09:40:37 +0200 Subject: [PATCH 053/664] Add CREDENTIALS_BASE_URL to MFE environment. --- playbooks/roles/mfe/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 5f1e8c019f2..02c5a9bf0e8 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -58,6 +58,7 @@ MFE_SITE_NAME: "" MFE_MARKETING_SITE_BASE_URL: "{{ MFE_LMS_BASE_URL }}" MFE_CONTACT_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}/contact" MFE_CSRF_TOKEN_API_PATH: "/csrf/api/v1/token" +MFE_CREDENTIALS_BASE_URL: "" MFE_REFRESH_ACCESS_TOKEN_ENDPOINT: "{{ MFE_LMS_BASE_URL }}/login_refresh" MFE_DATA_API_BASE_URL: "{{ MFE_LMS_BASE_URL }}" MFE_ACCESS_TOKEN_COOKIE_NAME: "{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}" @@ -105,6 +106,7 @@ MFE_ENVIRONMENT_DEFAULT: MARKETING_SITE_BASE_URL: "{{ MFE_MARKETING_SITE_BASE_URL }}" CONTACT_URL: "{{ MFE_CONTACT_URL }}" CSRF_TOKEN_API_PATH: "{{ MFE_CSRF_TOKEN_API_PATH }}" + CREDENTIALS_BASE_URL: "{{ MFE_CREDENTIALS_BASE_URL }}" REFRESH_ACCESS_TOKEN_ENDPOINT: "{{ MFE_REFRESH_ACCESS_TOKEN_ENDPOINT }}" DATA_API_BASE_URL: "{{ MFE_DATA_API_BASE_URL }}" ACCESS_TOKEN_COOKIE_NAME: "{{ MFE_ACCESS_TOKEN_COOKIE_NAME }}" From 1909bafde44584ec7bd04f7cb6ee6ac0bfa6b521 Mon Sep 17 00:00:00 2001 From: Max Sokolski Date: Tue, 8 Jun 2021 04:58:40 +0300 Subject: [PATCH 054/664] feat: Payment and Ecommerce MFEs support in Native Installation (#6408) * feat: Payment and Ecommerce MFEs support in Native Installation * feat: install ecommerce mfes only if ecommerce is deployed * docs: update CHANGELOG.md and mfe_deployer README.rst Co-authored-by: Jillian Vogel --- CHANGELOG.md | 20 ++++++++ playbooks/openedx_native.yml | 15 ++++++ playbooks/roles/ecommerce/defaults/main.yml | 23 ++++++++- playbooks/roles/mfe_deployer/README.rst | 47 +++++++++++++++++++ .../roles/mfe_deployer/defaults/main.yml | 14 ++++++ playbooks/roles/mfe_deployer/tasks/main.yml | 2 +- .../app/nginx/sites-available/concerns/mfe.j2 | 2 +- .../roles/mfe_flags_setup/defaults/main.yml | 1 + 8 files changed, 121 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e4e61b89a..411beb4b2b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,26 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-06-07 + - In `openedx_native.yml` + - Added configuration variable ECOMMERCE_CSRF_TRUSTED_ORIGINS to allow payment mfe to interact with ecommerce service + - Added configuration variable ECOMMERCE_CORS_ORIGIN_WHITELIST to allow cross domain interation between mfes and ecommerce service + - Added new conditional variable MFE_DEPLOY_ECOMMERCE_MFES to not build ecommerce related MFEs w/o ecommerce service + - Created SiteConfiguration for default Site to enable ecommerce MFE + - Added configuration variable EDXAPP_ORDER_HISTORY_MICROFRONTEND_URL + - Set ECOMMERCE_CORS_ALLOW_CREDENTIALS to true + - Added new configuration variable ECOMMERCE_ENABLE_PAYMENT_MFE + - Role ecommerce + - Added new configuration variable ECOMMERCE_ENABLE_PAYMENT_MFE with default value to false + - Updated `create_or_update_site` management command to set `enable-microfrontend-for-basket-page` and `payment-microfrontend-url` flags + - Role mfe_deployer + - Added MFES_ECOMMERCE list for ecommerce related MFEs + - Added new configuration variable MFE_DEPLOY_ECOMMERCE_MFES + - Added new deploy_mfes variable to collect list of all MFEs to deploy + - Changed looping from `MFES` to `deploy_mfes` list internally + - Role mfe_flags_setup + - Added new flag `order_history.redirect_to_microfrontend` + - 2021-06-05 - Remove ENABLE_INSTRUCTOR_ANALYTICS setting, which was removed from edx-platform in 2015 diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 9498e0970e0..8b82a6e8032 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -32,6 +32,11 @@ EDXAPP_ACCOUNT_MICROFRONTEND_URL: "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}/account" EDXAPP_LMS_NGINX_PORT: '80' EDX_PLATFORM_VERSION: 'master' + EDXAPP_ORDER_HISTORY_MICROFRONTEND_URL: "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}/ecommerce/orders" + EDXAPP_SITE_CONFIGURATION: + - values: + ENABLE_ORDER_HISTORY_MICROFRONTEND: "{{ SANDBOX_ENABLE_ECOMMERCE }}" + # Set to false if deployed behind another proxy/load balancer. NGINX_SET_X_FORWARDED_HEADERS: True DISCOVERY_URL_ROOT: 'http://localhost:{{ DISCOVERY_NGINX_PORT }}' @@ -51,8 +56,17 @@ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false EDXAPP_ENABLE_MEMCACHE: true EDXAPP_ENABLE_ELASTIC_SEARCH: true + # Ecommerce + ECOMMERCE_CORS_ORIGIN_WHITELIST: [ + "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", + ] + ECOMMERCE_CSRF_TRUSTED_ORIGINS: [ + "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", + ] + ECOMMERCE_CORS_ALLOW_CREDENTIALS: true # For the mfe role. COMMON_ECOMMERCE_BASE_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}' + ECOMMERCE_ENABLE_PAYMENT_MFE: true roles: - role: swapfile SWAPFILE_SIZE: 4GB @@ -121,4 +135,5 @@ - role: user_retirement_pipeline when: COMMON_RETIREMENT_SERVICE_SETUP - role: mfe_deployer + MFE_DEPLOY_ECOMMERCE_MFES: "{{ SANDBOX_ENABLE_ECOMMERCE }}" - role: mfe_flags_setup diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 747acc43890..d7974c79a6e 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -201,6 +201,7 @@ ECOMMERCE_DISCOVERY_SERVICE_URL: 'http://localhost:8008' ECOMMERCE_ENTERPRISE_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}' ECOMMERCE_CORS_ORIGIN_WHITELIST: [] +ECOMMERCE_CSRF_TRUSTED_ORIGINS: [] ECOMMERCE_CORS_URLS_REGEX: '' ECOMMERCE_CORS_ALLOW_CREDENTIALS: false @@ -250,6 +251,7 @@ ecommerce_config: SESSION_COOKIE_SECURE: '{{ ECOMMERCE_SESSION_COOKIE_SECURE}}' CORS_ORIGIN_WHITELIST: "{{ ECOMMERCE_CORS_ORIGIN_WHITELIST }}" + CSRF_TRUSTED_ORIGINS: "{{ ECOMMERCE_CSRF_TRUSTED_ORIGINS }}" CORS_URLS_REGEX: "{{ ECOMMERCE_CORS_URLS_REGEX }}" CORS_ALLOW_CREDENTIALS: "{{ ECOMMERCE_CORS_ALLOW_CREDENTIALS }}" @@ -282,6 +284,9 @@ ECOMMERCE_HERMES_ENABLED: "{{ COMMON_HERMES_ENABLED }}" ECOMMERCE_DECRYPT_CONFIG_ENABLED: "{{ COMMON_DECRYPT_CONFIG_ENABLED }}" ECOMMERCE_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" +# MFEs default settings +ECOMMERCE_ENABLE_PAYMENT_MFE: false + # # vars are namespace with the module name. # @@ -336,7 +341,23 @@ ecommerce_redhat_pkgs: [] ecommerce_post_migrate_commands: - command: './manage.py oscar_populate_countries --initial-only' when: true - - command: './manage.py create_or_update_site --site-id=1 --site-domain={{ ECOMMERCE_ECOMMERCE_URL_ROOT.split("://")[1] }} --partner-code=edX --partner-name="Open edX" --lms-url-root={{ ECOMMERCE_LMS_URL_ROOT }} --client-side-payment-processor=cybersource --payment-processors=cybersource,paypal --sso-client-id={{ ECOMMERCE_SOCIAL_AUTH_EDX_OAUTH2_KEY }} --sso-client-secret={{ ECOMMERCE_SOCIAL_AUTH_EDX_OAUTH2_SECRET }} --backend-service-client-id={{ ECOMMERCE_BACKEND_SERVICE_EDX_OAUTH2_KEY }} --backend-service-client-secret={{ ECOMMERCE_BACKEND_SERVICE_EDX_OAUTH2_SECRET }} --from-email staff@example.com --discovery_api_url={{ ECOMMERCE_DISCOVERY_SERVICE_URL }}/api/v1/' + - command: > + ./manage.py create_or_update_site + --site-id=1 + --site-domain={{ ECOMMERCE_ECOMMERCE_URL_ROOT.split("://")[1] }} + --partner-code=edX --partner-name="Open edX" + --lms-url-root={{ ECOMMERCE_LMS_URL_ROOT }} + --client-side-payment-processor=cybersource + --payment-processors=cybersource,paypal + --sso-client-id={{ ECOMMERCE_SOCIAL_AUTH_EDX_OAUTH2_KEY }} + --sso-client-secret={{ ECOMMERCE_SOCIAL_AUTH_EDX_OAUTH2_SECRET }} + --backend-service-client-id={{ ECOMMERCE_BACKEND_SERVICE_EDX_OAUTH2_KEY }} + --backend-service-client-secret={{ ECOMMERCE_BACKEND_SERVICE_EDX_OAUTH2_SECRET }} + --from-email staff@example.com + --discovery_api_url={{ ECOMMERCE_DISCOVERY_SERVICE_URL }}/api/v1/ + {{ " --enable-microfrontend-for-basket-page=true" if ECOMMERCE_ENABLE_PAYMENT_MFE else "" }} + {{ " --payment-microfrontend-url="~EDXAPP_LMS_BASE_SCHEME~"://"~MFE_BASE~"/payment" if ECOMMERCE_ENABLE_PAYMENT_MFE else "" }} + when: '{{ ecommerce_create_demo_data }}' - command: './manage.py create_demo_data --partner=edX' when: '{{ ecommerce_create_demo_data }}' diff --git a/playbooks/roles/mfe_deployer/README.rst b/playbooks/roles/mfe_deployer/README.rst index 1e49ecda19c..39fbf49e56b 100644 --- a/playbooks/roles/mfe_deployer/README.rst +++ b/playbooks/roles/mfe_deployer/README.rst @@ -34,6 +34,14 @@ When running this role, you'll need to set the following variables: - **site_name**: Used to define the Environment SITE_NAME, used to build the MFE. By default it takes the value of ``MFE_DEPLOY_SITE_NAME``. - **standalone_nginx**: To indicate if the MFE will be deployed in a separated nginx file or if it will be in a shared nginx file with the other MFEs, by default it takes the value of ``MFE_DEPLOY_STANDALONE_NGINX``. + +- ``MFES_ECOMMERCE``: list of all ecommerce related MFEs you want to install. The structure matches MFES list. + + +Ecommerce related MFEs will be built in case of ecommerce service to be installed. +``MFE_DEPLOY_ECOMMERCE_MFES`` conditional variable is responsible for this and based on ``SANDBOX_ENABLE_ECOMMERCE`` variable. + + Deployment using subdirectories _______________________________ @@ -52,6 +60,14 @@ By default ``MFE_DEPLOY_STANDALONE_NGINX`` is false, which means that all the mi repo: frontend-app-account public_path: "/account/" + MFES_ECOMMERCE: + - name: payment + repo: frontend-app-payment + public_path: "/payment/" + - name: ecommerce + repo: frontend-app-ecommerce + public_path: "/ecommerce/" + ### edxapp Configurations ### See comprehensive example below @@ -75,6 +91,14 @@ If we want to deploy the microfrontends in different subdomains, we should turn - name: account repo: frontend-app-account + MFES_ECOMMERCE: + - name: payment + repo: frontend-app-payment + public_path: "/payment/" + - name: ecommerce + repo: frontend-app-ecommerce + public_path: "/ecommerce/" + MFE_DEPLOY_STANDALONE_NGINX: true ### edxapp Configurations @@ -123,6 +147,14 @@ __________________________________________________________ repo: frontend-app-account public_path: "/account/" + MFES_ECOMMERCE: + - name: payment + repo: frontend-app-payment + public_path: "/payment/" + - name: ecommerce + repo: frontend-app-ecommerce + public_path: "/ecommerce/" + MFE_DEPLOY_STANDALONE_NGINX: false MFE_DEPLOY_COMMON_HOSTNAME: '{{ MFE_BASE }}' @@ -147,10 +179,25 @@ __________________________________________________________ - "{{ EDXAPP_CMS_BASE }}" - "{{ MFE_BASE }}" + EDXAPP_SITE_CONFIGURATION: + - values: + ENABLE_ORDER_HISTORY_MICROFRONTEND: "{{ SANDBOX_ENABLE_ECOMMERCE }}" + # MFE Links EDXAPP_LMS_WRITABLE_GRADEBOOK_URL: 'https://{{ MFE_BASE}}/gradebook' EDXAPP_PROFILE_MICROFRONTEND_URL: 'https://{{ MFE_BASE}}/profile/u/' EDXAPP_ACCOUNT_MICROFRONTEND_URL: 'https://{{ MFE_BASE}}/account' + EDXAPP_ORDER_HISTORY_MICROFRONTEND_URL: 'https://{{ MFE_BASE }}/ecommerce/orders' + + ## ecommerce Configuration + ECOMMERCE_CORS_ORIGIN_WHITELIST: [ + "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", + ] + ECOMMERCE_CSRF_TRUSTED_ORIGINS: [ + "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", + ] + ECOMMERCE_CORS_ALLOW_CREDENTIALS: true + ECOMMERCE_ENABLE_PAYMENT_MFE: true .. _decision record about asymmetric JWT: https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst .. _Developer Documentation: https://edx.readthedocs.io/projects/edx-developer-docs/en/latest/developers_guide/micro_frontends_in_open_edx.html#overriding-brand-specific-elements diff --git a/playbooks/roles/mfe_deployer/defaults/main.yml b/playbooks/roles/mfe_deployer/defaults/main.yml index 28bad674173..3edd1c580f6 100644 --- a/playbooks/roles/mfe_deployer/defaults/main.yml +++ b/playbooks/roles/mfe_deployer/defaults/main.yml @@ -18,6 +18,20 @@ MFES: repo: frontend-app-account public_path: "/account/" +MFES_ECOMMERCE: + - name: payment + repo: frontend-app-payment + public_path: "/payment/" + - name: ecommerce + repo: frontend-app-ecommerce + public_path: "/ecommerce/" + +MFE_DEPLOY_ECOMMERCE_MFES: false +ecommerce_mfes: "{{ MFE_DEPLOY_ECOMMERCE_MFES | ternary(MFES_ECOMMERCE, []) }}" + +# Collect list of all MFEs to deploy +deploy_mfes: "{{ MFES + ecommerce_mfes }}" + MFE_DEPLOY_PUBLIC_PATH: "/" MFE_DEPLOY_SITE_NAME: "" diff --git a/playbooks/roles/mfe_deployer/tasks/main.yml b/playbooks/roles/mfe_deployer/tasks/main.yml index 9ada594b130..6497a90116b 100644 --- a/playbooks/roles/mfe_deployer/tasks/main.yml +++ b/playbooks/roles/mfe_deployer/tasks/main.yml @@ -17,7 +17,7 @@ MFE_PUBLIC_PATH: '{{ custom_mfe.public_path | default(MFE_DEPLOY_PUBLIC_PATH) }}' MFE_SITE_NAME: '{{ custom_mfe.site_name | default(MFE_DEPLOY_SITE_NAME) }}' MFE_STANDALONE_NGINX: '{{ custom_mfe.standalone_nginx | default(MFE_DEPLOY_STANDALONE_NGINX) }}' - loop: "{{ MFES }}" + loop: "{{ deploy_mfes }}" loop_control: loop_var: custom_mfe tags: diff --git a/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/concerns/mfe.j2 b/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/concerns/mfe.j2 index 8cb52723f81..881b5d9ac0c 100644 --- a/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/concerns/mfe.j2 +++ b/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/concerns/mfe.j2 @@ -1,4 +1,4 @@ -{% for mfe in MFES %} +{% for mfe in deploy_mfes %} location ~ ^{{ mfe.public_path }}?(.*)$ { root {{ COMMON_APP_DIR }}/{{ mfe.name }}/{{ mfe.repo }}/dist/; diff --git a/playbooks/roles/mfe_flags_setup/defaults/main.yml b/playbooks/roles/mfe_flags_setup/defaults/main.yml index 82ff5c998ec..d72d9c799ee 100644 --- a/playbooks/roles/mfe_flags_setup/defaults/main.yml +++ b/playbooks/roles/mfe_flags_setup/defaults/main.yml @@ -2,3 +2,4 @@ MFE_FLAGS_SETUP_FLAGS_LIST: - account.redirect_to_microfrontend + - order_history.redirect_to_microfrontend From 3d8d7add09c5c1397a00e9aaeb8d295b47df17db Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 14 Jun 2021 11:14:13 -0400 Subject: [PATCH 055/664] fix:pass celery always_eager flag as it is consumed by celery4 Based on https://github.com/celery/celery/blob/04d921b3fbcd9cc42285edeb916c41df507fe691/docs/history/whatsnew-4.0.rst this and several other celery settings changed in the course of our py3.8 upgrade. This is one which matters a great deal for sandboxes (which shouldn't rely on celery for the execution of asynch tasks). There are some more which I'll follow up with after I understand better why their apparent lack isn't impacting production builds' operation of celery. --- playbooks/roles/registrar/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/registrar/defaults/main.yml b/playbooks/roles/registrar/defaults/main.yml index cf9486c81ad..cd3794120f2 100644 --- a/playbooks/roles/registrar/defaults/main.yml +++ b/playbooks/roles/registrar/defaults/main.yml @@ -122,7 +122,7 @@ registrar_service_config_overrides: CORS_ORIGIN_WHITELIST: "{{ REGISTRAR_CORS_ORIGIN_WHITELIST }}" CSRF_TRUSTED_ORIGINS: "{{ REGISTRAR_CSRF_TRUSTED_ORIGINS }}" CSRF_COOKIE_SECURE: "{{ REGISTRAR_CSRF_COOKIE_SECURE }}" - CELERY_ALWAYS_EAGER: '{{ REGISTRAR_CELERY_ALWAYS_EAGER }}' + CELERY_TASK_ALWAYS_EAGER: '{{ REGISTRAR_CELERY_ALWAYS_EAGER }}' CELERY_BROKER_TRANSPORT: '{{ REGISTRAR_CELERY_BROKER_TRANSPORT }}' CELERY_BROKER_USER: '{{ REGISTRAR_CELERY_BROKER_USER }}' CELERY_BROKER_PASSWORD: '{{ REGISTRAR_CELERY_BROKER_PASSWORD }}' From 27ea6d56a003749454fc542ccc3326c64795008e Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 14 Jun 2021 11:21:25 -0400 Subject: [PATCH 056/664] fix: follow-up from #6444 --- playbooks/roles/registrar/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/registrar/defaults/main.yml b/playbooks/roles/registrar/defaults/main.yml index cf9486c81ad..df50c7e60f4 100644 --- a/playbooks/roles/registrar/defaults/main.yml +++ b/playbooks/roles/registrar/defaults/main.yml @@ -128,9 +128,9 @@ registrar_service_config_overrides: CELERY_BROKER_PASSWORD: '{{ REGISTRAR_CELERY_BROKER_PASSWORD }}' CELERY_BROKER_HOSTNAME: '{{ REGISTRAR_CELERY_BROKER_HOSTNAME }}' CELERY_BROKER_VHOST: '{{ REGISTRAR_CELERY_BROKER_VHOST }}' - CELERY_DEFAULT_EXCHANGE: 'registrar' - CELERY_DEFAULT_ROUTING_KEY: 'registrar' - CELERY_DEFAULT_QUEUE: '{{ registrar_celery_default_queue }}' + CELERY_TASK_DEFAULT_EXCHANGE: 'registrar' + CELERY_TASK_DEFAULT_ROUTING_KEY: 'registrar' + CELERY_TASK_DEFAULT_QUEUE: '{{ registrar_celery_default_queue }}' # See edx_django_service_automated_users for an example of what this should be REGISTRAR_AUTOMATED_USERS: {} From 534d106c520595e4e658593c8d694b8fbc86d7b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Wed, 9 Jun 2021 21:55:56 +0200 Subject: [PATCH 057/664] fix: run celerybeat only once and on one instance The current implementation of celerybeat usage works properly if and only if one concurrent worker process runs per queue using the nested beat process started by the `--beat` flag. In case the the celerybeat process is running as a separate process not as part of the workers, that partially solves the problem. Either in that case, if multiple instances are running in a cluster, without proper process supervising across instances will result in (at least) dumplicated scheduling, meaning that the scheduler will cause more work for the workers and results will be duplicated. Taking the above into consideration, to solve this issue, this commit introduces single-beat package that wraps the celerybeat process and keeps track of the process that acquired the lock in redis. In case one of the instances get killed or the locking process crashes, the lock will be released and another instance or process will take over its place. --- playbooks/roles/edxapp/defaults/main.yml | 4 ++ playbooks/roles/edxapp/tasks/deploy.yml | 12 ++++++ .../templates/edx/app/edxapp/scheduler.sh.j2 | 22 +++++++++++ .../conf.d.available/workers.conf.j2 | 37 ++++++++++++++++++- 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 3e1ac091713..3da608e5dd7 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -175,6 +175,10 @@ EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" EDXAPP_ENABLE_CELERY_BEAT: false +EDXAPP_SINGLE_BEAT_LOCK_TIME: 60 +# EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2 +EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29 +EDXAPP_SINGLE_BEAT_VERSION: "0.4.2" EDXAPP_BRANCH_IO_KEY: "" EDXAPP_AUTH_USE_OPENID_PROVIDER: true diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index ffef1ba5a15..c811fe74474 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,6 +357,7 @@ with_items: - "lms.sh" - "cms.sh" + - "scheduler.sh" - "worker.sh" - "reload_lms_config.sh" - "reload_cms_config.sh" @@ -364,6 +365,17 @@ - install - install:configuration +- name: install single-beat to control celerybeat processes + command: "{{ edxapp_venv_dir }}/bin/pip install single-beat=={{ EDXAPP_SINGLE_BEAT_VERSION }}" + args: + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + environment: "{{ edxapp_environment }}" + when: EDXAPP_ENABLE_CELERY_BEAT + tags: + - install + - install:app-requirements + - name: import custom tinymce plugins include_role: name: "tinymce_plugins" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 new file mode 100644 index 00000000000..acf54f89058 --- /dev/null +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# {{ ansible_managed }} + +{% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} +source {{ edxapp_app_dir }}/edxapp_env +{% if COMMON_ENABLE_NEWRELIC_APP %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %} + +export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini" +if command -v ec2metadata >/dev/null 2>&1; then + INSTANCEID=$(ec2metadata --instance-id); + HOSTNAME=$(hostname) + export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" +fi +{% else %} +{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %} +{% endif %} + +# We exec so that celery is the child of supervisor and can be managed properly + +exec {{ executable }} $@ diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 13a68d58bce..95c753a4b38 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -7,7 +7,7 @@ directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/worker.sh {{ '--beat' if EDXAPP_ENABLE_CELERY_BEAT|bool else '' }} --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} +command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }} ; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit @@ -20,3 +20,38 @@ autorestart=true [group:edxapp_worker] programs={%- for w in edxapp_workers %}{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}{%- if not loop.last %},{%- endif %}{%- endfor %} + + +{% if EDXAPP_ENABLE_CELERY_BEAT|bool and EDXAPP_CELERY_BROKER_TRANSPORT == "redis" %} +[program:celerybeat_scheduler] +environment= + {% if COMMON_ENABLE_NEWRELIC_APP %} + NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-lms, + NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}, + NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }}, + {% endif -%} + LANG={{ EDXAPP_LANG }}, + PYTHONPATH={{ edxapp_code_dir }}, + SERVICE_VARIANT=lms, + BOTO_CONFIG="{{ edxapp_app_dir }}/.boto", + EDX_REST_API_CLIENT_NAME=edx.lms.core.default, + SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }}, + SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }}, + SINGLE_BEAT_IDENTIFIER="celerybeat", + SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}", + SINGLE_BEAT_WAIT_MODE="supervised" +user={{ common_web_user }} +directory={{ edxapp_code_dir }} +stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log +stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log + +command={{ edxapp_app_dir }}/scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" + +killasgroup=true +stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }} +; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit +; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers +; are auto restarted even when exiting with code 0. +; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. +autorestart=true +{% endif %} From 30518b6a64866ca43fccdf752ec9eda2b8af21ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Wed, 16 Jun 2021 12:31:54 +0200 Subject: [PATCH 058/664] refactor: rename scheduler.sh to beat_scheduler.sh --- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- .../edx/app/edxapp/{scheduler.sh.j2 => beat_scheduler.sh.j2} | 0 .../edx/app/supervisor/conf.d.available/workers.conf.j2 | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename playbooks/roles/edxapp/templates/edx/app/edxapp/{scheduler.sh.j2 => beat_scheduler.sh.j2} (100%) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index c811fe74474..479ed323bd7 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,7 +357,7 @@ with_items: - "lms.sh" - "cms.sh" - - "scheduler.sh" + - "beat_scheduler.sh" - "worker.sh" - "reload_lms_config.sh" - "reload_cms_config.sh" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 similarity index 100% rename from playbooks/roles/edxapp/templates/edx/app/edxapp/scheduler.sh.j2 rename to playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 95c753a4b38..245eb933c42 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -45,7 +45,7 @@ directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" +command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" killasgroup=true stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }} From 15e027b3a926ee4a6537bbebf7f2e2c71b3c54d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Wed, 16 Jun 2021 12:35:53 +0200 Subject: [PATCH 059/664] refactor: use pip package install and rename task --- playbooks/roles/edxapp/tasks/deploy.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 479ed323bd7..174b4878a43 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -365,16 +365,17 @@ - install - install:configuration -- name: install single-beat to control celerybeat processes - command: "{{ edxapp_venv_dir }}/bin/pip install single-beat=={{ EDXAPP_SINGLE_BEAT_VERSION }}" - args: - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - environment: "{{ edxapp_environment }}" - when: EDXAPP_ENABLE_CELERY_BEAT - tags: - - install - - install:app-requirements +- name: install single-beat to run only one celerybeat scheduler + pip: + name: single-beat + version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}" + virtualenv: "{{ edxapp_venv_dir }}" + state: present + become_user: "{{ edxapp_user }}" + when: EDXAPP_ENABLE_CELERY_BEAT + tags: + - install + - install:app-requirements - name: import custom tinymce plugins include_role: From 945fe9df4aaef4bc1dfc565eab4bbcc0b12f9ee8 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Thu, 17 Jun 2021 16:30:45 -0400 Subject: [PATCH 060/664] Unindent a few entries that needd to be unindented (#6450) --- playbooks/roles/edxapp/tasks/deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 174b4878a43..e8ec2949961 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -371,11 +371,11 @@ version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}" virtualenv: "{{ edxapp_venv_dir }}" state: present - become_user: "{{ edxapp_user }}" - when: EDXAPP_ENABLE_CELERY_BEAT - tags: - - install - - install:app-requirements + become_user: "{{ edxapp_user }}" + when: EDXAPP_ENABLE_CELERY_BEAT + tags: + - install + - install:app-requirements - name: import custom tinymce plugins include_role: From d0e1dadbe734c0eebe44968254471948409c2af6 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 17 Jun 2021 16:57:02 -0400 Subject: [PATCH 061/664] fix: warning about duplicately set setting JIRA:CR-3789 --- playbooks/roles/edxapp/defaults/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 3da608e5dd7..ffe0a396904 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -249,8 +249,6 @@ EDXAPP_ENABLE_EDXNOTES: false EDXAPP_ENABLE_CREDIT_ELIGIBILITY: false EDXAPP_ENABLE_CREDIT_API: false -EDXAPP_ENABLE_SPECIAL_EXAMS: false - EDXAPP_CUSTOM_COURSES_EDX: false EDXAPP_ENABLE_SYSADMIN_DASHBOARD: false From 85d9205e59746ee085512e51f548e5311cb4ea74 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 18 Jun 2021 14:12:24 +0500 Subject: [PATCH 062/664] fix ecommerce Docker CI build --- playbooks/roles/browsers/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/browsers/defaults/main.yml b/playbooks/roles/browsers/defaults/main.yml index 1e47d2b1c3a..4be1a8af01c 100644 --- a/playbooks/roles/browsers/defaults/main.yml +++ b/playbooks/roles/browsers/defaults/main.yml @@ -18,6 +18,7 @@ browser_deb_pkgs: - ubuntu-restricted-extras - xdg-utils - xvfb + - wget # Firefox for Xenial FIREFOX_VERSION: version 59.* From 98caacc0c5de2f7c0ca1428fddf3429d152277dd Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 28 Jun 2021 14:21:12 +0500 Subject: [PATCH 063/664] add mongo_4_2 role --- playbooks/mongo_4_2.yml | 29 ++ playbooks/roles/mongo_4_2/defaults/main.yml | 112 +++++ .../disable-transparent-hugepages.service | 11 + playbooks/roles/mongo_4_2/meta/main.yml | 5 + playbooks/roles/mongo_4_2/tasks/main.yml | 415 ++++++++++++++++++ .../templates/log-mongo-serverStatus.sh.j2 | 3 + .../mongo_4_2/templates/mongo_logrotate.j2 | 46 ++ .../roles/mongo_4_2/templates/mongod.conf.j2 | 46 ++ 8 files changed, 667 insertions(+) create mode 100644 playbooks/mongo_4_2.yml create mode 100644 playbooks/roles/mongo_4_2/defaults/main.yml create mode 100644 playbooks/roles/mongo_4_2/files/etc/systemd/system/disable-transparent-hugepages.service create mode 100644 playbooks/roles/mongo_4_2/meta/main.yml create mode 100644 playbooks/roles/mongo_4_2/tasks/main.yml create mode 100644 playbooks/roles/mongo_4_2/templates/log-mongo-serverStatus.sh.j2 create mode 100644 playbooks/roles/mongo_4_2/templates/mongo_logrotate.j2 create mode 100644 playbooks/roles/mongo_4_2/templates/mongod.conf.j2 diff --git a/playbooks/mongo_4_2.yml b/playbooks/mongo_4_2.yml new file mode 100644 index 00000000000..50e75177f24 --- /dev/null +++ b/playbooks/mongo_4_2.yml @@ -0,0 +1,29 @@ +# Manages a mongo cluster. +# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG +# as used by mongo_replica_set in the mongo_4_2 role. +# +# If you are initializing a cluster, your command might look like: +# ansible-playbook mongo_4_2.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml +# If you just want to deploy an updated replica set config, you can run +# ansible-playbook mongo_4_2.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set +# +# ADDING A NEW CLUSTER MEMBER +# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory +# ansible-playbook mongo_4_2.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml +- name: Deploy MongoDB + hosts: all + become: True + gather_facts: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - mongo_4_2 + - munin_node + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG diff --git a/playbooks/roles/mongo_4_2/defaults/main.yml b/playbooks/roles/mongo_4_2/defaults/main.yml new file mode 100644 index 00000000000..09da0fcd5a3 --- /dev/null +++ b/playbooks/roles/mongo_4_2/defaults/main.yml @@ -0,0 +1,112 @@ +mongo_logappend: true + +#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle +mongo_logrotate: reopen + +MONGO_VERSION_MAJOR_MINOR: "4.2" +MONGO_VERSION_PATCH: "14" +PYMONGO_VERSION: "3.11.2" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +mongo_port: "27017" +mongo_extra_conf: '' +mongo_key_file: '/etc/mongodb_key' + +mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo" +mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo" +mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal" +mongo_user: mongodb + +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" +MONGODB_REPO_BIONIC: "deb http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" + +mongodb_debian_pkgs: + - "mongodb-org={{ MONGO_VERSION }}" + - "mongodb-org-server={{ MONGO_VERSION }}" + - "mongodb-org-shell={{ MONGO_VERSION }}" + - "mongodb-org-mongos={{ MONGO_VERSION }}" + - "mongodb-org-tools={{ MONGO_VERSION }}" + + + +mongo_configure_replica_set: true + +# Vars Meant to be overridden +MONGO_ADMIN_USER: 'admin' +MONGO_ADMIN_PASSWORD: 'password' +MONGO_USERS: + - user: cs_comments_service + password: password + database: cs_comments_service + roles: readWrite + - user: edxapp + password: password + database: edxapp + roles: readWrite + +# This default setting is approriate for a single machine installation +# This will need to be overridden for setups where mongo is on its own server +# and/or you are configuring mongo replication. If the override value is +# 0.0.0.0 mongo will listen on all IPs. The value may also be set to a +# specific IP. +MONGO_BIND_IP: 127.0.0.1 + +MONGO_REPL_SET: "rs0" +MONGO_AUTH: true + +MONGO_CLUSTER_KEY: "CHANGEME" + +# Cluster member configuration +# Fed directly into mongodb_replica_set module +MONGO_RS_CONFIG: + _id: '{{ MONGO_REPL_SET }}' + members: + - host: '127.0.0.1' + +# Storage engine options in 3.2: "mmapv1" or "wiredTiger" +# 3.2 and 3.4 default to wiredTiger +MONGO_STORAGE_ENGINE: "wiredTiger" + +# List of dictionaries as described in the mount_ebs role's default +# for the volumes. +# Useful if you want to store your mongo data and/or journal on separate +# disks from the root volume. By default, they will end up mongo_data_dir +# on the root disk. +MONGO_VOLUMES: [] + +# WiredTiger takes a number of optional configuration settings +# which can be defined as a yaml structure in your secure configuration. +MONGO_STORAGE_ENGINE_OPTIONS: !!null + +mongo_logpath: "{{ mongo_log_dir }}/mongodb.log" +mongo_dbpath: "{{ mongo_data_dir }}/mongodb" + +# In environments that do not require durability (devstack / Jenkins) +# you can disable the journal to reduce disk usage +mongo_enable_journal: true + +MONGO_LOG_SERVERSTATUS: true + +# Vars for configuring a mongo backup node. If enabled, this node will be provisioned with a script that uses mongodump +# to backup the database to an ebs volume at a period set by mongo_backup_cron. +# Set MONGO_BACKUP_ENABLED to true to enable. If enabled, all the other MONGO_BACKUP_ vars must be set according to your +# setup. +MONGO_BACKUP_ENABLED: false +MONGO_BACKUP_NODE: "" # note: most likely the ip address of the instance on which to perform the backups +MONGO_BACKUP_EBS_VOLUME_DEVICE: "" +MONGO_BACKUP_EBS_VOLUME_ID: "" +MONGO_BACKUP_AUTH_DATABASE: "" +MONGO_BACKUP_PRUNE_OLDER_THAN_DATE: "" # passed to `date -d`; should be a relative date like "-30days" +MONGO_BACKUP_SNITCH_URL: "" # Optional URL that will be used to ping a monitoring service (such as Dead Man's Snitch) upon successful completion of a backup. +MONGO_BACKUP_VOLUME_MOUNT_PATH: "/mnt/mongo-backup" +MONGO_BACKUP_SNAPSHOT_DESC: "mongo-backup" +mongo_backup_script_path: "/usr/local/sbin/backup-mongo.sh" +mongo_backup_cron: + minute: '12' + hour: '*/12' + day: '*' + month: '*' + weekday: '*' + +# Internal variable set to true dynamically if backups enabled and playbook running on MONGO_BACKUP_NODE. Do not +# manually override. +is_backup_node: false diff --git a/playbooks/roles/mongo_4_2/files/etc/systemd/system/disable-transparent-hugepages.service b/playbooks/roles/mongo_4_2/files/etc/systemd/system/disable-transparent-hugepages.service new file mode 100644 index 00000000000..282c9e122c3 --- /dev/null +++ b/playbooks/roles/mongo_4_2/files/etc/systemd/system/disable-transparent-hugepages.service @@ -0,0 +1,11 @@ +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +Before=mongod.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' + +[Install] +RequiredBy=mongod.service diff --git a/playbooks/roles/mongo_4_2/meta/main.yml b/playbooks/roles/mongo_4_2/meta/main.yml new file mode 100644 index 00000000000..d7223454526 --- /dev/null +++ b/playbooks/roles/mongo_4_2/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - common + - role: mount_ebs + volumes: "{{ MONGO_VOLUMES }}" diff --git a/playbooks/roles/mongo_4_2/tasks/main.yml b/playbooks/roles/mongo_4_2/tasks/main.yml new file mode 100644 index 00000000000..127a90ae906 --- /dev/null +++ b/playbooks/roles/mongo_4_2/tasks/main.yml @@ -0,0 +1,415 @@ +--- +- name: Add disable transparent huge pages systemd service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + copy: + src: etc/systemd/system/disable-transparent-hugepages.service + dest: "/etc/systemd/system/disable-transparent-hugepages.service" + owner: root + group: root + mode: 0644 + tags: + - "hugepages" + - "install" + - "install:configuration" + +- name: Enable/start disable transparent huge pages service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + service: + name: disable-transparent-hugepages + enabled: yes + state: started + tags: + - "hugepages" + - "manage" + - "manage:start" + +- name: install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ PYMONGO_VERSION }}" + extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" + state: present + retries: 3 + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + when: ansible_distribution_release != 'focal' + +# mongo 4.2 does not have any source list for Focal +# use Bionci repo source list to install mongo 4.2 +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO_BIONIC }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + when: ansible_distribution_release == 'focal' + +- name: install mongo server and recommends + apt: + pkg: "{{ item }}" + state: present + install_recommends: yes + force: yes + update_cache: yes + register: install_mongo_package + with_items: "{{ mongodb_debian_pkgs }}" + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: create mongo dirs + file: + path: "{{ item }}" + state: directory + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + with_items: + - "{{ mongo_data_dir }}" + - "{{ mongo_dbpath }}" + - "{{ mongo_log_dir }}" + - "{{ mongo_journal_dir }}" + tags: + - "install" + - "install:app-configuration" + +# This will error when run on a new replica set, so we ignore_errors +# and connect anonymously next. +- name: determine if there is a replica set already + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + run_once: true + register: authed_replica_set_already_configured + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +- name: Try checking the replica set with no user/pass in case this is a new box + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + run_once: true + register: unauthed_replica_set_already_configured + when: authed_replica_set_already_configured.failed is defined + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +# We use these in the templates but also to control a whole bunch of logic +- name: set facts that default to not initializing a replica set + set_fact: + initialize_replica_set: false + skip_replica_set: false + tags: + - "install" + - "install:app-configuration" + - "update_mongod_conf" + - "manage" + - "manage:db-replication" + +# If either auth or unauthed access comes back with a replica set, we +# do not want to initialize one. Since initialization requires a bunch +# of extra templating and restarting, it's not something we want to do on +# existing boxes. +- name: track if you have a replica set + set_fact: + initialize_replica_set: true + skip_replica_set: true + when: authed_replica_set_already_configured.status is not defined + and unauthed_replica_set_already_configured.status is not defined + tags: + - "manage" + - "manage:db-replication" + +- name: warn about unconfigured replica sets + debug: msg="You do not appear to have a Replica Set configured, deploying one for you" + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: copy mongodb key file + copy: + content: "{{ MONGO_CLUSTER_KEY }}" + dest: "{{ mongo_key_file }}" + mode: 0600 + owner: mongodb + group: mongodb + register: update_mongod_key + tags: + - "manage" + - "manage:db-replication" + - "mongodb_key" + +# If skip_replica_set is true, this template will not contain a replica set stanza +# because of the fact above. +- name: copy configuration template + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + register: update_mongod_conf + tags: + - "install" + - "install:app-configuration" + - "manage" + - "manage:db-replication" + - "update_mongod_conf" + +# This sets the is_backup_node var by checking whether +# mongo backups are enabled AND we're currently running against the designated mongo backup node. +# This allows backup-related tasks below to determine whether or not they should run on the current mongo node. +- name: determine if backup tasks should run + set_fact: + is_backup_node: true + when: MONGO_BACKUP_ENABLED and '{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}' == '{{ MONGO_BACKUP_NODE }}' + tags: + - "backup:mongo" + +- name: install logrotate configuration + template: + src: mongo_logrotate.j2 + dest: /etc/logrotate.d/hourly/mongo + tags: + - "backup:mongo" + - "install" + - "install:app-configuration" + - "logrotate" + +- name: install prereqs for backup script + apt: + pkg: "{{ item }}" + state: present + update_cache: yes + with_items: + - jq + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install backup script + template: + src: backup-mongo.sh.j2 + dest: "{{ mongo_backup_script_path }}" + mode: 0700 + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: add mongo backup script to cron + cron: + name: mongo backup job + minute: "{{ mongo_backup_cron.minute | default('12') }}" + hour: "{{ mongo_backup_cron.hour | default('*/12') }}" + day: "{{ mongo_backup_cron.day | default('*') }}" + month: "{{ mongo_backup_cron.month | default('*') }}" + weekday: "{{ mongo_backup_cron.weekday | default('*') }}" + job: "{{ mongo_backup_script_path }} >> {{ mongo_log_dir }}/mongo-backup.log 2>&1" + become: yes + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: format mongo backup volume + filesystem: + dev: "{{ MONGO_BACKUP_EBS_VOLUME_DEVICE }}" + fstype: ext4 + force: true + ignore_errors: true + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# We only try passwordless superuser creation when +# we're initializing the replica set and need to use +# the localhost exemption to create a user who will be +# able to initialize the replica set. +# We can only create the users on one machine, the one +# where we will initialize the replica set. If we +# create users on multiple hosts, then they will fail +# to come into the replica set. +- name: create super user + mongodb_user: + name: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + database: admin + roles: root + when: initialize_replica_set + run_once: true + tags: + - "manage" + - "manage:db-replication" + +# Now that the localhost exemption has been used to create the superuser, we need +# to add replica set to our configuration. This will never happen if we detected +# a replica set in the 'determine if there is a replica set already' task. +- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set + set_fact: + skip_replica_set: false + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: re-copy configuration template with replica set enabled + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: restart mongo service + service: + name: mongod + state: restarted + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: configure replica set + mongodb_replica_set: + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + rs_config: "{{ MONGO_RS_CONFIG }}" + run_once: true + register: replset_status + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + - "manage:db-replication-configuration" + +# During initial replica set configuration, it can take a few seconds to vote +# a primary and for all members to reflect that status. During that window, +# use creation or other writes can fail. The best wait/check seems to be repeatedly +# checking the replica set status until we see a PRIMARY in the results. +- name: Wait for the replica set to update and (if needed) elect a primary + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + register: status + until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list + when: mongo_configure_replica_set + retries: 5 + delay: 2 + run_once: true + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + +- name: create mongodb users in a replica set + mongodb_user: + database: "{{ item.database }}" + login_database: 'admin' + login_user: "{{ MONGO_ADMIN_USER }}" + login_password: "{{ MONGO_ADMIN_PASSWORD }}" + name: "{{ item.user }}" + password: "{{ item.password }}" + roles: "{{ item.roles }}" + state: present + replica_set: "{{ MONGO_REPL_SET }}" + with_items: "{{ MONGO_USERS }}" + run_once: true + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-users" + - "manage:db-replication" + +- name: ensure mongo starts at boot time + service: + name: mongod + enabled: yes + tags: + - "manage" + - "manage:start" + +- name: add serverStatus logging script + template: + src: "log-mongo-serverStatus.sh.j2" + dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh" + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + mode: 0700 + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" + +- name: add serverStatus logging script to cron + cron: + name: mongostat logging job + minute: "*/3" + job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1 + become: yes + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" diff --git a/playbooks/roles/mongo_4_2/templates/log-mongo-serverStatus.sh.j2 b/playbooks/roles/mongo_4_2/templates/log-mongo-serverStatus.sh.j2 new file mode 100644 index 00000000000..04649d55ad1 --- /dev/null +++ b/playbooks/roles/mongo_4_2/templates/log-mongo-serverStatus.sh.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON +/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' diff --git a/playbooks/roles/mongo_4_2/templates/mongo_logrotate.j2 b/playbooks/roles/mongo_4_2/templates/mongo_logrotate.j2 new file mode 100644 index 00000000000..f2fb4483566 --- /dev/null +++ b/playbooks/roles/mongo_4_2/templates/mongo_logrotate.j2 @@ -0,0 +1,46 @@ +{{ mongo_log_dir }}/serverStatus.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} + +{% if is_backup_node %} +{{ mongo_log_dir }}/mongo-backup.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} +{% endif %} + +{{ mongo_log_dir }}/mongodb.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M + postrotate + /usr/bin/killall -USR1 mongod + endscript +} diff --git a/playbooks/roles/mongo_4_2/templates/mongod.conf.j2 b/playbooks/roles/mongo_4_2/templates/mongod.conf.j2 new file mode 100644 index 00000000000..b7d4b4a1efe --- /dev/null +++ b/playbooks/roles/mongo_4_2/templates/mongod.conf.j2 @@ -0,0 +1,46 @@ +# {{ ansible_managed }} +# mongodb.conf + + +storage: + # Where to store the data. + dbPath: {{ mongo_dbpath }} + # Storage Engine + engine: {{ MONGO_STORAGE_ENGINE }} + # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling + journal: +{% if mongo_enable_journal %} + enabled: true +{% else %} + enabled: false +{% endif %} +{% if MONGO_STORAGE_ENGINE_OPTIONS %} + {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} +{% endif %} + +systemLog: + #where to log + destination: file + path: "{{ mongo_logpath }}" +{% if mongo_logappend %} + logAppend: true +{% else %} + logAppend: false +{% endif %} + logRotate: {{ mongo_logrotate }} + +{% if not skip_replica_set %} +replication: + replSetName: {{ MONGO_REPL_SET }} + +security: + authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }} + keyFile: {{ mongo_key_file }} + +{% endif %} +net: + bindIp: {{ MONGO_BIND_IP }} + port: {{ mongo_port }} + + +{{ mongo_extra_conf }} From 0c5ee53460f8c27b83bf054e4d495e9dd327bce7 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 28 Jun 2021 16:42:52 +0000 Subject: [PATCH 064/664] fix: Suppress logging when generating EC2 keypair (was logging private key) (#6458) --- playbooks/continuous_delivery/launch_instance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index a53e096728b..4b8593b94b9 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -46,6 +46,7 @@ name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" region: "{{ ec2_region }}" register: ssh_key_register + no_log: True - name: Ensure artifact directory exists file: From 37852385cdcf807d023686148b1047ea4d5b32ec Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 28 Jun 2021 16:32:04 -0400 Subject: [PATCH 065/664] Try bumping Python to fix snadbox --- util/install/ansible-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 5e0ca53020a..49f6833a361 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -37,7 +37,7 @@ PIP_VERSION="20.3.4" SETUPTOOLS_VERSION="44.1.0" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" -PYTHON_VERSION="3.5" +PYTHON_VERSION="3.8" ANSIBLE_DIR="/tmp/ansible" CONFIGURATION_DIR="/tmp/configuration" EDX_PPA_KEY_SERVER="keyserver.ubuntu.com" From 23627009585648fbb5a21b59955f205a5f4508d4 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 29 Jun 2021 09:27:37 +0500 Subject: [PATCH 066/664] update base ami and set python version to 3.8 for focal --- util/install/ansible-bootstrap.sh | 7 ++++++- util/jenkins/ansible-provision.sh | 6 ++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 49f6833a361..3a51e877c75 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -37,7 +37,6 @@ PIP_VERSION="20.3.4" SETUPTOOLS_VERSION="44.1.0" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" -PYTHON_VERSION="3.8" ANSIBLE_DIR="/tmp/ansible" CONFIGURATION_DIR="/tmp/configuration" EDX_PPA_KEY_SERVER="keyserver.ubuntu.com" @@ -83,6 +82,12 @@ EOF exit 1; fi +if [[ "${SHORT_DIST}" == focal ]] ;then + PYTHON_VERSION="3.8" +else + PYTHON_VERSION="3.5" +fi + EDX_PPA="deb http://ppa.edx.org ${SHORT_DIST} main" # Upgrade the OS diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 5133ae2881a..1b6b54a5c03 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -131,12 +131,10 @@ if [[ -z $ami ]]; then ami="ami-087649e61b3299e66" elif [[ $server_type == "ubuntu_18.04" ]]; then ami="ami-07ebfd5b3428b6f4d" - elif [[ $server_type == "ubuntu_20.04" ]]; then - ami="ami-05cf2c352da0bfb2e" + elif [[ $server_type == "ubuntu_20.04" || $server_type == "full_edx_installation_from_scratch" ]]; then + ami="ami-0dd76f917833aac4b" # Ansible will always use Python3 interpreter on Ubuntu 20.04 hosts to execute modules extra_var_arg+=' -e ansible_python_interpreter=auto' - elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then - ami="ami-092546daafcc8bc0d" fi fi From 26c061eb46c6dd13a07321959fe7c5dc133a03fc Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 29 Jun 2021 16:08:57 +0500 Subject: [PATCH 067/664] add mongo 4.4 role --- playbooks/mongo_4_4.yml | 29 ++ playbooks/roles/mongo_4_4/defaults/main.yml | 111 +++++ .../disable-transparent-hugepages.service | 11 + playbooks/roles/mongo_4_4/meta/main.yml | 5 + playbooks/roles/mongo_4_4/tasks/main.yml | 402 ++++++++++++++++++ .../templates/log-mongo-serverStatus.sh.j2 | 3 + .../mongo_4_4/templates/mongo_logrotate.j2 | 46 ++ .../roles/mongo_4_4/templates/mongod.conf.j2 | 46 ++ 8 files changed, 653 insertions(+) create mode 100644 playbooks/mongo_4_4.yml create mode 100644 playbooks/roles/mongo_4_4/defaults/main.yml create mode 100644 playbooks/roles/mongo_4_4/files/etc/systemd/system/disable-transparent-hugepages.service create mode 100644 playbooks/roles/mongo_4_4/meta/main.yml create mode 100644 playbooks/roles/mongo_4_4/tasks/main.yml create mode 100644 playbooks/roles/mongo_4_4/templates/log-mongo-serverStatus.sh.j2 create mode 100644 playbooks/roles/mongo_4_4/templates/mongo_logrotate.j2 create mode 100644 playbooks/roles/mongo_4_4/templates/mongod.conf.j2 diff --git a/playbooks/mongo_4_4.yml b/playbooks/mongo_4_4.yml new file mode 100644 index 00000000000..b11f769fccd --- /dev/null +++ b/playbooks/mongo_4_4.yml @@ -0,0 +1,29 @@ +# Manages a mongo cluster. +# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG +# as used by mongo_replica_set in the mongo_4_4 role. +# +# If you are initializing a cluster, your command might look like: +# ansible-playbook mongo_4_4.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml +# If you just want to deploy an updated replica set config, you can run +# ansible-playbook mongo_4_4.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set +# +# ADDING A NEW CLUSTER MEMBER +# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory +# ansible-playbook mongo_4_4.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml +- name: Deploy MongoDB + hosts: all + become: True + gather_facts: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - mongo_4_4 + - munin_node + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG diff --git a/playbooks/roles/mongo_4_4/defaults/main.yml b/playbooks/roles/mongo_4_4/defaults/main.yml new file mode 100644 index 00000000000..ea9b0ca391d --- /dev/null +++ b/playbooks/roles/mongo_4_4/defaults/main.yml @@ -0,0 +1,111 @@ +mongo_logappend: true + +#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle +mongo_logrotate: reopen + +MONGO_VERSION_MAJOR_MINOR: "4.4" +MONGO_VERSION_PATCH: "6" +PYMONGO_VERSION: "3.11.2" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +mongo_port: "27017" +mongo_extra_conf: '' +mongo_key_file: '/etc/mongodb_key' + +mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo" +mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo" +mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal" +mongo_user: mongodb + +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" + +mongodb_debian_pkgs: + - "mongodb-org={{ MONGO_VERSION }}" + - "mongodb-org-server={{ MONGO_VERSION }}" + - "mongodb-org-shell={{ MONGO_VERSION }}" + - "mongodb-org-mongos={{ MONGO_VERSION }}" + - "mongodb-org-tools={{ MONGO_VERSION }}" + + + +mongo_configure_replica_set: true + +# Vars Meant to be overridden +MONGO_ADMIN_USER: 'admin' +MONGO_ADMIN_PASSWORD: 'password' +MONGO_USERS: + - user: cs_comments_service + password: password + database: cs_comments_service + roles: readWrite + - user: edxapp + password: password + database: edxapp + roles: readWrite + +# This default setting is approriate for a single machine installation +# This will need to be overridden for setups where mongo is on its own server +# and/or you are configuring mongo replication. If the override value is +# 0.0.0.0 mongo will listen on all IPs. The value may also be set to a +# specific IP. +MONGO_BIND_IP: 127.0.0.1 + +MONGO_REPL_SET: "rs0" +MONGO_AUTH: true + +MONGO_CLUSTER_KEY: "CHANGEME" + +# Cluster member configuration +# Fed directly into mongodb_replica_set module +MONGO_RS_CONFIG: + _id: '{{ MONGO_REPL_SET }}' + members: + - host: '127.0.0.1' + +# Storage engine options in 3.2: "mmapv1" or "wiredTiger" +# 3.2 and 3.4 default to wiredTiger +MONGO_STORAGE_ENGINE: "wiredTiger" + +# List of dictionaries as described in the mount_ebs role's default +# for the volumes. +# Useful if you want to store your mongo data and/or journal on separate +# disks from the root volume. By default, they will end up mongo_data_dir +# on the root disk. +MONGO_VOLUMES: [] + +# WiredTiger takes a number of optional configuration settings +# which can be defined as a yaml structure in your secure configuration. +MONGO_STORAGE_ENGINE_OPTIONS: !!null + +mongo_logpath: "{{ mongo_log_dir }}/mongodb.log" +mongo_dbpath: "{{ mongo_data_dir }}/mongodb" + +# In environments that do not require durability (devstack / Jenkins) +# you can disable the journal to reduce disk usage +mongo_enable_journal: true + +MONGO_LOG_SERVERSTATUS: true + +# Vars for configuring a mongo backup node. If enabled, this node will be provisioned with a script that uses mongodump +# to backup the database to an ebs volume at a period set by mongo_backup_cron. +# Set MONGO_BACKUP_ENABLED to true to enable. If enabled, all the other MONGO_BACKUP_ vars must be set according to your +# setup. +MONGO_BACKUP_ENABLED: false +MONGO_BACKUP_NODE: "" # note: most likely the ip address of the instance on which to perform the backups +MONGO_BACKUP_EBS_VOLUME_DEVICE: "" +MONGO_BACKUP_EBS_VOLUME_ID: "" +MONGO_BACKUP_AUTH_DATABASE: "" +MONGO_BACKUP_PRUNE_OLDER_THAN_DATE: "" # passed to `date -d`; should be a relative date like "-30days" +MONGO_BACKUP_SNITCH_URL: "" # Optional URL that will be used to ping a monitoring service (such as Dead Man's Snitch) upon successful completion of a backup. +MONGO_BACKUP_VOLUME_MOUNT_PATH: "/mnt/mongo-backup" +MONGO_BACKUP_SNAPSHOT_DESC: "mongo-backup" +mongo_backup_script_path: "/usr/local/sbin/backup-mongo.sh" +mongo_backup_cron: + minute: '12' + hour: '*/12' + day: '*' + month: '*' + weekday: '*' + +# Internal variable set to true dynamically if backups enabled and playbook running on MONGO_BACKUP_NODE. Do not +# manually override. +is_backup_node: false diff --git a/playbooks/roles/mongo_4_4/files/etc/systemd/system/disable-transparent-hugepages.service b/playbooks/roles/mongo_4_4/files/etc/systemd/system/disable-transparent-hugepages.service new file mode 100644 index 00000000000..282c9e122c3 --- /dev/null +++ b/playbooks/roles/mongo_4_4/files/etc/systemd/system/disable-transparent-hugepages.service @@ -0,0 +1,11 @@ +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +Before=mongod.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' + +[Install] +RequiredBy=mongod.service diff --git a/playbooks/roles/mongo_4_4/meta/main.yml b/playbooks/roles/mongo_4_4/meta/main.yml new file mode 100644 index 00000000000..d7223454526 --- /dev/null +++ b/playbooks/roles/mongo_4_4/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - common + - role: mount_ebs + volumes: "{{ MONGO_VOLUMES }}" diff --git a/playbooks/roles/mongo_4_4/tasks/main.yml b/playbooks/roles/mongo_4_4/tasks/main.yml new file mode 100644 index 00000000000..f16432b6322 --- /dev/null +++ b/playbooks/roles/mongo_4_4/tasks/main.yml @@ -0,0 +1,402 @@ +--- +- name: Add disable transparent huge pages systemd service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + copy: + src: etc/systemd/system/disable-transparent-hugepages.service + dest: "/etc/systemd/system/disable-transparent-hugepages.service" + owner: root + group: root + mode: 0644 + tags: + - "hugepages" + - "install" + - "install:configuration" + +- name: Enable/start disable transparent huge pages service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + service: + name: disable-transparent-hugepages + enabled: yes + state: started + tags: + - "hugepages" + - "manage" + - "manage:start" + +- name: install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ PYMONGO_VERSION }}" + extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" + state: present + retries: 3 + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install mongo server and recommends + apt: + pkg: "{{ item }}" + state: present + install_recommends: yes + force: yes + update_cache: yes + register: install_mongo_package + with_items: "{{ mongodb_debian_pkgs }}" + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: create mongo dirs + file: + path: "{{ item }}" + state: directory + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + with_items: + - "{{ mongo_data_dir }}" + - "{{ mongo_dbpath }}" + - "{{ mongo_log_dir }}" + - "{{ mongo_journal_dir }}" + tags: + - "install" + - "install:app-configuration" + +# This will error when run on a new replica set, so we ignore_errors +# and connect anonymously next. +- name: determine if there is a replica set already + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + run_once: true + register: authed_replica_set_already_configured + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +- name: Try checking the replica set with no user/pass in case this is a new box + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + run_once: true + register: unauthed_replica_set_already_configured + when: authed_replica_set_already_configured.failed is defined + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +# We use these in the templates but also to control a whole bunch of logic +- name: set facts that default to not initializing a replica set + set_fact: + initialize_replica_set: false + skip_replica_set: false + tags: + - "install" + - "install:app-configuration" + - "update_mongod_conf" + - "manage" + - "manage:db-replication" + +# If either auth or unauthed access comes back with a replica set, we +# do not want to initialize one. Since initialization requires a bunch +# of extra templating and restarting, it's not something we want to do on +# existing boxes. +- name: track if you have a replica set + set_fact: + initialize_replica_set: true + skip_replica_set: true + when: authed_replica_set_already_configured.status is not defined + and unauthed_replica_set_already_configured.status is not defined + tags: + - "manage" + - "manage:db-replication" + +- name: warn about unconfigured replica sets + debug: msg="You do not appear to have a Replica Set configured, deploying one for you" + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: copy mongodb key file + copy: + content: "{{ MONGO_CLUSTER_KEY }}" + dest: "{{ mongo_key_file }}" + mode: 0600 + owner: mongodb + group: mongodb + register: update_mongod_key + tags: + - "manage" + - "manage:db-replication" + - "mongodb_key" + +# If skip_replica_set is true, this template will not contain a replica set stanza +# because of the fact above. +- name: copy configuration template + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + register: update_mongod_conf + tags: + - "install" + - "install:app-configuration" + - "manage" + - "manage:db-replication" + - "update_mongod_conf" + +# This sets the is_backup_node var by checking whether +# mongo backups are enabled AND we're currently running against the designated mongo backup node. +# This allows backup-related tasks below to determine whether or not they should run on the current mongo node. +- name: determine if backup tasks should run + set_fact: + is_backup_node: true + when: MONGO_BACKUP_ENABLED and '{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}' == '{{ MONGO_BACKUP_NODE }}' + tags: + - "backup:mongo" + +- name: install logrotate configuration + template: + src: mongo_logrotate.j2 + dest: /etc/logrotate.d/hourly/mongo + tags: + - "backup:mongo" + - "install" + - "install:app-configuration" + - "logrotate" + +- name: install prereqs for backup script + apt: + pkg: "{{ item }}" + state: present + update_cache: yes + with_items: + - jq + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install backup script + template: + src: backup-mongo.sh.j2 + dest: "{{ mongo_backup_script_path }}" + mode: 0700 + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: add mongo backup script to cron + cron: + name: mongo backup job + minute: "{{ mongo_backup_cron.minute | default('12') }}" + hour: "{{ mongo_backup_cron.hour | default('*/12') }}" + day: "{{ mongo_backup_cron.day | default('*') }}" + month: "{{ mongo_backup_cron.month | default('*') }}" + weekday: "{{ mongo_backup_cron.weekday | default('*') }}" + job: "{{ mongo_backup_script_path }} >> {{ mongo_log_dir }}/mongo-backup.log 2>&1" + become: yes + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: format mongo backup volume + filesystem: + dev: "{{ MONGO_BACKUP_EBS_VOLUME_DEVICE }}" + fstype: ext4 + force: true + ignore_errors: true + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# We only try passwordless superuser creation when +# we're initializing the replica set and need to use +# the localhost exemption to create a user who will be +# able to initialize the replica set. +# We can only create the users on one machine, the one +# where we will initialize the replica set. If we +# create users on multiple hosts, then they will fail +# to come into the replica set. +- name: create super user + mongodb_user: + name: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + database: admin + roles: root + when: initialize_replica_set + run_once: true + tags: + - "manage" + - "manage:db-replication" + +# Now that the localhost exemption has been used to create the superuser, we need +# to add replica set to our configuration. This will never happen if we detected +# a replica set in the 'determine if there is a replica set already' task. +- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set + set_fact: + skip_replica_set: false + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: re-copy configuration template with replica set enabled + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: restart mongo service + service: + name: mongod + state: restarted + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: configure replica set + mongodb_replica_set: + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + rs_config: "{{ MONGO_RS_CONFIG }}" + run_once: true + register: replset_status + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + - "manage:db-replication-configuration" + +# During initial replica set configuration, it can take a few seconds to vote +# a primary and for all members to reflect that status. During that window, +# use creation or other writes can fail. The best wait/check seems to be repeatedly +# checking the replica set status until we see a PRIMARY in the results. +- name: Wait for the replica set to update and (if needed) elect a primary + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + register: status + until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list + when: mongo_configure_replica_set + retries: 5 + delay: 2 + run_once: true + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + +- name: create mongodb users in a replica set + mongodb_user: + database: "{{ item.database }}" + login_database: 'admin' + login_user: "{{ MONGO_ADMIN_USER }}" + login_password: "{{ MONGO_ADMIN_PASSWORD }}" + name: "{{ item.user }}" + password: "{{ item.password }}" + roles: "{{ item.roles }}" + state: present + replica_set: "{{ MONGO_REPL_SET }}" + with_items: "{{ MONGO_USERS }}" + run_once: true + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-users" + - "manage:db-replication" + +- name: ensure mongo starts at boot time + service: + name: mongod + enabled: yes + tags: + - "manage" + - "manage:start" + +- name: add serverStatus logging script + template: + src: "log-mongo-serverStatus.sh.j2" + dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh" + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + mode: 0700 + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" + +- name: add serverStatus logging script to cron + cron: + name: mongostat logging job + minute: "*/3" + job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1 + become: yes + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" diff --git a/playbooks/roles/mongo_4_4/templates/log-mongo-serverStatus.sh.j2 b/playbooks/roles/mongo_4_4/templates/log-mongo-serverStatus.sh.j2 new file mode 100644 index 00000000000..04649d55ad1 --- /dev/null +++ b/playbooks/roles/mongo_4_4/templates/log-mongo-serverStatus.sh.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON +/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' diff --git a/playbooks/roles/mongo_4_4/templates/mongo_logrotate.j2 b/playbooks/roles/mongo_4_4/templates/mongo_logrotate.j2 new file mode 100644 index 00000000000..f2fb4483566 --- /dev/null +++ b/playbooks/roles/mongo_4_4/templates/mongo_logrotate.j2 @@ -0,0 +1,46 @@ +{{ mongo_log_dir }}/serverStatus.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} + +{% if is_backup_node %} +{{ mongo_log_dir }}/mongo-backup.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} +{% endif %} + +{{ mongo_log_dir }}/mongodb.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M + postrotate + /usr/bin/killall -USR1 mongod + endscript +} diff --git a/playbooks/roles/mongo_4_4/templates/mongod.conf.j2 b/playbooks/roles/mongo_4_4/templates/mongod.conf.j2 new file mode 100644 index 00000000000..b7d4b4a1efe --- /dev/null +++ b/playbooks/roles/mongo_4_4/templates/mongod.conf.j2 @@ -0,0 +1,46 @@ +# {{ ansible_managed }} +# mongodb.conf + + +storage: + # Where to store the data. + dbPath: {{ mongo_dbpath }} + # Storage Engine + engine: {{ MONGO_STORAGE_ENGINE }} + # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling + journal: +{% if mongo_enable_journal %} + enabled: true +{% else %} + enabled: false +{% endif %} +{% if MONGO_STORAGE_ENGINE_OPTIONS %} + {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} +{% endif %} + +systemLog: + #where to log + destination: file + path: "{{ mongo_logpath }}" +{% if mongo_logappend %} + logAppend: true +{% else %} + logAppend: false +{% endif %} + logRotate: {{ mongo_logrotate }} + +{% if not skip_replica_set %} +replication: + replSetName: {{ MONGO_REPL_SET }} + +security: + authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }} + keyFile: {{ mongo_key_file }} + +{% endif %} +net: + bindIp: {{ MONGO_BIND_IP }} + port: {{ mongo_port }} + + +{{ mongo_extra_conf }} From ee3158b84c48499dbd66b86d8984b425681acdf2 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 24 May 2021 14:59:44 +0530 Subject: [PATCH 068/664] add feature of uploading edx services log to s3 bucket --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 29eebfe7cc2..090806c6f2b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,4 +13,9 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript + lastaction + {% if COMMON_OBJECT_STORE_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + {% endif -%} + endscript } From c85cd31e3e04c6f3887334a1cca19224b4dbbde9 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 31 May 2021 11:12:09 +0530 Subject: [PATCH 069/664] add edx services own flag to upload logs --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- playbooks/roles/common_vars/defaults/main.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 090806c6f2b..796329e6fb0 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,8 +14,8 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" {% endif -%} endscript } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 4949012d8aa..d46c9b52bbb 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,6 +23,9 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" +COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_LOG_EDX_SYNC : False + COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From 8b6124c17bee461dfc7c48dacb7f6e678d452d7b Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:01:16 +0530 Subject: [PATCH 070/664] rename flag to more appropriate name --- playbooks/roles/common_vars/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index d46c9b52bbb..bc6539b628b 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,8 +23,8 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" -COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" -COMMON_OBJECT_STORE_LOG_EDX_SYNC : False +COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_EDX_LOG_SYNC : False COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" From 2e7700a088461c1bf9560459bcf618e33e51b93a Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:02:47 +0530 Subject: [PATCH 071/664] fix the edx serverices log folder --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 796329e6fb0..92de9dbd261 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,8 +14,9 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" {% endif -%} endscript } From 1a4a9b30d8b04a3aa859390ca41d298081c29a08 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:29:27 +0530 Subject: [PATCH 072/664] handle the case where file name same in both the services --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 92de9dbd261..c8528b25c7b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -15,8 +15,8 @@ endscript lastaction {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" {% endif -%} endscript } From 549a930c7ceba42acdf92eeb6b421b76ccdab8a7 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:30:56 +0530 Subject: [PATCH 073/664] add commands to sync the edx services logs --- playbooks/retire_host.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 32fddd184f1..9684c8ec9ae 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,8 +37,10 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" - + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + - name: Run minos verification hosts: "{{TARGET}}" become: True From 8562f4d1a6cf14fb8eca865064e3251e6c694b53 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 7 Jun 2021 17:00:29 +0530 Subject: [PATCH 074/664] compress the command with items --- playbooks/retire_host.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 9684c8ec9ae..19388cc0d59 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,9 +37,11 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" + with_items: + - { src: '/tracking/', dest: '{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}' } + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } + - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - name: Run minos verification hosts: "{{TARGET}}" From 2b56c5b6cb9fdd07ff1365ec8b7952c4456573bf Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 075/664] Revert "compress the command with items" This reverts commit 8562f4d1a6cf14fb8eca865064e3251e6c694b53. --- playbooks/retire_host.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 19388cc0d59..9684c8ec9ae 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,11 +37,9 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" - with_items: - - { src: '/tracking/', dest: '{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}' } - - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" - name: Run minos verification hosts: "{{TARGET}}" From 7023d9fba5a81bfad01a99e6be0da3dcc92fe32d Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 076/664] Revert "add commands to sync the edx services logs" This reverts commit 549a930c7ceba42acdf92eeb6b421b76ccdab8a7. --- playbooks/retire_host.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 9684c8ec9ae..32fddd184f1 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,10 +37,8 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" - + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" + - name: Run minos verification hosts: "{{TARGET}}" become: True From 469f7c8c4eabcf176f8d4de53922c8243b6d9ce9 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 077/664] Revert "handle the case where file name same in both the services" This reverts commit 1a4a9b30d8b04a3aa859390ca41d298081c29a08. --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index c8528b25c7b..92de9dbd261 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -15,8 +15,8 @@ endscript lastaction {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" {% endif -%} endscript } From 58a242d070eff2257a923315b73472ba31cae9d4 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 078/664] Revert "fix the edx serverices log folder" This reverts commit 2e7700a088461c1bf9560459bcf618e33e51b93a. --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 92de9dbd261..796329e6fb0 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,9 +14,8 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" {% endif -%} endscript } From 9416dd71b5ad39823e92358fe3a1894797d251b4 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 079/664] Revert "rename flag to more appropriate name" This reverts commit 8b6124c17bee461dfc7c48dacb7f6e678d452d7b. --- playbooks/roles/common_vars/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index bc6539b628b..d46c9b52bbb 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,8 +23,8 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" -COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" -COMMON_OBJECT_STORE_EDX_LOG_SYNC : False +COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_LOG_EDX_SYNC : False COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" From 922dbbf8f6b20e9122fcba4ffa7611f500e95ec4 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 080/664] Revert "add edx services own flag to upload logs" This reverts commit c85cd31e3e04c6f3887334a1cca19224b4dbbde9. --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- playbooks/roles/common_vars/defaults/main.yml | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 796329e6fb0..090806c6f2b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,8 +14,8 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" + {% if COMMON_OBJECT_STORE_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" {% endif -%} endscript } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index d46c9b52bbb..4949012d8aa 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,9 +23,6 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" -COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" -COMMON_OBJECT_STORE_LOG_EDX_SYNC : False - COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From 878a4f9426d64b14732634c8cb3ad7b7dd701758 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 30 Jun 2021 08:00:42 +0500 Subject: [PATCH 081/664] Revert "add feature of uploading edx services log to s3 bucket" This reverts commit ee3158b84c48499dbd66b86d8984b425681acdf2. --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 090806c6f2b..29eebfe7cc2 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,9 +13,4 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript - lastaction - {% if COMMON_OBJECT_STORE_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" - {% endif -%} - endscript } From e7266b92768c0477e9d5dc73ec9fec7a7e1b4a1b Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 2 Jul 2021 15:18:06 -0400 Subject: [PATCH 082/664] Fix bug, use cluster name instead of stale var --- util/check_rds_configs/check_rds_configs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/check_rds_configs/check_rds_configs.py b/util/check_rds_configs/check_rds_configs.py index 70864438d06..6e22ae55f06 100644 --- a/util/check_rds_configs/check_rds_configs.py +++ b/util/check_rds_configs/check_rds_configs.py @@ -109,8 +109,9 @@ def cli(db_engine, ignore): for cluster in db_clusters: arn = cluster['DBClusterArn'] + db_cluster_identifier = cluster['DBClusterIdentifier'] tags = rds.list_tags_for_resource(ResourceName=arn)['TagList'] - exit_status, clusters_without_tags = check_tags(clusters_without_tags, db_identifier, tags) + exit_status, clusters_without_tags = check_tags(clusters_without_tags, db_cluster_identifier, tags) if cluster['CopyTagsToSnapshot'] == False: cluster_with_disabled_snapshot_tags.append(cluster['DBClusterIdentifier']) From 80e2afa3213eb755d46d44531116ea0e02cad547 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 2 Jul 2021 15:18:19 -0400 Subject: [PATCH 083/664] Add prints to debug jenkins job failure --- util/check_rds_configs/check_rds_configs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/check_rds_configs/check_rds_configs.py b/util/check_rds_configs/check_rds_configs.py index 6e22ae55f06..7ff1a3315cb 100644 --- a/util/check_rds_configs/check_rds_configs.py +++ b/util/check_rds_configs/check_rds_configs.py @@ -99,6 +99,7 @@ def cli(db_engine, ignore): arn = instance['DBInstanceArn'] tags = rds.list_tags_for_resource(ResourceName=arn)['TagList'] db_identifier = instance['DBInstanceIdentifier'] + print("Checking tags on DB instance {}".format(db_identifier)) exit_status, instances_without_tags = check_tags(instances_without_tags, db_identifier, tags) if db_identifier not in ignore_rds and "test" not in db_identifier: @@ -111,12 +112,14 @@ def cli(db_engine, ignore): arn = cluster['DBClusterArn'] db_cluster_identifier = cluster['DBClusterIdentifier'] tags = rds.list_tags_for_resource(ResourceName=arn)['TagList'] + print("Checking cluster tags on DB cluster {}".format(db_cluster_identifier)) exit_status, clusters_without_tags = check_tags(clusters_without_tags, db_cluster_identifier, tags) if cluster['CopyTagsToSnapshot'] == False: cluster_with_disabled_snapshot_tags.append(cluster['DBClusterIdentifier']) for instance in cluster['DBClusterMembers']: db_identifier = instance['DBInstanceIdentifier'] + print("Checking tags on cluster DB instance {}".format(db_identifier)) if db_identifier not in ignore_rds and "test" not in db_identifier: db_instance_parameter_groups[db_identifier]['cluster'] = cluster['DBClusterParameterGroup'] if instance["DBClusterParameterGroupStatus"] != "in-sync": @@ -124,6 +127,7 @@ def cli(db_engine, ignore): for db_identifier, parameter_groups in db_instance_parameter_groups.items(): + print("Checking paramter groups on DB {}".format(db_identifier)) instance_parameter_group_name = parameter_groups['instance']['DBParameterGroupName'] if parameter_groups['instance']['ParameterApplyStatus'] != "in-sync": instances_out_of_sync_with_instance_parameters.append(db_identifier) From b0f7e53b833e691b1d50832b116580cfdb601eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cabrita?= Date: Mon, 5 Jul 2021 01:31:58 +0100 Subject: [PATCH 084/664] feat: Allow installing extra requirements when using the `credentials` role. (#6449) This makes the `credentials` role support the use of `edx_django_service_extra_requirements` as done originally in [!6400](https://github.com/edx/configuration/pull/6400) Related tickets: * [BB-4342](https://tasks.opencraft.com/browse/BB-4342) --- CHANGELOG.md | 4 ++++ playbooks/roles/credentials/defaults/main.yml | 2 ++ playbooks/roles/credentials/meta/main.yml | 1 + 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 411beb4b2b1..7cb5b9e13e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-06-17 + - Role credentials + - Installs extra python packages specified in `CREDENTIALS_EXTRA_REQUIREMENTS` (defaults to `[]`). + - 2021-06-07 - In `openedx_native.yml` - Added configuration variable ECOMMERCE_CSRF_TRUSTED_ORIGINS to allow payment mfe to interact with ecommerce service diff --git a/playbooks/roles/credentials/defaults/main.yml b/playbooks/roles/credentials/defaults/main.yml index ff6e555ce54..4609382c5b4 100644 --- a/playbooks/roles/credentials/defaults/main.yml +++ b/playbooks/roles/credentials/defaults/main.yml @@ -108,6 +108,8 @@ CREDENTIALS_GUNICORN_EXTRA: '' CREDENTIALS_EXTRA_APPS: [] +CREDENTIALS_EXTRA_REQUIREMENTS: [] + CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE: false CREDENTIALS_CERTIFICATE_LANGUAGES: diff --git a/playbooks/roles/credentials/meta/main.yml b/playbooks/roles/credentials/meta/main.yml index 7d0866525c7..f95f098be69 100644 --- a/playbooks/roles/credentials/meta/main.yml +++ b/playbooks/roles/credentials/meta/main.yml @@ -40,6 +40,7 @@ dependencies: edx_django_service_backend_service_edx_oauth2_secret: '{{ CREDENTIALS_BACKEND_SERVICE_EDX_OAUTH2_SECRET }}' edx_django_service_social_auth_redirect_is_https: '{{ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}' edx_django_service_extra_apps: '{{ CREDENTIALS_EXTRA_APPS }}' + edx_django_service_extra_requirements: '{{ CREDENTIALS_EXTRA_REQUIREMENTS }}' edx_django_service_session_expire_at_browser_close: '{{ CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' edx_django_service_node_version: '{{ CREDENTIALS_NODE_VERSION }}' edx_django_service_automated_users: '{{ CREDENTIALS_AUTOMATED_USERS }}' From e92097addf7558a2e4f1ba1e2a6b24567e1fd1e8 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 6 Jul 2021 16:52:19 +0500 Subject: [PATCH 085/664] update sandbox base AMI to use ubuntu 20.04 image --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 1b6b54a5c03..d7a215f3edb 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -128,7 +128,7 @@ fi if [[ -z $ami ]]; then if [[ $server_type == "full_edx_installation" ]]; then - ami="ami-087649e61b3299e66" + ami="ami-03780d7dca0f55038" elif [[ $server_type == "ubuntu_18.04" ]]; then ami="ami-07ebfd5b3428b6f4d" elif [[ $server_type == "ubuntu_20.04" || $server_type == "full_edx_installation_from_scratch" ]]; then From 1c1385d2639434649191b07a232233cf5e8ed38e Mon Sep 17 00:00:00 2001 From: Arbab Khalil Date: Thu, 8 Jul 2021 14:02:19 +0500 Subject: [PATCH 086/664] Updated mongo role in edx CI playbook --- playbooks/edx_continuous_integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 13e76e137ea..d8276715853 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -22,7 +22,7 @@ - role: edxlocal tags: edxlocal - memcache - - mongo_4_0 + - mongo_4_2 - role: redis - { role: 'edxapp', celery_worker: True } - edxapp From 7ed65a66a6789c67858f13fdd3224cf2ed31cc5a Mon Sep 17 00:00:00 2001 From: pkulkark Date: Thu, 1 Jul 2021 15:44:17 +0530 Subject: [PATCH 087/664] feat: Add feature to sync edx services logs This allows edx services logs to be synced to object storage, when the flag COMMON_OBJECT_STORE_EDX_LOG_SYNC is set to true. --- playbooks/retire_host.yml | 6 ++++++ .../templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 6 ++++++ playbooks/roles/common_vars/defaults/main.yml | 3 +++ 3 files changed, 15 insertions(+) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 32fddd184f1..6c6168357be 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -38,6 +38,12 @@ # This catches the case where tracking.log is 0b - name: Sync again command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" + - name: Sync edX services logs again + command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" + with_items: + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } + - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } + when: COMMON_OBJECT_STORE_EDX_LOG_SYNC - name: Run minos verification hosts: "{{TARGET}}" diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 29eebfe7cc2..c8528b25c7b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,4 +13,10 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript + lastaction + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + {% endif -%} + endscript } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 4949012d8aa..e57b9fee9a0 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,6 +23,9 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" +COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_EDX_LOG_SYNC : false + COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From 9323d3695c7fee42b567cb8740c34eb7ae1b1d49 Mon Sep 17 00:00:00 2001 From: pkulkark Date: Thu, 8 Jul 2021 15:24:39 +0530 Subject: [PATCH 088/664] fix: move if statement to contain "lastaction" --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index c8528b25c7b..aa1bf1ce8c8 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,10 +13,10 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} lastaction - {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" - {% endif -%} endscript + {% endif -%} } From 281b28976e2eb89c70510bc542d73763a67f5777 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 13:45:12 -0400 Subject: [PATCH 089/664] Fix retire_host.yml --- playbooks/retire_host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 6c6168357be..c8a7965a1dc 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -43,7 +43,7 @@ with_items: - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - when: COMMON_OBJECT_STORE_EDX_LOG_SYNC + when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true - name: Run minos verification hosts: "{{TARGET}}" From 482b1ff87c0ca61643f7d87feca198d71a6809f9 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 13:57:00 -0400 Subject: [PATCH 090/664] Revert "Fix retire_host.yml" This reverts commit 281b28976e2eb89c70510bc542d73763a67f5777. --- playbooks/retire_host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index c8a7965a1dc..6c6168357be 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -43,7 +43,7 @@ with_items: - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true + when: COMMON_OBJECT_STORE_EDX_LOG_SYNC - name: Run minos verification hosts: "{{TARGET}}" From 78960e197cc4256af4921dba70b0c3ea46b8a352 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 13:57:20 -0400 Subject: [PATCH 091/664] Revert "fix: move if statement to contain "lastaction"" This reverts commit 9323d3695c7fee42b567cb8740c34eb7ae1b1d49. --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index aa1bf1ce8c8..c8528b25c7b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,10 +13,10 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript - {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} lastaction + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + {% endif -%} endscript - {% endif -%} } From c57481dcdd1a4be744cb94d65f4f11584fb1e439 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 13:57:24 -0400 Subject: [PATCH 092/664] Revert "feat: Add feature to sync edx services logs" This reverts commit 7ed65a66a6789c67858f13fdd3224cf2ed31cc5a. --- playbooks/retire_host.yml | 6 ------ .../templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 6 ------ playbooks/roles/common_vars/defaults/main.yml | 3 --- 3 files changed, 15 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 6c6168357be..32fddd184f1 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -38,12 +38,6 @@ # This catches the case where tracking.log is 0b - name: Sync again command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" - - name: Sync edX services logs again - command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" - with_items: - - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - when: COMMON_OBJECT_STORE_EDX_LOG_SYNC - name: Run minos verification hosts: "{{TARGET}}" diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index c8528b25c7b..29eebfe7cc2 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,10 +13,4 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript - lastaction - {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" - {% endif -%} - endscript } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index e57b9fee9a0..4949012d8aa 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,9 +23,6 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" -COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" -COMMON_OBJECT_STORE_EDX_LOG_SYNC : false - COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From 67f4fe01c7ce5f98baa925f13c65b963d0d7b919 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 14:20:23 -0400 Subject: [PATCH 093/664] feat: Add feature to sync edx services logs --- playbooks/retire_host.yml | 6 ++++++ .../templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 6 ++++++ playbooks/roles/common_vars/defaults/main.yml | 3 +++ 3 files changed, 15 insertions(+) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 32fddd184f1..6c6168357be 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -38,6 +38,12 @@ # This catches the case where tracking.log is 0b - name: Sync again command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" + - name: Sync edX services logs again + command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" + with_items: + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } + - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } + when: COMMON_OBJECT_STORE_EDX_LOG_SYNC - name: Run minos verification hosts: "{{TARGET}}" diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 29eebfe7cc2..aa1bf1ce8c8 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,4 +13,10 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} + lastaction + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + endscript + {% endif -%} } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 4949012d8aa..e57b9fee9a0 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,6 +23,9 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" +COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_EDX_LOG_SYNC : false + COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From e72d40a299be0ae56ef2147dca4727be219c7820 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 14:20:36 -0400 Subject: [PATCH 094/664] Fix retire_host.yml --- playbooks/retire_host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 6c6168357be..c8a7965a1dc 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -43,7 +43,7 @@ with_items: - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - when: COMMON_OBJECT_STORE_EDX_LOG_SYNC + when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true - name: Run minos verification hosts: "{{TARGET}}" From adde3a293bf938cec10e9087323efcd67eb8eea9 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 14:22:11 -0400 Subject: [PATCH 095/664] Fix retire_host.yml missing variable --- playbooks/retire_host.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index c8a7965a1dc..6b1b6b8f94e 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -41,8 +41,8 @@ - name: Sync edX services logs again command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" with_items: - - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } - - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | defaut("logs/edx/") }}lms/' } + - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | default("logs/edx/") }}cms/' } when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true - name: Run minos verification From aa9ed9d173f3d2e2e6aaace14092c3255da270b7 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 15:36:19 -0400 Subject: [PATCH 096/664] Fix typo --- playbooks/retire_host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 6b1b6b8f94e..9f1a3d0dc2d 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -41,7 +41,7 @@ - name: Sync edX services logs again command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" with_items: - - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | defaut("logs/edx/") }}lms/' } + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | default("logs/edx/") }}lms/' } - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | default("logs/edx/") }}cms/' } when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true From 88fdd78c59214a152ee2644200d833a6f2a468de Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 8 Jul 2021 15:45:45 -0400 Subject: [PATCH 097/664] Fix conditional --- playbooks/retire_host.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 9f1a3d0dc2d..fc7a4541780 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -43,7 +43,7 @@ with_items: - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | default("logs/edx/") }}lms/' } - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX | default("logs/edx/") }}cms/' } - when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC is true + when: (COMMON_OBJECT_STORE_EDX_LOG_SYNC is defined) and COMMON_OBJECT_STORE_EDX_LOG_SYNC == true - name: Run minos verification hosts: "{{TARGET}}" From d1f7063a36b19c4c06c6b1e40b62906a9f699a3f Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 12 Jul 2021 11:14:52 +0500 Subject: [PATCH 098/664] Upgrade ES version to 7.10 --- playbooks/roles/elasticsearch/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/elasticsearch/defaults/main.yml b/playbooks/roles/elasticsearch/defaults/main.yml index 9216f66c0c7..f46548ff6fc 100644 --- a/playbooks/roles/elasticsearch/defaults/main.yml +++ b/playbooks/roles/elasticsearch/defaults/main.yml @@ -13,4 +13,4 @@ elasticsearch_group: "elasticsearch" # Defaults for a single server installation. ELASTICSEARCH_CLUSTER_MEMBERS: [] ELASTICSEARCH_HEAP_SIZE: "512m" -ELASTICSEARCH_VERSION: "7.8.1" +ELASTICSEARCH_VERSION: "7.10.2" From 6b73c099627cb4d62f6d887a8572b092b6bd1c0b Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 12 Jul 2021 14:13:59 +0500 Subject: [PATCH 099/664] Update sandbox AMI for ES upgrade --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index d7a215f3edb..6ca9115a0cb 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -128,7 +128,7 @@ fi if [[ -z $ami ]]; then if [[ $server_type == "full_edx_installation" ]]; then - ami="ami-03780d7dca0f55038" + ami="ami-0644020c3c81d30ba" elif [[ $server_type == "ubuntu_18.04" ]]; then ami="ami-07ebfd5b3428b6f4d" elif [[ $server_type == "ubuntu_20.04" || $server_type == "full_edx_installation_from_scratch" ]]; then From b9db454538c3e032f55434a1be91f073dcda4591 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 14 Jul 2021 11:37:54 +0500 Subject: [PATCH 100/664] feat: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- playbooks/roles/mongo_mms/tasks/main.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index 7d1d2bd7c2a..afb927c6223 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "10.24.1.6720-1" + version: "11.2.0.6985-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" diff --git a/playbooks/roles/mongo_mms/tasks/main.yml b/playbooks/roles/mongo_mms/tasks/main.yml index dca68602e95..71253e4e39a 100644 --- a/playbooks/roles/mongo_mms/tasks/main.yml +++ b/playbooks/roles/mongo_mms/tasks/main.yml @@ -11,13 +11,12 @@ when: MMSAPIKEY is not defined # Install mongoDB agent -- name: download xenial mongodb agent +- name: download mongodb agent get_url: url: "{{ base_url }}/{{ item.dir }}/{{ item.agent }}_{{ item.version }}_{{ pkg_arch }}.{{ os_version }}.{{ pkg_format }}" dest: "/tmp/{{ item.agent }}-{{ item.version }}.{{ pkg_format }}" register: download_mongodb_deb with_items: "{{ mongodb_agent }}" - when: ansible_distribution_release == 'xenial' - name: install mongodb agent apt: From e920d5ff91dd587ed3fd2900f5091a36b2d70bf0 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Thu, 15 Jul 2021 15:49:21 +0800 Subject: [PATCH 101/664] Specify mongo replica set protocolVersion. (#6468) Mongo 4 no longer supports protocolVersion 0, so we need to explicitly specify protocolVersion: 1. --- playbooks/roles/mongo_4_0/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/mongo_4_0/defaults/main.yml b/playbooks/roles/mongo_4_0/defaults/main.yml index 38d59a67f2c..591e5826591 100644 --- a/playbooks/roles/mongo_4_0/defaults/main.yml +++ b/playbooks/roles/mongo_4_0/defaults/main.yml @@ -59,6 +59,7 @@ MONGO_CLUSTER_KEY: "CHANGEME" # Fed directly into mongodb_replica_set module MONGO_RS_CONFIG: _id: '{{ MONGO_REPL_SET }}' + protocolVersion: 1 members: - host: '127.0.0.1' From 29555e7fee15fce8ba270fd89e90548d9c71f3bd Mon Sep 17 00:00:00 2001 From: Justin Hynes Date: Thu, 15 Jul 2021 13:00:07 -0400 Subject: [PATCH 102/664] chore!: Remove unused `AUDIT_CERT_CUTOFF_DATE` setting from configuration [MICROBA-1351] [DEPR-159] - Remove unused `AUDIT_CERT_CUTOFF_DATE` setting from configuration. --- docker/build/edxapp/lms.yml | 1 - playbooks/roles/edxapp/defaults/main.yml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 9e0a085ac23..4674488ec20 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -20,7 +20,6 @@ ANALYTICS_DASHBOARD_URL: http://localhost:18110/courses API_ACCESS_FROM_EMAIL: api-requests@example.com API_ACCESS_MANAGER_EMAIL: api-access@example.com API_DOCUMENTATION_URL: http://course-catalog-api-guide.readthedocs.io/en/latest/ -AUDIT_CERT_CUTOFF_DATE: null AUTH_DOCUMENTATION_URL: http://course-catalog-api-guide.readthedocs.io/en/latest/authentication/index.html AUTH_PASSWORD_VALIDATORS: - NAME: django.contrib.auth.password_validation.UserAttributeSimilarityValidator diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index ffe0a396904..f89eb3f8fb6 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -199,9 +199,6 @@ EDXAPP_CREDIT_HELP_LINK_URL: "" EDXAPP_PARTNER_SUPPORT_EMAIL: '' -# ISO-8601 formatted date string, or null -EDXAPP_AUDIT_CERT_CUTOFF_DATE: null - EDXAPP_PLATFORM_NAME: 'Your Platform Name Here' EDXAPP_PLATFORM_DESCRIPTION: 'Your Platform Description Here' EDXAPP_STUDIO_NAME: 'Studio' @@ -1570,7 +1567,6 @@ lms_env_config: CREDIT_HELP_LINK_URL: "{{ EDXAPP_CREDIT_HELP_LINK_URL }}" MAILCHIMP_NEW_USER_LIST_ID: "{{ EDXAPP_MAILCHIMP_NEW_USER_LIST_ID }}" CONTACT_MAILING_ADDRESS: "{{ EDXAPP_CONTACT_MAILING_ADDRESS }}" - AUDIT_CERT_CUTOFF_DATE: "{{ EDXAPP_AUDIT_CERT_CUTOFF_DATE }}" API_ACCESS_MANAGER_EMAIL: "{{ EDXAPP_API_ACCESS_MANAGER_EMAIL }}" API_ACCESS_FROM_EMAIL: "{{ EDXAPP_API_ACCESS_FROM_EMAIL }}" API_DOCUMENTATION_URL: "{{ EDXAPP_API_DOCUMENTATION_URL }}" From 92ffe2eccbc0f5d5b085d48cce7d8684fecf2e73 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 19 Jul 2021 10:57:06 +0500 Subject: [PATCH 103/664] feat: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index afb927c6223..f61b76d2f11 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.2.0.6985-1" + version: "11.2.1.6990-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From a380f3ca18060a9fc376392644c70fedef7c4909 Mon Sep 17 00:00:00 2001 From: Saleem Latif Date: Mon, 21 Jun 2021 10:52:25 +0500 Subject: [PATCH 104/664] Corrected tableau_installer_dir path by removing a forward slash at the end. --- playbooks/roles/tableau/defaults/main.yml | 5 +++- playbooks/roles/tableau/tasks/main.yml | 30 ++++++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/tableau/defaults/main.yml b/playbooks/roles/tableau/defaults/main.yml index d243ce25d73..f38d29a7b44 100644 --- a/playbooks/roles/tableau/defaults/main.yml +++ b/playbooks/roles/tableau/defaults/main.yml @@ -13,10 +13,11 @@ tableau_data_dir: "{{ COMMON_DATA_DIR }}/tableau" tableau_app_dir: "{{ COMMON_APP_DIR }}/tableau" tableau_log_dir: "{{ COMMON_LOG_DIR }}/tableau" tableau_code_dir: "{{ tableau_app_dir }}/tableau" -tableau_installer_dir: "{{ tableau_code_dir }}/linux/automated-installer/" +tableau_installer_dir: "{{ tableau_code_dir }}/linux/automated-installer" TABLEAU_ADMIN_USER: tableau TABLEAU_ADMIN_PASSWORD: 'tableau001-pass' +tableau_user_group: "tableau" tableau_user: "{{ TABLEAU_ADMIN_USER }}" tableau_user_createhome: 'yes' tableau_user_shell: '/bin/false' @@ -50,3 +51,5 @@ tableau_registration_config: department: "Enterprise" first_name: "{{ TABLEAU_REGISTRATION_CONFIG_USER_FIRST_NAME }}" email: "{{ TABLEAU_REGISTRATION_CONFIG_USER_EMAIL }}" + +TABLEAU_LICENSE_KEY: "" diff --git a/playbooks/roles/tableau/tasks/main.yml b/playbooks/roles/tableau/tasks/main.yml index 5b93f0aca3a..9e6d49e0779 100644 --- a/playbooks/roles/tableau/tasks/main.yml +++ b/playbooks/roles/tableau/tasks/main.yml @@ -2,12 +2,24 @@ # - group_vars/all # - common/tasks/main.yml --- + +- name: create main group for tableau's default user + group: + name: "{{ tableau_user_group }}" + state: "present" + tags: + - install + - install:base + - name: create application user user: name: "{{ tableau_user }}" home: "{{ tableau_app_dir }}" createhome: "{{ tableau_user_createhome }}" shell: "{{ tableau_user_shell }}" + group: "{{ tableau_user_group }}" + groups: sudo + password: "{{ TABLEAU_ADMIN_PASSWORD | password_hash('sha512') }}" tags: - install - install:base @@ -17,7 +29,7 @@ path: "{{ item.path }}" state: directory owner: "{{ tableau_user }}" - group: "{{ common_web_group }}" + group: "{{ tableau_user_group }}" mode: "{{ item.mode | default(0755) }}" with_items: - { path: "{{ tableau_app_dir }}" } @@ -52,6 +64,7 @@ repo: "{{ tableau_repo }}" version: "{{ tableau_version }}" accept_hostkey: yes + force: yes become_user: "{{ tableau_user }}" register: tableau_checkout tags: @@ -97,9 +110,20 @@ - install - install:code -- name: Run automated installer to actually setup the tableau server - shell: "{{ tableau_installer_dir }}/automated-installer -s {{ tableau_installer_dir }}/secrets -f {{ tableau_installer_dir }}/config.json -r {{ tableau_installer_dir }}/registration.json --accepteula {{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }}" +- name: Ensure "{{ tableau_user }}" is in th sudoers file. + lineinfile: + path: "/etc/sudoers" + state: "present" + regexp: '^%{{ tableau_user }} ALL=' + line: '%{{ tableau_user }} ALL=(ALL) NOPASSWD: ALL' sudo: true tags: - install - install:code + +- name: Run automated installer to actually setup the tableau server + shell: "sudo {{ tableau_installer_dir }}/automated-installer -s {{ tableau_installer_dir }}/secrets -f {{ tableau_installer_dir }}/config.json -r {{ tableau_installer_dir }}/registration.json --accepteula {{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }} -k '{{ TABLEAU_LICENSE_KEY }}'" + become_user: "{{ tableau_user }}" + tags: + - install + - install:code From b5c1067c32d2c2000c6346655982175cb21f64fc Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 11 Jun 2021 14:02:56 -0400 Subject: [PATCH 105/664] Add retries to NewRelic package install --- .../newrelic_infrastructure/tasks/main.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/playbooks/roles/newrelic_infrastructure/tasks/main.yml b/playbooks/roles/newrelic_infrastructure/tasks/main.yml index 2c1205f8145..2a54446dfbb 100644 --- a/playbooks/roles/newrelic_infrastructure/tasks/main.yml +++ b/playbooks/roles/newrelic_infrastructure/tasks/main.yml @@ -39,6 +39,10 @@ - install - install:app-requirements when: ansible_distribution == 'Ubuntu' + retries: 10 + delay: 10 + register: result + until: result is succeeded # For focal, use the bionic repo for now. - name: Install apt repository for New Relic Infrastructure if neither bionic nor focal @@ -50,6 +54,10 @@ - install - install:app-requirements when: ansible_distribution == 'Ubuntu' and (ansible_distribution_release != 'bionic' and ansible_distribution_release != 'focal') + retries: 10 + delay: 10 + register: nr_apt_repo + until: nr_apt_repo is succeeded - name: Install apt repository for New Relic Infrastructure if bionic apt_repository: @@ -60,6 +68,10 @@ - install - install:app-requirements when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'bionic' + retries: 10 + delay: 10 + register: nr_apt_repo_bionic + until: nr_apt_repo_bionic is succeeded # For focal, use the bionic repo for now. - name: Install apt repository for New Relic Infrastructure if focal @@ -71,6 +83,12 @@ - install - install:app-requirements when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal' + retries: 10 + delay: 10 + register: result + until: result is succeeded + register: nr_apt_repo_focal + until: nr_apt_repo_focal is succeeded - name: Install newrelic related system packages for Ubuntu apt: @@ -81,6 +99,10 @@ - install - install:app-requirements when: ansible_distribution == 'Ubuntu' + retries: 10 + delay: 10 + register: nr_apt_pkg + until: nr_apt_pkg is succeeded - name: Configure the New Relic Servers yum repository yum_repository: @@ -94,6 +116,12 @@ - install - install:app-requirements when: ansible_distribution == 'Amazon' + retries: 10 + delay: 10 + register: result + until: result is succeeded + register: nr_yum_repo + until: nr_yum_repo is succeeded - name: Install newrelic related system packages for Amazon yum: @@ -105,6 +133,12 @@ - install - install:app-requirements when: ansible_distribution == 'Amazon' + retries: 10 + delay: 10 + register: result + until: result is succeeded + register: nr_yum_pkg + until: nr_yum_pkg is succeeded - name: Install newrelic display name script copy: From fdc63b75cc5b14df6bd5f84c59cbb459f2aad2fb Mon Sep 17 00:00:00 2001 From: ha-D Date: Mon, 19 Jul 2021 07:16:33 +0000 Subject: [PATCH 106/664] fix: update ES config to work with newer versions The elasticsearch config in ansible files were not being used by recent elasticsearch versions and system defaults were applied instead. Also, a few configs were deprecated or needed changing to work. --- playbooks/roles/elasticsearch/tasks/main.yml | 27 ++++- .../etc/elasticsearch/elasticsearch.yml.j2 | 111 +++++++++++++----- .../edx/etc/elasticsearch/jvm.options.j2 | 79 +++++++++++++ .../edx/etc/elasticsearch/logging.yml.j2 | 56 --------- .../templates/etc/default/elasticsearch.j2 | 66 +++++++---- .../systemd/system/elasticsearch.service.j2 | 66 +++++++++++ 6 files changed, 290 insertions(+), 115 deletions(-) create mode 100644 playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/jvm.options.j2 delete mode 100644 playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/logging.yml.j2 create mode 100644 playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 diff --git a/playbooks/roles/elasticsearch/tasks/main.yml b/playbooks/roles/elasticsearch/tasks/main.yml index cbf3e35c536..e32749f1a1f 100644 --- a/playbooks/roles/elasticsearch/tasks/main.yml +++ b/playbooks/roles/elasticsearch/tasks/main.yml @@ -88,16 +88,35 @@ template: src: edx/etc/elasticsearch/elasticsearch.yml.j2 dest: "{{ elasticsearch_cfg_dir }}/elasticsearch.yml" - mode: 0744 + mode: 0644 tags: - install - install:configuration - name: drop the elasticsearch logging config + copy: + src: /etc/elasticsearch/log4j2.properties + dest: "{{ elasticsearch_cfg_dir }}/log4j2.properties" + remote_src: yes + mode: 0644 + tags: + - install + - install:configuration + +- name: drop the elasticsearch jvm config + template: + src: edx/etc/elasticsearch/jvm.options.j2 + dest: "{{ elasticsearch_cfg_dir }}/jvm.options" + mode: 0644 + tags: + - install + - install:configuration + +- name: drop the elasticsearch systemd service config template: - src: edx/etc/elasticsearch/logging.yml.j2 - dest: "{{ elasticsearch_cfg_dir }}/logging.yml" - mode: 0744 + src: lib/systemd/system/elasticsearch.service.j2 + dest: "/lib/systemd/system/elasticsearch.service" + mode: 0644 tags: - install - install:configuration diff --git a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2 b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2 index 79366921aab..82b7d007ccb 100644 --- a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2 +++ b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/elasticsearch.yml.j2 @@ -1,45 +1,96 @@ # {{ ansible_managed }} - -# Path to directory where to store index data allocated for this node. +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +#node.name: node-1 +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): # path.data: {{ elasticsearch_data_dir }} - +# # Path to log files: # path.logs: {{ elasticsearch_log_dir }} - -# ElasticSearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. # -# Set this property to true to lock the memory: +# ----------------------------------- Memory ----------------------------------- # -bootstrap.mlockall: true - -# Disable dynamic scripting as it is insecure and we don't use it -# See: http://bouk.co/blog/elasticsearch-rce/ -# CVE: CVE-2014-3120 -script.disable_dynamic: true - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. +# Lock the memory on startup: # -# 1. Disable multicast discovery (enabled by default): +#bootstrap.memory_lock: true # -# discovery.zen.ping.multicast.enabled: false +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. # -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +#network.host: 192.168.0.1 +# +# Set a custom port for HTTP: +# +#http.port: 9200 # -# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] -{% if ELASTICSEARCH_CLUSTER_MEMBERS|length > 1 -%} - -discovery.zen.ping.unicast.hosts: ['{{ELASTICSEARCH_CLUSTER_MEMBERS|join("\',\'") }}'] - -{% endif -%} - {% if vagrant_cluster|bool %} -network: - host: {{ ansible_ssh_host }} +network.host: {{ ansible_ssh_host }} {% endif %} +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +{% if ELASTICSEARCH_CLUSTER_MEMBERS|length > 1 -%} +discovery.seed_hosts: ['{{ELASTICSEARCH_CLUSTER_MEMBERS|join("\',\'") }}'] +{% endif -%} +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, consult the gateway module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +#action.destructive_requires_name: true diff --git a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/jvm.options.j2 b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/jvm.options.j2 new file mode 100644 index 00000000000..09c07afb7a2 --- /dev/null +++ b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/jvm.options.j2 @@ -0,0 +1,79 @@ +# {{ ansible_managed }} + +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ ELASTICSEARCH_HEAP_SIZE }} +-Xmx{{ ELASTICSEARCH_HEAP_SIZE }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +8-13:-XX:+UseConcMarkSweepGC +8-13:-XX:CMSInitiatingOccupancyFraction=75 +8-13:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1 GC is only supported on JDK version 10 or later +# to use G1GC, uncomment the next two lines and update the version on the +# following three lines to your version of the JDK +# 10-13:-XX:-UseConcMarkSweepGC +# 10-13:-XX:-UseCMSInitiatingOccupancyOnly +14-:-XX:+UseG1GC +14-:-XX:G1ReservePercent=25 +14-:-XX:InitiatingHeapOccupancyPercent=30 + +## JVM temporary directory +-Djava.io.tmpdir=${ES_TMPDIR} + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=/var/lib/elasticsearch + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log + +## JDK 8 GC logging +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:/var/log/elasticsearch/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m diff --git a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/logging.yml.j2 b/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/logging.yml.j2 deleted file mode 100644 index 6978a6fdc2e..00000000000 --- a/playbooks/roles/elasticsearch/templates/edx/etc/elasticsearch/logging.yml.j2 +++ /dev/null @@ -1,56 +0,0 @@ -# you can override this using by setting a system property, for example -Des.logger.level=DEBUG -es.logger.level: INFO -rootLogger: ${es.logger.level}, console, file -logger: - # log action execution errors for easier debugging - action: DEBUG - # reduce the logging for aws, too much is logged under the default INFO - com.amazonaws: WARN - - # gateway - #gateway: DEBUG - #index.gateway: DEBUG - - # peer shard recovery - #indices.recovery: DEBUG - - # discovery - #discovery: TRACE - - index.search.slowlog: TRACE, index_search_slow_log_file - index.indexing.slowlog: TRACE, index_indexing_slow_log_file - -additivity: - index.search.slowlog: false - index.indexing.slowlog: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_indexing_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" \ No newline at end of file diff --git a/playbooks/roles/elasticsearch/templates/etc/default/elasticsearch.j2 b/playbooks/roles/elasticsearch/templates/etc/default/elasticsearch.j2 index 2556cc9ed88..f577bab3b34 100644 --- a/playbooks/roles/elasticsearch/templates/etc/default/elasticsearch.j2 +++ b/playbooks/roles/elasticsearch/templates/etc/default/elasticsearch.j2 @@ -1,38 +1,54 @@ # {{ ansible_managed }} +################################ +# Elasticsearch +################################ -# Run ElasticSearch as this user ID and group ID -#ES_USER=elasticsearch -#ES_GROUP=elasticsearch +# Elasticsearch home directory +#ES_HOME=/usr/share/elasticsearch -# Heap Size (defaults to 256m min, 1g max) -ES_HEAP_SIZE={{ ELASTICSEARCH_HEAP_SIZE }} +# Elasticsearch Java path +#JAVA_HOME= -# Heap new generation -#ES_HEAP_NEWSIZE= +# Elasticsearch configuration directory +# Note: this setting will be shared with command-line tools +ES_PATH_CONF={{ elasticsearch_cfg_dir }} -# max direct memory -#ES_DIRECT_SIZE= +# Elasticsearch PID directory +#PID_DIR=/var/run/elasticsearch -# Maximum number of open files, defaults to 65535. -#MAX_OPEN_FILES=65535 +# Additional Java OPTS +#ES_JAVA_OPTS= -# Maximum locked memory size. Set to "unlimited" if you use the -# bootstrap.mlockall option in elasticsearch.yml. You must also set -# ES_HEAP_SIZE. -#MAX_LOCKED_MEMORY=unlimited +# Configure restart on package upgrade (true, every other setting will lead to not restarting) +#RESTART_ON_UPGRADE=true -# ElasticSearch log directory -LOG_DIR={{ elasticsearch_log_dir }} +################################ +# Elasticsearch service +################################ -# ElasticSearch data directory -DATA_DIR={{ COMMON_DATA_DIR }} +# SysV init.d +# +# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process +ES_STARTUP_SLEEP_TIME=5 -# ElasticSearch work directory -#WORK_DIR=/tmp/elasticsearch +################################ +# System properties +################################ -# ElasticSearch configuration directory -CONF_DIR={{ elasticsearch_cfg_dir }} +# Specifies the maximum file descriptor number that can be opened by this process +# When using Systemd, this setting is ignored and the LimitNOFILE defined in +# /usr/lib/systemd/system/elasticsearch.service takes precedence +#MAX_OPEN_FILES=65535 -# Additional Java OPTS -#ES_JAVA_OPTS= +# The maximum number of bytes of memory that may be locked into RAM +# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option +# in elasticsearch.yml. +# When using systemd, LimitMEMLOCK must be set in a unit file such as +# /etc/systemd/system/elasticsearch.service.d/override.conf. +#MAX_LOCKED_MEMORY=unlimited + +# Maximum number of VMA (Virtual Memory Areas) a process can own +# When using Systemd, this setting is ignored and the 'vm.max_map_count' +# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf +#MAX_MAP_COUNT=262144 diff --git a/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 b/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 new file mode 100644 index 00000000000..f0539d520c8 --- /dev/null +++ b/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 @@ -0,0 +1,66 @@ +[Unit] +Description=Elasticsearch +Documentation=https://www.elastic.co +Wants=network-online.target +After=network-online.target + +[Service] +Type=notify +RuntimeDirectory=elasticsearch +PrivateTmp=true +Environment=ES_HOME=/usr/share/elasticsearch +Environment=ES_PATH_CONF=/etc/elasticsearch +Environment=PID_DIR=/var/run/elasticsearch +Environment=ES_SD_NOTIFY=true +EnvironmentFile=-/etc/default/elasticsearch + +WorkingDirectory=/usr/share/elasticsearch + +User={{ elasticsearch_user }} +Group={{ elasticsearch_group }} + +ExecStart=/usr/share/elasticsearch/bin/systemd-entrypoint -p ${PID_DIR}/elasticsearch.pid --quiet + +# StandardOutput is configured to redirect to journalctl since +# some error messages may be logged in standard output before +# elasticsearch logging system is initialized. Elasticsearch +# stores its logs in /var/log/elasticsearch and does not use +# journalctl by default. If you also want to enable journalctl +# logging, you can simply remove the "quiet" option from ExecStart. +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65535 + +# Specifies the maximum number of processes +LimitNPROC=4096 + +# Specifies the maximum size of virtual memory +LimitAS=infinity + +# Specifies the maximum file size +LimitFSIZE=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop the Java process +KillSignal=SIGTERM + +# Send the signal only to the JVM rather than its control group +KillMode=process + +# Java process is never killed +SendSIGKILL=no + +# When a JVM receives a SIGTERM signal it exits with code 143 +SuccessExitStatus=143 + +# Allow a slow startup before the systemd notifier module kicks in to extend the timeout +TimeoutStartSec=90 + +[Install] +WantedBy=multi-user.target + +# Built for packages-7.10.2 (packages) From 5116e71680d137d9d86d6bc9c7c3ea1539fc7a91 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 20 Jul 2021 13:20:59 -0400 Subject: [PATCH 107/664] Bump mysql version --- playbooks/roles/mysql/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 10a69f54b60..0d11fff4491 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -13,9 +13,9 @@ mysql_release_specific_debian_pkgs: mysql_debian_pkgs: "{{ mysql_debian_pkgs_default + mysql_release_specific_debian_pkgs[ansible_distribution_release] }}" mysql_server_pkg: "{{ 'mysql-server-5.7' if mysql_server_version_5_7 is defined and (mysql_server_version_5_7 | bool) else 'mysql-server-5.6' }}" -mysql_server_5_7_pkg: "mysql-server=5.7.34-1ubuntu18.04" -mysql_client_5_7_pkg: "mysql-client=5.7.34-1ubuntu18.04" -mysql_community_server_5_7_pkg: "mysql-server=5.7.34-1ubuntu18.04" +mysql_server_5_7_pkg: "mysql-server=5.7.35-1ubuntu18.04" +mysql_client_5_7_pkg: "mysql-client=5.7.35-1ubuntu18.04" +mysql_community_server_5_7_pkg: "mysql-server=5.7.35-1ubuntu18.04" mysql_dir: /etc/mysql From 4c458b336f91a2995906d4bb2096e89618e1856c Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 21 Jul 2021 17:02:28 -0400 Subject: [PATCH 108/664] Unpin docker-compose to fix packer build --- playbooks/roles/docker-tools/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml index 740a0871cd9..6a3ab29e00f 100644 --- a/playbooks/roles/docker-tools/tasks/main.yml +++ b/playbooks/roles/docker-tools/tasks/main.yml @@ -81,8 +81,7 @@ - name: install docker-compose pip: - # We used to pin this at 1.9.0 but may not need to anymore: https://github.com/ansible/ansible/issues/20492 - name: "docker-compose==1.10.0" + name: docker-compose tags: - install - install:system-requirements From 47c5568879b55ef624e19712015e0dfaef0091c6 Mon Sep 17 00:00:00 2001 From: ha-D Date: Mon, 19 Jul 2021 10:07:19 +0000 Subject: [PATCH 109/664] fix: increase systemd timeout for starting ES Elasticsearch often requires more time to startup on OCIM instances than the default timeout. --- playbooks/roles/elasticsearch/defaults/main.yml | 1 + .../templates/lib/systemd/system/elasticsearch.service.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/elasticsearch/defaults/main.yml b/playbooks/roles/elasticsearch/defaults/main.yml index f46548ff6fc..42e887323d8 100644 --- a/playbooks/roles/elasticsearch/defaults/main.yml +++ b/playbooks/roles/elasticsearch/defaults/main.yml @@ -13,4 +13,5 @@ elasticsearch_group: "elasticsearch" # Defaults for a single server installation. ELASTICSEARCH_CLUSTER_MEMBERS: [] ELASTICSEARCH_HEAP_SIZE: "512m" +ELASTICSEARCH_START_TIMEOUT: "600" ELASTICSEARCH_VERSION: "7.10.2" diff --git a/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 b/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 index f0539d520c8..94165ee6204 100644 --- a/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 +++ b/playbooks/roles/elasticsearch/templates/lib/systemd/system/elasticsearch.service.j2 @@ -58,7 +58,7 @@ SendSIGKILL=no SuccessExitStatus=143 # Allow a slow startup before the systemd notifier module kicks in to extend the timeout -TimeoutStartSec=90 +TimeoutStartSec={{ ELASTICSEARCH_START_TIMEOUT }} [Install] WantedBy=multi-user.target From d49a31265606cd4b84dc7327ca758482001c848a Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Sun, 25 Jul 2021 13:17:25 -0300 Subject: [PATCH 110/664] feat: Improve blockstore deployment Allow setting more Django service variables. --- playbooks/roles/blockstore/defaults/main.yml | 6 ++++-- playbooks/roles/blockstore/meta/main.yml | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/blockstore/defaults/main.yml b/playbooks/roles/blockstore/defaults/main.yml index 096b4ecc7b4..6027f3c97f0 100644 --- a/playbooks/roles/blockstore/defaults/main.yml +++ b/playbooks/roles/blockstore/defaults/main.yml @@ -27,14 +27,16 @@ BLOCKSTORE_REPOS: blockstore_gunicorn_host: '127.0.0.1' blockstore_gunicorn_port: '8250' -blockstore_gunicorn_timeout: '300' -BLOCKSTORE_GUNICORN_WORKERS: '2' +BLOCKSTORE_GUNICORN_WORKERS: 2 BLOCKSTORE_GUNICORN_EXTRA: '' BLOCKSTORE_GUNICORN_EXTRA_CONF: '' BLOCKSTORE_GUNICORN_WORKER_CLASS: 'gevent' BLOCKSTORE_GUNICORN_MAX_REQUESTS: null +# This controls both gunicorn and nginx proxy timeouts. +BLOCKSTORE_REQUEST_TIMEOUT: 300 + BLOCKSTORE_NGINX_HOSTNAME: '~^((stage|prod)-)?{{ blockstore_service_name }}.*' BLOCKSTORE_NGINX_PORT: '1{{ blockstore_gunicorn_port }}' BLOCKSTORE_SSL_NGINX_PORT: '4{{ blockstore_gunicorn_port }}' diff --git a/playbooks/roles/blockstore/meta/main.yml b/playbooks/roles/blockstore/meta/main.yml index d5b56b920c3..b2058f15c50 100644 --- a/playbooks/roles/blockstore/meta/main.yml +++ b/playbooks/roles/blockstore/meta/main.yml @@ -14,20 +14,22 @@ dependencies: edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ blockstore_service_name }}' edx_django_service_repos: '{{ BLOCKSTORE_REPOS }}' edx_django_service_version: '{{ BLOCKSTORE_VERSION }}' + edx_django_service_gunicorn_timeout: '{{ BLOCKSTORE_REQUEST_TIMEOUT }}' edx_django_service_gunicorn_port: '{{ blockstore_gunicorn_port }}' edx_django_service_gunicorn_extra: '{{ BLOCKSTORE_GUNICORN_EXTRA }}' + edx_django_service_gunicorn_workers: '{{ BLOCKSTORE_GUNICORN_WORKERS }}' edx_django_service_gunicorn_worker_class: '{{ BLOCKSTORE_GUNICORN_WORKER_CLASS }}' edx_django_service_gunicorn_max_requests: '{{ BLOCKSTORE_GUNICORN_MAX_REQUESTS }}' edx_django_service_hostname: '{{ BLOCKSTORE_NGINX_HOSTNAME }}' edx_django_service_max_webserver_upload: 10 edx_django_service_nginx_port: '{{ BLOCKSTORE_NGINX_PORT }}' + edx_django_service_nginx_read_timeout: '{{ BLOCKSTORE_REQUEST_TIMEOUT }}' edx_django_service_ssl_nginx_port: '{{ BLOCKSTORE_SSL_NGINX_PORT }}' edx_django_service_default_db_name: '{{ BLOCKSTORE_DEFAULT_DB_NAME }}' edx_django_service_db_user: '{{ BLOCKSTORE_DATABASE_USER }}' edx_django_service_db_password: '{{ BLOCKSTORE_DATABASE_PASSWORD }}' edx_django_service_default_db_host: '{{ BLOCKSTORE_DATABASE_HOST }}' edx_django_service_default_db_atomic_requests: true - edx_django_service_default_db_conn_max_age: '{{ BLOCKSTORE_DATABASE_CONN_MAX_AGE }}' edx_django_service_django_settings_module: '{{ BLOCKSTORE_DJANGO_SETTINGS_MODULE }}' edx_django_service_secret_key: '{{ BLOCKSTORE_SECRET_KEY }}' edx_django_service_automated_users: '{{ BLOCKSTORE_AUTOMATED_USERS }}' From 022d36bd00c219caec9973bb6c7cf2d01cfa0199 Mon Sep 17 00:00:00 2001 From: Saleem Latif Date: Mon, 26 Jul 2021 16:54:11 +0500 Subject: [PATCH 111/664] Fixed argument order as suggested by Mike. --- playbooks/roles/tableau/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/tableau/tasks/main.yml b/playbooks/roles/tableau/tasks/main.yml index 9e6d49e0779..99b08f3ec46 100644 --- a/playbooks/roles/tableau/tasks/main.yml +++ b/playbooks/roles/tableau/tasks/main.yml @@ -122,7 +122,7 @@ - install:code - name: Run automated installer to actually setup the tableau server - shell: "sudo {{ tableau_installer_dir }}/automated-installer -s {{ tableau_installer_dir }}/secrets -f {{ tableau_installer_dir }}/config.json -r {{ tableau_installer_dir }}/registration.json --accepteula {{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }} -k '{{ TABLEAU_LICENSE_KEY }}'" + shell: "sudo {{ tableau_installer_dir }}/automated-installer -k '{{ TABLEAU_LICENSE_KEY }}' -s {{ tableau_installer_dir }}/secrets -f {{ tableau_installer_dir }}/config.json -r {{ tableau_installer_dir }}/registration.json --accepteula {{ tableau_installer_dir }}/{{ tableau_server_deb_pkg_name }}" become_user: "{{ tableau_user }}" tags: - install From c2abac340d40b1dcbbc3e9faa1a5affe6a4ae062 Mon Sep 17 00:00:00 2001 From: Guruprasad Lakshmi Narayanan Date: Mon, 19 Jul 2021 20:01:39 +0530 Subject: [PATCH 112/664] feat: allow writing extra requirements to a requirements file This is useful for apps like ecommerce, which use tox to run the migrations and the dependencies for the tox environments are expected to be defined in a requirements file. --- CHANGELOG.md | 8 ++++++++ playbooks/roles/ecommerce/defaults/main.yml | 1 + playbooks/roles/ecommerce/meta/main.yml | 1 + playbooks/roles/edx_django_service/tasks/main.yml | 11 +++++++++++ 4 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cb5b9e13e9..3da4e5bf6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-07-19 + - Role: edx_django_service + - Allows writing extra requirements to an 'extra.txt' requirements file in the service's requirements directory. + - Role: ecommerce + - Adds an optional flag to write the extra requirements to an 'extra.txt' file since many of the app's setup commands + use tox and that creates its own environments separate from the default ecommerce virtualenv environment where the + `ECOMMERCE_EXTRA_REQUIREMENTS` requirements are installed. + - 2021-06-17 - Role credentials - Installs extra python packages specified in `CREDENTIALS_EXTRA_REQUIREMENTS` (defaults to `[]`). diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index d7974c79a6e..2680a854653 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -32,6 +32,7 @@ ECOMMERCE_REPOS: # version: 1.0.1 # - name: git+https://git.myproject.org/MyProject#egg=MyProject ECOMMERCE_EXTRA_REQUIREMENTS: [] +ECOMMERCE_ADD_EXTRA_REQUIREMENTS_TO_REQUIREMENTS_FILE: false # depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC # and a key being provided via NEWRELIC_LICENSE_KEY diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index 8b4fceed69b..f10b421dbd1 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -22,6 +22,7 @@ dependencies: edx_django_service_debian_pkgs_extra: '{{ ecommerce_debian_pkgs + ecommerce_release_specific_debian_pkgs[ansible_distribution_release] }}' edx_django_service_django_settings_module: '{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}' edx_django_service_extra_requirements: '{{ ECOMMERCE_EXTRA_REQUIREMENTS }}' + edx_django_service_add_extra_requirements_to_requirements_file: '{{ ECOMMERCE_ADD_EXTRA_REQUIREMENTS_TO_REQUIREMENTS_FILE }}' edx_django_service_repos: '{{ ECOMMERCE_REPOS }}' edx_django_service_environment_extra: '{{ ecommerce_environment }}' edx_django_service_gunicorn_extra: '{{ ECOMMERCE_GUNICORN_EXTRA }}' diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 5055f762694..500c50b5bf7 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -193,6 +193,17 @@ - install - install:app-requirements +- name: add extra requirements to extra.txt + lineinfile: + path: "{{ edx_django_service_code_dir }}/requirements/extra.txt" + line: "{{ item.name }}" + become_user: "{{ edx_django_service_user }}" + with_items: "{{ edx_django_service_extra_requirements }}" + when: edx_django_service_add_extra_requirements_to_requirements_file is defined and edx_django_service_add_extra_requirements_to_requirements_file + tags: + - install + - install:app-requirements + - name: Check for existing make_migrate container command: "docker ps -aq --filter name='{{ edx_django_service_name }}.make_migrate'" register: edx_django_service_make_migrate_container From a5fe532a40caf8a1f367bda19ef24f0140a1def8 Mon Sep 17 00:00:00 2001 From: Alfred Chavez Urday Date: Mon, 26 Jul 2021 17:23:54 -0500 Subject: [PATCH 113/664] [FAL-2000] Remove jdk role from openedx-native playbook and elasticsearch role dependencies (#6483) * remove depedency to oraclejdk in elasticsearch role * remove jdk from opencraft_native --- playbooks/openedx_native.yml | 1 - playbooks/roles/elasticsearch/meta/main.yml | 1 - playbooks/roles/elasticsearch/tasks/main.yml | 24 +++++++++----------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 8b82a6e8032..f2d249e5ade 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -114,7 +114,6 @@ - role: demo when: DEMO_ROLE_ENABLED - oauth_client_setup - - oraclejdk - role: elasticsearch when: EDXAPP_ENABLE_ELASTIC_SEARCH - forum diff --git a/playbooks/roles/elasticsearch/meta/main.yml b/playbooks/roles/elasticsearch/meta/main.yml index 1d056c68ab5..2083f0e1251 100644 --- a/playbooks/roles/elasticsearch/meta/main.yml +++ b/playbooks/roles/elasticsearch/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - common - - oraclejdk diff --git a/playbooks/roles/elasticsearch/tasks/main.yml b/playbooks/roles/elasticsearch/tasks/main.yml index cbf3e35c536..5daa1aa776c 100644 --- a/playbooks/roles/elasticsearch/tasks/main.yml +++ b/playbooks/roles/elasticsearch/tasks/main.yml @@ -5,7 +5,6 @@ # Dependencies: # # * common -# * oraclejdk # # Example play: # @@ -25,11 +24,10 @@ # - hosts: tag_role_elasticsearch:&tag_environment_stage # roles: # - common -# - oraclejdk # - elasticsearch # -- name: Install Elasticsearch repo key +- name: Install Elasticsearch repo key apt_key: url: "{{ elasticsearch_apt_key_url }}" state: present @@ -48,9 +46,9 @@ - name: install elasticsearch apt: pkg: "{{ elasticsearch_package_name }}={{ ELASTICSEARCH_VERSION }}" - state: present + state: present install_recommends: yes - force: yes + force: yes update_cache: yes tags: - install @@ -78,7 +76,7 @@ - name: update elasticsearch defaults template: - src: etc/default/elasticsearch.j2 + src: etc/default/elasticsearch.j2 dest: /etc/default/elasticsearch tags: - install @@ -86,7 +84,7 @@ - name: drop the elasticsearch config template: - src: edx/etc/elasticsearch/elasticsearch.yml.j2 + src: edx/etc/elasticsearch/elasticsearch.yml.j2 dest: "{{ elasticsearch_cfg_dir }}/elasticsearch.yml" mode: 0744 tags: @@ -95,7 +93,7 @@ - name: drop the elasticsearch logging config template: - src: edx/etc/elasticsearch/logging.yml.j2 + src: edx/etc/elasticsearch/logging.yml.j2 dest: "{{ elasticsearch_cfg_dir }}/logging.yml" mode: 0744 tags: @@ -103,18 +101,18 @@ - install:configuration - name: Ensure elasticsearch is enabled and started - service: + service: name: elasticsearch - state: started + state: started enabled: yes tags: - manage - manage:start - name: Restart elastic when there has been an upgrade - service: - name: elasticsearch - state: restarted + service: + name: elasticsearch + state: restarted enabled: yes when: elasticsearch_reinstall.changed tags: From 4cb43b4bfd869579605c12fdeb64ba12b063f266 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 27 Jul 2021 13:39:41 +0500 Subject: [PATCH 114/664] use wildcard to select available subversion of mysql --- playbooks/roles/mysql/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 0d11fff4491..46466395a2d 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -13,9 +13,9 @@ mysql_release_specific_debian_pkgs: mysql_debian_pkgs: "{{ mysql_debian_pkgs_default + mysql_release_specific_debian_pkgs[ansible_distribution_release] }}" mysql_server_pkg: "{{ 'mysql-server-5.7' if mysql_server_version_5_7 is defined and (mysql_server_version_5_7 | bool) else 'mysql-server-5.6' }}" -mysql_server_5_7_pkg: "mysql-server=5.7.35-1ubuntu18.04" -mysql_client_5_7_pkg: "mysql-client=5.7.35-1ubuntu18.04" -mysql_community_server_5_7_pkg: "mysql-server=5.7.35-1ubuntu18.04" +mysql_server_5_7_pkg: "mysql-server=5.7.*" +mysql_client_5_7_pkg: "mysql-client=5.7.*" +mysql_community_server_5_7_pkg: "mysql-server=5.7.*" mysql_dir: /etc/mysql From 0ac931e4cf6cb53988e5985001e8f54fc0b61a57 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Wed, 28 Jul 2021 16:35:34 +0500 Subject: [PATCH 115/664] build: install py38 requirements in edxapp-sandbox --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index f89eb3f8fb6..ffe6c0f9d31 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1690,7 +1690,7 @@ base_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/base.txt" django_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/django.txt" openstack_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/openstack.txt" -sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/{% if edxapp_sandbox_python_version == 'python2.7' %}base.txt{% else %}py35.txt{% endif %}" +sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/py38.txt" # The Python requirements files in the order they should be installed. This order should # match the order of PYTHON_REQ_FILES in edx-platform/pavelib/prereqs.py. From 0e3d8ab82b5180a01e22b3ffaae09f4fa870b82b Mon Sep 17 00:00:00 2001 From: SaadYousaf Date: Thu, 29 Jul 2021 07:19:00 +0500 Subject: [PATCH 116/664] feat: update devstack configurations for forum to use elasticsearch 7.10 --- playbooks/roles/forum/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 0562ef1583c..7319a3598a8 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -85,7 +85,7 @@ devstack_forum_env: RACK_ENV: "development" SINATRA_ENV: "development" SEARCH_SERVER: "http://edx.devstack.elasticsearch:9200/" - SEARCH_SERVER_ES7: "http://edx.devstack.elasticsearch7:9200/" + SEARCH_SERVER_ES7: "http://edx.devstack.elasticsearch710:9200/" MONGOHQ_URL: "mongodb://cs_comments_service:password@edx.devstack.mongo:27017/cs_comments_service" MONGOID_AUTH_MECH: "{{ FORUM_MONGO_AUTH_MECH }}" From 8b028d7d2ecc33786ab3ffcff24f7b4a02a7ae0c Mon Sep 17 00:00:00 2001 From: Jazib Humayun Date: Fri, 30 Jul 2021 20:26:28 +0500 Subject: [PATCH 117/664] fix: docker play installation order up then DE new DE jenkins new play tries to add jenkins user in docker group but Docker is installing later in the order of installation sequence that's why docker group will be available for jenkins play to add jenkins user in docker group, So re-order and moving docker play up in the installation sequence --- playbooks/jenkins_data_engineering_new.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/jenkins_data_engineering_new.yml b/playbooks/jenkins_data_engineering_new.yml index 7a4728686e4..0b3cdf0d5c1 100644 --- a/playbooks/jenkins_data_engineering_new.yml +++ b/playbooks/jenkins_data_engineering_new.yml @@ -28,9 +28,9 @@ roles: - role: aws when: COMMON_ENABLE_AWS_ROLE + - docker-tools - jenkins_data_engineering_new - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE tags: - newreliconly - - docker-tools From b2bf69e0dad429792693f4365827e498811b3991 Mon Sep 17 00:00:00 2001 From: Samuel Walladge Date: Mon, 2 Aug 2021 14:02:45 +0930 Subject: [PATCH 118/664] feat: Add ansible tasks to init mongodb indexes (#6462) Indexes can improve performance, so we add this as an option during provisioning. --- CHANGELOG.md | 6 ++++++ playbooks/roles/edxapp/defaults/main.yml | 2 ++ playbooks/roles/edxapp/tasks/deploy.yml | 11 +++++++++++ playbooks/roles/forum/defaults/main.yml | 2 ++ playbooks/roles/forum/tasks/deploy.yml | 12 ++++++++++++ 5 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da4e5bf6b5..2cd9986a751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-07-29 + - Role edxapp + - Add `EDXAPP_ENABLE_MONGODB_INDEXES` configuration variable to optionally set up indexes on edxapp mongodb. + - Role forum + - Add `FORUM_ENABLE_MONGODB_INDEXES` configuration variable to optionally set up indexes on forum mongodb. + - 2021-07-19 - Role: edx_django_service - Allows writing extra requirements to an 'extra.txt' requirements file in the service's requirements directory. diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index f89eb3f8fb6..a2872ece431 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1819,3 +1819,5 @@ edxapp_staticfiles_storage_overrides: !!null # SiteConfiguration instance. EDXAPP_SITE_CONFIGURATION: {} + +EDXAPP_ENABLE_MONGODB_INDEXES: false diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index e8ec2949961..c1446b165c6 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -529,3 +529,14 @@ tags: - manage - manage:db + +- name: ensure indexes on contentstore and modulestore dbs + shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} ensure_indexes" + args: + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ common_web_user }}" + when: EDXAPP_ENABLE_MONGODB_INDEXES + run_once: yes + tags: + - manage + - manage:db diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 7319a3598a8..cc9a1909d36 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -54,6 +54,8 @@ FORUM_RESTART_DELAY: 60 # Set to rebuild the forum ElasticSearch index from the database. FORUM_REBUILD_INDEX: false +FORUM_ENABLE_MONGODB_INDEXES: false + forum_base_env: &forum_base_env RBENV_ROOT: "{{ forum_rbenv_root }}" GEM_HOME: "{{ forum_gem_root }}" diff --git a/playbooks/roles/forum/tasks/deploy.yml b/playbooks/roles/forum/tasks/deploy.yml index 29796463c9f..c3cab2c68db 100644 --- a/playbooks/roles/forum/tasks/deploy.yml +++ b/playbooks/roles/forum/tasks/deploy.yml @@ -108,6 +108,18 @@ tags: - manage +- name: ensure indexes on forum mongo db + command: "{{ forum_code_dir }}/bin/rake db:init" + args: + chdir: "{{ forum_code_dir }}" + become_user: "{{ forum_user }}" + environment: "{{ forum_base_env }}" + when: FORUM_ENABLE_MONGODB_INDEXES + run_once: yes + tags: + - manage + - manage:db + - include: test.yml tags: - deploy From 351476400a8d9917116f8c619de60dac59793c60 Mon Sep 17 00:00:00 2001 From: connorhaugh Date: Thu, 5 Aug 2021 09:55:45 -0400 Subject: [PATCH 119/664] chore: upgrade devstack edx-platform es to 7.10 --- docker/build/edxapp/lms.yml | 2 +- docker/build/edxapp/studio.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 4674488ec20..397cce5db7f 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -241,7 +241,7 @@ EDXNOTES_PUBLIC_API: http://localhost:18120/api/v1 EDX_API_KEY: PUT_YOUR_API_KEY_HERE EDX_PLATFORM_REVISION: master ELASTIC_SEARCH_CONFIG: -- host: edx.devstack.elasticsearch +- host: edx.devstack.elasticsearch710 port: 9200 use_ssl: false EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index f8136ee5364..1058cc847f0 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -217,7 +217,7 @@ ECOMMERCE_PUBLIC_URL_ROOT: http://localhost:8002 EDXMKTG_USER_INFO_COOKIE_NAME: edx-user-info EDX_PLATFORM_REVISION: master ELASTIC_SEARCH_CONFIG: -- host: edx.devstack.elasticsearch +- host: edx.devstack.elasticsearch710 port: 9200 use_ssl: false EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend From 5a534d6bd0e126361f651cf8572dd5ac7f04b313 Mon Sep 17 00:00:00 2001 From: sarina Date: Mon, 9 Aug 2021 10:34:29 -0400 Subject: [PATCH 120/664] build: add edx-community-bot workflow --- .../workflows/pr-automerge-open-release.yml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/pr-automerge-open-release.yml diff --git a/.github/workflows/pr-automerge-open-release.yml b/.github/workflows/pr-automerge-open-release.yml new file mode 100644 index 00000000000..8bf09a27b8d --- /dev/null +++ b/.github/workflows/pr-automerge-open-release.yml @@ -0,0 +1,67 @@ +# For non-draft changes to Named Release branches: +# - Check if the user belongs to a maintainers team. +# - If so, approve the pull request. +# - Tag community-engineering (for now) and the maintainers team. +# - Merge the PR when the author comments `@edx-community-bot merge`. +# Required organization secrets +# - CC_GITHUB_TOKEN=... +# - CC_TEAM_CHAMPIONS=org/team-name +# - CC_TEAM_CONTRIBUTORS_ORG=org +# - CC_TEAM_CONTRIBUTORS_TEAM=team-name +--- +name: automerge BTR open-release PRs +on: + issue_comment: + branches: + - open-release/* + types: + - created + - edited + pull_request_target: + branches: + - open-release/* + types: + - opened + - edited + - ready_for_review +jobs: + automerge: + if: ${{ (github.event.issue.pull_request && !github.event.issue.pull_request.draft) || (github.event.pull_request && !github.event.pull_request.draft) }} + runs-on: ubuntu-latest + steps: + - name: lookup teams + id: teams + uses: tspascoal/get-user-teams-membership@v1 + with: + username: "${{ github.actor }}" + organization: ${{ secrets.CC_TEAM_CONTRIBUTORS_ORG }} + team: ${{ secrets.CC_TEAM_CONTRIBUTORS_TEAM }} + GITHUB_TOKEN: "${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" + - name: approve PR + if: ${{ steps.teams.outputs.isTeamMember == 'true' && (github.event.action == 'opened' || github.event.action == 'ready_for_review') }} + uses: andrewmusgrave/automatic-pull-request-review@0.0.5 + with: + repo-token: ${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + event: APPROVE + body: | + :+1: + + When you're ready to merge, add a comment that says + > @edx-community-bot merge + + and we'll handle the rest! + CC: @${{ secrets.CC_TEAM_CHAMPIONS }} @${{ secrets.CC_TEAM_CONTRIBUTORS_ORG }}/${{ secrets.CC_TEAM_CONTRIBUTORS_TEAM }} + - name: label PR as auto-mergeable + if: ${{ steps.teams.outputs.isTeamMember == 'true' && contains(github.event.comment.body, '@edx-community-bot merge') }} + uses: andymckay/labeler@978f846c4ca6299fd136f465b42c5e87aca28cac + with: + add-labels: 'automerge' + repo-token: ${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + - name: automerge + uses: "pascalgn/automerge-action@v0.13.1" + env: + GITHUB_TOKEN: "${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" + MERGE_COMMIT_MESSAGE: | + merge(#{pullRequest.number}): {pullRequest.title} + + {pullRequest.body} From b388d2aec1ed6c2103c617ec9abe51cd6afa2896 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 10 Aug 2021 14:06:12 +0500 Subject: [PATCH 121/664] fix: rate exceeded and throttling exception in check_rds_config script --- util/check_rds_configs/check_rds_configs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/check_rds_configs/check_rds_configs.py b/util/check_rds_configs/check_rds_configs.py index 7ff1a3315cb..4b87cbb7dd5 100644 --- a/util/check_rds_configs/check_rds_configs.py +++ b/util/check_rds_configs/check_rds_configs.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import print_function import boto3 +from botocore.config import Config import click tags_key_list = ["deployment", "environment", "cluster"] @@ -168,5 +169,5 @@ def cli(db_engine, ignore): if __name__ == '__main__': - rds = boto3.client('rds') + rds = boto3.client('rds', config=Config(connect_timeout=5, read_timeout=60, retries={'max_attempts': 15})) cli() From 421cc7b9d259cc7e363ca905b0e6b92d8bc6e5be Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Tue, 10 Aug 2021 10:58:45 -0400 Subject: [PATCH 122/664] feat: Move prospectus to install Node directly. Since prospectus does not use Python for any of its code, installing Python introduced more points of failure compared to simply installing Node directly onto the machines. This PR removes all installation of Python and nodeenv from prospectus builds. --- playbooks/roles/prospectus/defaults/main.yml | 6 +-- playbooks/roles/prospectus/tasks/main.yml | 48 ++++++-------------- 2 files changed, 16 insertions(+), 38 deletions(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 2ae1486bc3e..44b99710e71 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,16 +37,12 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '12.21.0' +PROSPECTUS_NODE_VERSION: '12' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' -prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' -prospectus_nodeenv_dir: '{{ prospectus_home }}/nodeenvs/{{ prospectus_service_name }}' -prospectus_nodeenv_bin: '{{prospectus_nodeenv_dir}}/bin' prospectus_app_dir: "{{ COMMON_APP_DIR }}/prospectus" prospectus_user: 'root' prospectus_env_vars: - PATH: "{{ prospectus_nodeenv_bin }}:{{ prospectus_venv_dir }}/bin:{{ ansible_env.PATH }}" NODE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" ACTIVE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" USE_COURSE_URL_SLUGS: "{{ PROSPECTUS_USE_COURSE_URL_SLUGS }}" diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index a0b1499bbc2..95c405d66e0 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -59,43 +59,27 @@ register: prospectus_checkout when: PROSPECTUS_GIT_IDENTITY == "none" -- name: add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - when: prospectus_use_python3 - -- name: install python3.8 - apt: - pkg: - - python3.8-dev - - python3.8-distutils - when: prospectus_use_python3 +- name: Install the gpg key for nodejs LTS + apt_key: + url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" + state: present tags: - install - install:system-requirements -- name: Add prospectus configuration file - template: - src: ".env.environment.j2" - dest: "{{ prospectus_code_dir }}/config/.env.keys" - mode: "0644" - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - -- name: Install nodeenv - apt: - name: nodeenv - become_user: "{{ prospectus_user }}" - environment: "{{ prospectus_env_vars }}" +- name: Install the nodejs LTS repos + apt_repository: + repo: "deb https://deb.nodesource.com/node_{{ PROSPECTUS_NODE_VERSION }}.x {{ ansible_distribution_release }} main" + state: present + update_cache: yes tags: - install - install:system-requirements -# Install node -- name: Create nodeenv - shell: "nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" - become_user: "{{ prospectus_user }}" - environment: "{{ prospectus_env_vars }}" +- name: Install node + apt: + name: nodejs + state: present tags: - install - install:system-requirements @@ -132,9 +116,8 @@ - install - install:app-requirements -# Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: install node dependencies - shell: "{{ prospectus_nodeenv_bin }}/npm install --unsafe-perm=true --allow-root" + shell: "npm install --unsafe-perm=true --allow-root" args: chdir: "{{ prospectus_code_dir }}" environment: "{{ prospectus_env_vars }}" @@ -143,9 +126,8 @@ - install - install:app-requirements -# Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: run prospectus build - shell: "{{ prospectus_nodeenv_bin }}/npm run build" + shell: "npm run build" args: chdir: "{{ prospectus_code_dir }}" environment: "{{ prospectus_env_vars }}" From 739d12d6a256ae67942c36399312bde385d177b0 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 10 Aug 2021 20:23:24 +0500 Subject: [PATCH 123/664] fix: add retry logic to add-apt-repository tasks --- playbooks/roles/common/tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 368d4cb0c79..4233ca5a22b 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -59,11 +59,21 @@ - name: Add git apt repository apt_repository: repo: "{{ common_git_ppa }}" + update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 when: ansible_distribution in common_debian_variants - name: Add ppa for watchman package apt_repository: repo: "ppa:linuxuprising/apps" + update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 when: > ansible_distribution in common_debian_variants and ({{ devstack | default(False) }} or {{ edx_django_service_is_devstack | default(False) }}) @@ -100,6 +110,11 @@ - name: add deadsnakes repository apt_repository: repo: "ppa:deadsnakes/ppa" + update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' tags: - install From 1e79e4ead5bfcd9c305cbfa1fe48d7610f84b7d5 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Tue, 10 Aug 2021 12:54:41 -0400 Subject: [PATCH 124/664] Revert "feat: Move prospectus to install Node directly." --- playbooks/roles/prospectus/defaults/main.yml | 6 ++- playbooks/roles/prospectus/tasks/main.yml | 48 ++++++++++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 44b99710e71..2ae1486bc3e 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,12 +37,16 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '12' +PROSPECTUS_NODE_VERSION: '12.21.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' +prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' +prospectus_nodeenv_dir: '{{ prospectus_home }}/nodeenvs/{{ prospectus_service_name }}' +prospectus_nodeenv_bin: '{{prospectus_nodeenv_dir}}/bin' prospectus_app_dir: "{{ COMMON_APP_DIR }}/prospectus" prospectus_user: 'root' prospectus_env_vars: + PATH: "{{ prospectus_nodeenv_bin }}:{{ prospectus_venv_dir }}/bin:{{ ansible_env.PATH }}" NODE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" ACTIVE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" USE_COURSE_URL_SLUGS: "{{ PROSPECTUS_USE_COURSE_URL_SLUGS }}" diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 95c405d66e0..a0b1499bbc2 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -59,27 +59,43 @@ register: prospectus_checkout when: PROSPECTUS_GIT_IDENTITY == "none" -- name: Install the gpg key for nodejs LTS - apt_key: - url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" - state: present +- name: add deadsnakes repo + apt_repository: + repo: ppa:deadsnakes/ppa + when: prospectus_use_python3 + +- name: install python3.8 + apt: + pkg: + - python3.8-dev + - python3.8-distutils + when: prospectus_use_python3 tags: - install - install:system-requirements -- name: Install the nodejs LTS repos - apt_repository: - repo: "deb https://deb.nodesource.com/node_{{ PROSPECTUS_NODE_VERSION }}.x {{ ansible_distribution_release }} main" - state: present - update_cache: yes +- name: Add prospectus configuration file + template: + src: ".env.environment.j2" + dest: "{{ prospectus_code_dir }}/config/.env.keys" + mode: "0644" + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + +- name: Install nodeenv + apt: + name: nodeenv + become_user: "{{ prospectus_user }}" + environment: "{{ prospectus_env_vars }}" tags: - install - install:system-requirements -- name: Install node - apt: - name: nodejs - state: present +# Install node +- name: Create nodeenv + shell: "nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" + become_user: "{{ prospectus_user }}" + environment: "{{ prospectus_env_vars }}" tags: - install - install:system-requirements @@ -116,8 +132,9 @@ - install - install:app-requirements +# Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: install node dependencies - shell: "npm install --unsafe-perm=true --allow-root" + shell: "{{ prospectus_nodeenv_bin }}/npm install --unsafe-perm=true --allow-root" args: chdir: "{{ prospectus_code_dir }}" environment: "{{ prospectus_env_vars }}" @@ -126,8 +143,9 @@ - install - install:app-requirements +# Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: run prospectus build - shell: "npm run build" + shell: "{{ prospectus_nodeenv_bin }}/npm run build" args: chdir: "{{ prospectus_code_dir }}" environment: "{{ prospectus_env_vars }}" From 9ea5647a1e1231e7bc5d08a14d1b8d90095eacbf Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 11 Aug 2021 11:27:48 -0400 Subject: [PATCH 125/664] fix: master's sandboxes site configuration should be enabled otherwise cache_programs won't pull make programs data accessible to the LMS --- playbooks/masters_sandbox.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/masters_sandbox.yml b/playbooks/masters_sandbox.yml index b7eb99e523b..e658196934a 100644 --- a/playbooks/masters_sandbox.yml +++ b/playbooks/masters_sandbox.yml @@ -39,6 +39,7 @@ shell: > . {{ edxapp_env_path }} && {{ edxapp_venv_dir }}/bin/python manage.py lms create_or_update_site_configuration {{dns_name}}.sandbox.edx.org --configuration '{"COURSE_CATALOG_API_URL":"https://discovery-{{dns_name}}.sandbox.edx.org/api/v1","email_from_address":"edX "}' + --enabled args: chdir: "{{ edxapp_code_dir }}" From 4e9c41d822123992dbad58ed4ce788f3425e41cc Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 11 Aug 2021 11:37:48 -0400 Subject: [PATCH 126/664] fix: default value for celery transport needed for when always_eager see https://github.com/edx/registrar/commit/bc128871b7eadbcfa0c12fc6d75946e2e91aba02 --- playbooks/roles/registrar/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/registrar/defaults/main.yml b/playbooks/roles/registrar/defaults/main.yml index cd5add560a6..35674942262 100644 --- a/playbooks/roles/registrar/defaults/main.yml +++ b/playbooks/roles/registrar/defaults/main.yml @@ -29,7 +29,7 @@ registrar_celery_default_queue: 'registrar.default' REGISTRAR_USE_PYTHON38: True REGISTRAR_CELERY_ALWAYS_EAGER: false -REGISTRAR_CELERY_BROKER_TRANSPORT: '' +REGISTRAR_CELERY_BROKER_TRANSPORT: 'redis' REGISTRAR_CELERY_BROKER_USER: '' REGISTRAR_CELERY_BROKER_PASSWORD: '' REGISTRAR_CELERY_BROKER_HOSTNAME: '' From ffc305a0504d6b6d99c1847ddacf3ea3201f6646 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 12 Aug 2021 11:37:50 +0000 Subject: [PATCH 127/664] install libmysqlclient-dev on gha runner and update version --- docker/build/github-actions-runner/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/build/github-actions-runner/Dockerfile b/docker/build/github-actions-runner/Dockerfile index 1f600f40950..61e6ef5e9cb 100644 --- a/docker/build/github-actions-runner/Dockerfile +++ b/docker/build/github-actions-runner/Dockerfile @@ -1,13 +1,14 @@ FROM ubuntu:focal # Set the github runner version -ARG RUNNER_VERSION="2.277.1" +ARG RUNNER_VERSION="2.279.0" ENV GITHUB_ORGANIZATION="" ENV GITHUB_ACCESS_TOKEN="" # Update and install necessary packages -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl iputils-ping jq software-properties-common build-essential libssl-dev libffi-dev python3 python3-venv python3-dev yamllint +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl iputils-ping jq software-properties-common build-essential libssl-dev \ + libffi-dev python3 python3-venv python3-dev libmysqlclient-dev yamllint # Add a github action runner user RUN useradd -m actions-runner From f394e97fdaa90152dc8d56aa16f965e3f8b1168b Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 12 Aug 2021 10:58:31 -0400 Subject: [PATCH 128/664] fix: retry accessing ppa for when launchpad is being finnicky JIRA:DOS-2550 --- playbooks/roles/common/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 4233ca5a22b..953be279ae8 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -127,6 +127,10 @@ install_recommends: yes state: present update_cache: yes + register: add_pkgs + until: add_pkgs|success + retries: 10 + delay: 5 when: ansible_distribution in common_debian_variants - name: Install role-independent packages useful for devstack. @@ -148,6 +152,10 @@ install_recommends: yes state: present update_cache: yes + register: add_pkgs + until: add_pkgs|success + retries: 10 + delay: 5 when: > ansible_distribution in common_debian_variants and ansible_distribution_release in old_python_ppa_releases and From 158b7e982d5f2f1c62ee0fc2bbfb08939e85dfa5 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Thu, 12 Aug 2021 11:01:21 -0400 Subject: [PATCH 129/664] feat: Add retry logic to apt tasks in prospectus role. --- playbooks/roles/nginx/tasks/main.yml | 5 +++++ playbooks/roles/prospectus/tasks/main.yml | 17 +++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 87282328b7c..35609892df3 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -62,6 +62,11 @@ apt_repository: repo: "{{ NGINX_APT_REPO }}" state: present + update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 notify: restart nginx tags: - install diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index a0b1499bbc2..868c5658fd9 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -62,6 +62,11 @@ - name: add deadsnakes repo apt_repository: repo: ppa:deadsnakes/ppa + update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 when: prospectus_use_python3 - name: install python3.8 @@ -69,6 +74,10 @@ pkg: - python3.8-dev - python3.8-distutils + register: add_pkgs + until: add_pkgs|success + retries: 10 + delay: 5 when: prospectus_use_python3 tags: - install @@ -87,6 +96,10 @@ name: nodeenv become_user: "{{ prospectus_user }}" environment: "{{ prospectus_env_vars }}" + register: add_pkg + until: add_pkg|success + retries: 10 + delay: 5 tags: - install - install:system-requirements @@ -157,6 +170,10 @@ - name: Install pngquant apt: name: "pngquant" + register: add_pkg + until: add_pkg|success + retries: 10 + delay: 5 tags: - install - install:system-requirements From 9a831a61539e015926c02d468ce1bd211579e126 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 13 Aug 2021 11:20:52 +0500 Subject: [PATCH 130/664] update: tools gp role for ubuntu focal --- playbooks/roles/ad_hoc_reporting/defaults/main.yml | 10 ++++++---- playbooks/roles/ad_hoc_reporting/tasks/main.yml | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/ad_hoc_reporting/defaults/main.yml b/playbooks/roles/ad_hoc_reporting/defaults/main.yml index 40ebc05e263..447157a0376 100644 --- a/playbooks/roles/ad_hoc_reporting/defaults/main.yml +++ b/playbooks/roles/ad_hoc_reporting/defaults/main.yml @@ -31,7 +31,7 @@ ad_hoc_reporting_role_name: ad_hoc_reporting ad_hoc_reporting_debian_pkgs: # for running ansible mysql module - - mysql-client-core-5.7 + - mysql-client-core-8.0 # includes mysqldump and others - mysql-client - libmysqlclient-dev @@ -39,12 +39,14 @@ ad_hoc_reporting_debian_pkgs: ad_hoc_reporting_pip_pkgs: # for running ansible mysql - - mysql-python + - PyMySQL MONGODB_APT_KEY: "7F0CEB10" MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com" -MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" -MONGO_VERSION: 3.2.12 +MONGO_VERSION_MAJOR_MINOR: "4.0" +MONGO_VERSION_PATCH: "26" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" # AD_HOC_REPORTING_REPLICA_DB_HOSTS: # - db_host: "{{ EDXAPP_MYSQL_REPLICA_HOST }}" diff --git a/playbooks/roles/ad_hoc_reporting/tasks/main.yml b/playbooks/roles/ad_hoc_reporting/tasks/main.yml index 5630423ac73..2b325ee4335 100644 --- a/playbooks/roles/ad_hoc_reporting/tasks/main.yml +++ b/playbooks/roles/ad_hoc_reporting/tasks/main.yml @@ -29,8 +29,7 @@ - name: add the mongodb signing key apt_key: - id: "{{ MONGODB_APT_KEY }}" - keyserver: "{{ MONGODB_APT_KEYSERVER }}" + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 tags: From c7bc4c25b844d9709ef2dc18a151d6e91f0d2c2a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 13 Aug 2021 11:50:19 +0500 Subject: [PATCH 131/664] fix: retry APT task if it fails --- playbooks/roles/aws/tasks/main.yml | 4 +++ playbooks/roles/common/tasks/main.yml | 8 +++--- .../roles/edx_django_service/tasks/main.yml | 15 +++++++---- playbooks/roles/edx_service/tasks/main.yml | 19 ++++++++++---- playbooks/roles/edxapp/tasks/main.yml | 22 +++++++++++++--- playbooks/roles/edxapp_common/tasks/main.yml | 4 +++ playbooks/roles/hermes/tasks/main.yml | 5 ++++ playbooks/roles/nginx/tasks/main.yml | 9 +++++++ playbooks/roles/nltk/tasks/main.yml | 6 ++++- playbooks/roles/prospectus/tasks/main.yml | 25 +++++++------------ 10 files changed, 82 insertions(+), 35 deletions(-) diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml index 821f509cc22..cd8f8f53c10 100644 --- a/playbooks/roles/aws/tasks/main.yml +++ b/playbooks/roles/aws/tasks/main.yml @@ -52,6 +52,10 @@ name: "{{ aws_debian_pkgs }}" state: present update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: ansible_distribution in common_debian_variants - name: Copy the Python requirements file diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 953be279ae8..cbb76003cae 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -127,8 +127,8 @@ install_recommends: yes state: present update_cache: yes - register: add_pkgs - until: add_pkgs|success + register: install_pkgs + until: install_pkgs|success retries: 10 delay: 5 when: ansible_distribution in common_debian_variants @@ -152,8 +152,8 @@ install_recommends: yes state: present update_cache: yes - register: add_pkgs - until: add_pkgs|success + register: install_pkgs + until: install_pkgs|success retries: 10 delay: 5 when: > diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 500c50b5bf7..31feb5b867b 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -61,16 +61,16 @@ - install - install:configuration -- name: add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - when: edx_django_service_use_python38 and not edx_django_service_enable_experimental_docker_shim - - name: install python3.8 apt: pkg: - python3.8-dev - python3.8-distutils + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: edx_django_service_use_python38 and not edx_django_service_enable_experimental_docker_shim tags: - install @@ -79,6 +79,11 @@ - name: install python3 apt: name: "{{ item }}" + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: edx_django_service_use_python3 and not edx_django_service_enable_experimental_docker_shim with_items: - python3-pip diff --git a/playbooks/roles/edx_service/tasks/main.yml b/playbooks/roles/edx_service/tasks/main.yml index 252547f2428..1b6991f3354 100644 --- a/playbooks/roles/edx_service/tasks/main.yml +++ b/playbooks/roles/edx_service/tasks/main.yml @@ -115,16 +115,16 @@ - install:configuration - install:app-configuration -- name: Add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - when: edx_service_use_python38 - - name: Install Python 3.8 apt: pkg: - python3.8-dev - python3.8-distutils + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: edx_service_use_python38 tags: - install @@ -133,6 +133,11 @@ - name: install python3 apt: name: "{{ item }}" + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: edx_service_use_python3 and not edx_service_use_python38 with_items: - python3-pip @@ -193,6 +198,10 @@ state: present update_cache: true cache_valid_time: 3600 + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 when: ansible_distribution in common_debian_variants tags: - install diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index 7fe73056e79..bc902250319 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -109,6 +109,10 @@ name: "{{ item }}" state: present update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 with_flattened: - "{{ edxapp_debian_pkgs_default }}" - "{{ edxapp_debian_pkgs_extra }}" @@ -129,6 +133,10 @@ repo: "deb https://deb.nodesource.com/node_{{ EDXAPP_NODE_VERSION }}.x {{ ansible_distribution_release }} main" state: present update_cache: yes + register: add_repo + until: add_repo|success + retries: 10 + delay: 5 tags: - install - install:base @@ -137,6 +145,11 @@ apt: name: nodejs state: present + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 tags: - install - install:base @@ -152,15 +165,16 @@ - install - install:base -- name: add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - - name: install python3.8 apt: pkg: - python3.8-dev - python3.8-distutils + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 tags: - install - install:system-requirements diff --git a/playbooks/roles/edxapp_common/tasks/main.yml b/playbooks/roles/edxapp_common/tasks/main.yml index b76c45c6960..b8284df5ab6 100644 --- a/playbooks/roles/edxapp_common/tasks/main.yml +++ b/playbooks/roles/edxapp_common/tasks/main.yml @@ -4,6 +4,10 @@ name: "{{ edxapp_common_debian_pkgs }}" state: present update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 tags: - install - install:base diff --git a/playbooks/roles/hermes/tasks/main.yml b/playbooks/roles/hermes/tasks/main.yml index c0c61a63f41..e2e4148b23e 100644 --- a/playbooks/roles/hermes/tasks/main.yml +++ b/playbooks/roles/hermes/tasks/main.yml @@ -33,6 +33,11 @@ - name: install python3.6 apt: name: "{{ item }}" + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 with_items: - python3.6 - python3-pip diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 35609892df3..bde5eb34ed9 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -44,6 +44,11 @@ apt: name: "{{ nginx_debian_pkgs }}" state: present + update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 notify: restart nginx tags: - install @@ -91,6 +96,10 @@ name: nginx state: latest update_cache: yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 notify: restart nginx tags: - install diff --git a/playbooks/roles/nltk/tasks/main.yml b/playbooks/roles/nltk/tasks/main.yml index ced54c4f538..025257edea0 100644 --- a/playbooks/roles/nltk/tasks/main.yml +++ b/playbooks/roles/nltk/tasks/main.yml @@ -1,7 +1,11 @@ --- - name: Install unzip - apt: pkg=unzip state=present + apt: pkg=unzip state=present update_cache=yes + register: install_pkgs + until: install_pkgs|success + retries: 10 + delay: 5 - name: create the nltk data directory and subdirectories file: path={{ NLTK_DATA_DIR }}/{{ item.path|dirname }} state=directory diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 868c5658fd9..c65618664f9 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -59,23 +59,14 @@ register: prospectus_checkout when: PROSPECTUS_GIT_IDENTITY == "none" -- name: add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - update_cache: yes - register: add_repo - until: add_repo|success - retries: 10 - delay: 5 - when: prospectus_use_python3 - - name: install python3.8 apt: pkg: - python3.8-dev - python3.8-distutils - register: add_pkgs - until: add_pkgs|success + update_cache: yes + register: install_pkgs + until: install_pkgs|success retries: 10 delay: 5 when: prospectus_use_python3 @@ -94,10 +85,11 @@ - name: Install nodeenv apt: name: nodeenv + update_cache: yes become_user: "{{ prospectus_user }}" environment: "{{ prospectus_env_vars }}" - register: add_pkg - until: add_pkg|success + register: install_pkgs + until: install_pkgs|success retries: 10 delay: 5 tags: @@ -170,8 +162,9 @@ - name: Install pngquant apt: name: "pngquant" - register: add_pkg - until: add_pkg|success + update_cache: yes + register: install_pkgs + until: install_pkgs|success retries: 10 delay: 5 tags: From 65312ef6a616855cc0b6199fa07834b2a5075368 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 17 Aug 2021 11:16:14 +0500 Subject: [PATCH 132/664] fix:nginx site template path neo4j --- playbooks/neo4j.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/neo4j.yml b/playbooks/neo4j.yml index e9de8545ae1..986db32e6a4 100644 --- a/playbooks/neo4j.yml +++ b/playbooks/neo4j.yml @@ -6,7 +6,7 @@ CLUSTER_NAME: 'coursegraph' roles: - role: nginx - nginx_template_dir: "../roles/neo4j/templates/edx/app/nginx/sites-available" + nginx_template_dir: "../../roles/neo4j/templates/edx/app/nginx/sites-available" nginx_sites: - coursegraph nginx_default_sites: From adcca1016fd2e5d30ab52bad937ee851967013aa Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 17 Aug 2021 16:04:27 -0400 Subject: [PATCH 133/664] fix: Mongo script broken by deprecation warning Mongo 4.2 prints a deprecation message to stdout which screws up our mongo replica scripts. This fixes the setting that's been renamed to get rid of the deprecation warning. Mongo has changesd rs.slaveOk() to rs.secondaryOk() --- playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2 b/playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2 index bf6fe87c701..dd9f47755ad 100644 --- a/playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2 +++ b/playbooks/roles/ad_hoc_reporting/templates/etc/mongorc.js.j2 @@ -1,5 +1,5 @@ // we only ever connect to secondaries, avoid people needing to remember to type this -rs.slaveOk(); +rs.secondaryOk(); // This uses the DB name rather than the replica set, which I think is more useful var prompt = function() { From ecae0fc26d63a40aa90fa956dc612e9ad383a807 Mon Sep 17 00:00:00 2001 From: Christie Rice <8483753+crice100@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:25:52 -0400 Subject: [PATCH 134/664] fix: Remove config for edx-certificates (#6501) Remove configuration for the edx-certificates repository as the repo is no longer used. MICROBA-1362 DEPR-160 --- CHANGELOG.md | 3 + docker/build/nginx/Dockerfile | 1 - playbooks/certs.yml | 18 --- playbooks/legacy_certificates.yml | 39 ------ playbooks/openedx_native.yml | 1 - playbooks/roles/certs/defaults/main.yml | 98 -------------- .../certs/files/example-key-ownertrust.txt | 1 - .../roles/certs/files/example-private-key.txt | 57 -------- .../roles/certs/files/example-public-key.txt | 30 ----- playbooks/roles/certs/meta/main.yml | 3 - playbooks/roles/certs/tasks/deploy.yml | 126 ------------------ playbooks/roles/certs/tasks/main.yml | 111 --------------- playbooks/roles/certs/tasks/tag_ec2.yml | 12 -- playbooks/roles/certs/templates/boto.j2 | 0 .../roles/certs/templates/certs.auth.json.j2 | 1 - playbooks/roles/certs/templates/certs.conf.j2 | 9 -- .../roles/certs/templates/certs.env.json.j2 | 1 - .../roles/edx_ansible/templates/update.j2 | 1 - playbooks/roles/nginx/defaults/main.yml | 2 - .../edx/app/nginx/sites-available/certs.j2 | 11 -- playbooks/sample_vars/server_vars.yml | 6 - util/install/native.sh | 1 - util/jenkins/ansible-provision.sh | 3 - vagrant/base/analyticstack/Vagrantfile | 1 - vagrant/release/analyticstack/Vagrantfile | 1 - 25 files changed, 3 insertions(+), 534 deletions(-) delete mode 100644 playbooks/certs.yml delete mode 100644 playbooks/legacy_certificates.yml delete mode 100644 playbooks/roles/certs/defaults/main.yml delete mode 100644 playbooks/roles/certs/files/example-key-ownertrust.txt delete mode 100644 playbooks/roles/certs/files/example-private-key.txt delete mode 100644 playbooks/roles/certs/files/example-public-key.txt delete mode 100644 playbooks/roles/certs/meta/main.yml delete mode 100644 playbooks/roles/certs/tasks/deploy.yml delete mode 100644 playbooks/roles/certs/tasks/main.yml delete mode 100644 playbooks/roles/certs/tasks/tag_ec2.yml delete mode 100644 playbooks/roles/certs/templates/boto.j2 delete mode 100644 playbooks/roles/certs/templates/certs.auth.json.j2 delete mode 100644 playbooks/roles/certs/templates/certs.conf.j2 delete mode 100644 playbooks/roles/certs/templates/certs.env.json.j2 delete mode 100644 playbooks/roles/nginx/templates/edx/app/nginx/sites-available/certs.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cd9986a751..0aa3c18e9d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-09-19 + - Remove configuration for edx-certificates, as that repo and service are no longer used. + - 2021-07-29 - Role edxapp - Add `EDXAPP_ENABLE_MONGODB_INDEXES` configuration variable to optionally set up indexes on edxapp mongodb. diff --git a/docker/build/nginx/Dockerfile b/docker/build/nginx/Dockerfile index 6ac27432e25..5120184c4a5 100644 --- a/docker/build/nginx/Dockerfile +++ b/docker/build/nginx/Dockerfile @@ -10,7 +10,6 @@ RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook nginx.yml -c loc -i '127.0.0.1,' \ -e@roles/edxapp/defaults/main.yml \ -e@roles/xqueue/defaults/main.yml \ - -e@roles/certs/defaults/main.yml \ -e@roles/forum/defaults/main.yml RUN echo "\ndaemon off;" >> /etc/nginx/nginx.conf diff --git a/playbooks/certs.yml b/playbooks/certs.yml deleted file mode 100644 index 5d2b2672eb1..00000000000 --- a/playbooks/certs.yml +++ /dev/null @@ -1,18 +0,0 @@ -- name: Deploy certs - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - certs - - role: datadog - when: COMMON_ENABLE_DATADOG - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER - - role: datadog-uninstall - when: not COMMON_ENABLE_DATADOG - diff --git a/playbooks/legacy_certificates.yml b/playbooks/legacy_certificates.yml deleted file mode 100644 index 110abfcd8f9..00000000000 --- a/playbooks/legacy_certificates.yml +++ /dev/null @@ -1,39 +0,0 @@ -# ansible-playbook -i ec2.py --limit="tag_cluster_certificates:&tag_environment_loadtest" legacy_certificates.yml -- name: Deploy legacy_certs - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - repo_url: "git@github.com:edx/certificates.git" - repo_path: "/opt/wwc/certificates" - CERTIFICATES_VERSION: "master" - git_ssh_script: "/opt/wwc/git.sh" - serial: "{{ serial_count }}" - tasks: - - name: stop certs service - service: name="certificates" state="stopped" - - name: checkout code - git: - repo: "{{ repo_url }}" - dest: "{{ repo_path }}" - version: "{{ CERTIFICATES_VERSION }}" - accept_hostkey: yes - environment: - GIT_SSH: "{{ git_ssh_script }}" - - name: install requirements - pip: - requirements: "{{ repo_path }}/requirements.txt" - virtualenv: "/opt/edx" - state: present - # Need to do this because the www-data user is not properly setup - # and can't run ssh. - - name: change owner to www-data - file: - path: "{{ repo_path }}" - owner: "www-data" - group: "www-data" - recurse: yes - state: "directory" - - name: start certs service - service: name="certificates" state="started" diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index f2d249e5ade..6bf4a1f5b7c 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -121,7 +121,6 @@ when: SANDBOX_ENABLE_DISCOVERY - role: xqueue update_users: True - - certs - edx_ansible - role: datadog when: COMMON_ENABLE_DATADOG diff --git a/playbooks/roles/certs/defaults/main.yml b/playbooks/roles/certs/defaults/main.yml deleted file mode 100644 index 11c1cbbbe13..00000000000 --- a/playbooks/roles/certs/defaults/main.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/edx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for role certs -# - -CERTS_QUEUE_URL: "http://localhost:18040" -CERTS_BUCKET: "" -# basic auth credentials for connecting -# to the xqueue server -CERTS_XQUEUE_AUTH_USER: "{{ COMMON_HTPASSWD_USER }}" -CERTS_XQUEUE_AUTH_PASS: "{{ COMMON_HTPASSWD_PASS }}" -# credentials for connecting to the xqueue server -CERTS_QUEUE_USER: "lms" -CERTS_QUEUE_PASS: "{{ COMMON_XQUEUE_LMS_PASSWORD }}" -# AWS credentials for certificate upload -CERTS_AWS_KEY: "" -CERTS_AWS_ID: "" -# GPG key ID, defaults to the dummy key -CERTS_KEY_ID: "FEF8D954" -# Contents of the identity for a private -# repo. Leave set to "none" if using the public -# certificate repo -CERTS_GIT_IDENTITY: "none" -# Path to public and private gpg key for signing -# the edX certificate. Default is a dummy key -CERTS_LOCAL_PRIVATE_KEY: "example-private-key.txt" - -# This defaults to the public certificates repo which is -# used for open-edx -CERTS_REPO: "https://github.com/edx/edx-certificates" -CERTS_NGINX_PORT: 18090 -CERTS_WEB_ROOT: "{{ certs_data_dir }}/www-data" -CERTS_URL: "http://localhost:{{ CERTS_NGINX_PORT }}" -CERTS_DOWNLOAD_URL: "http://localhost:{{ CERTS_NGINX_PORT }}" -CERTS_VERIFY_URL: "http://localhost:{{ CERTS_NGINX_PORT }}" -# Set to false if using s3 or if you don't want certificates -# copied to the web root -CERTS_COPY_TO_WEB_ROOT: true -CERTS_S3_UPLOAD: false - -# Can be set to a different repo for private -# templates, fonts, etc. -CERTS_TEMPLATE_DATA_DIR: 'template_data' -# this is the trust export, output of -# gpg --export-ownertrust -CERTS_OWNER_TRUST: "A9F9EAD11A0A6E7E5A037BDC044089B6FEF8D954:6:\n" - -# This is how long in seconds the cert agent will sleep before polling the queue again. -CERTS_QUEUE_POLL_FREQUENCY: 5 - -########## Internal role vars below - -certs_user: certs -certs_app_dir: "{{ COMMON_APP_DIR }}/certs" -certs_data_dir: "{{ COMMON_DATA_DIR }}/certs" -certs_code_dir: "{{ certs_app_dir }}/certificates" -certs_venvs_dir: "{{ certs_app_dir }}/venvs" -certs_venv_dir: "{{ certs_venvs_dir }}/certs" -certs_venv_bin: "{{ certs_venv_dir }}/bin" -certs_git_identity: "{{ certs_app_dir }}/certs-git-identity" -certs_requirements_file: "{{ certs_code_dir }}/requirements/base.txt" -CERTS_VERSION: 'master' -certs_gpg_dir: "{{ certs_app_dir }}/gnupg" -certs_env_config: - # CERTS_DATA is legacy, not used - CERT_DATA: {} - QUEUE_NAME: "certificates" - QUEUE_URL: "{{ CERTS_QUEUE_URL }}" - QUEUE_POLL_FREQUENCY: "{{ CERTS_QUEUE_POLL_FREQUENCY }}" - CERT_BUCKET: "{{ CERTS_BUCKET }}" - # gnupg signing key - CERT_KEY_ID: "{{ CERTS_KEY_ID }}" - LOGGING_ENV: "" - CERT_GPG_DIR: "{{ certs_gpg_dir }}" - CERT_URL: "{{ CERTS_URL }}" - CERT_DOWNLOAD_URL: "{{ CERTS_DOWNLOAD_URL }}" - CERT_WEB_ROOT: "{{ CERTS_WEB_ROOT }}" - COPY_TO_WEB_ROOT: "{{ CERTS_COPY_TO_WEB_ROOT }}" - S3_UPLOAD: "{{ CERTS_S3_UPLOAD }}" - CERT_VERIFY_URL: "{{ CERTS_VERIFY_URL }}" - TEMPLATE_DATA_DIR: "{{ CERTS_TEMPLATE_DATA_DIR }}" - -certs_auth_config: - QUEUE_USER: "{{ CERTS_QUEUE_USER }}" - QUEUE_PASS: "{{ CERTS_QUEUE_PASS }}" - QUEUE_AUTH_USER: "{{ CERTS_XQUEUE_AUTH_USER }}" - QUEUE_AUTH_PASS: "{{ CERTS_XQUEUE_AUTH_PASS }}" - CERT_KEY_ID: "{{ CERTS_KEY_ID }}" - CERT_AWS_ID: "{{ CERTS_AWS_ID }}" - CERT_AWS_KEY: "{{ CERTS_AWS_KEY }}" diff --git a/playbooks/roles/certs/files/example-key-ownertrust.txt b/playbooks/roles/certs/files/example-key-ownertrust.txt deleted file mode 100644 index 56439fa6820..00000000000 --- a/playbooks/roles/certs/files/example-key-ownertrust.txt +++ /dev/null @@ -1 +0,0 @@ -A9F9EAD11A0A6E7E5A037BDC044089B6FEF8D954:6: diff --git a/playbooks/roles/certs/files/example-private-key.txt b/playbooks/roles/certs/files/example-private-key.txt deleted file mode 100644 index 3ab8accbc3b..00000000000 --- a/playbooks/roles/certs/files/example-private-key.txt +++ /dev/null @@ -1,57 +0,0 @@ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.11 (GNU/Linux) - -lQOYBFJwVOkBCAC4heT6+P1sGgITAB5C+hKNr4RACS47K1nxgIiEqiFMIycluDmM -4kdqFInzDK8GHF2W5KijZmYf7LrWIg4+PmnyYAB7cO+eJUDfTE7n7bjGQL3LohJN -FTlRsXKOKGWoBqlytE3D16lQIIp0JkqB9sHO3Y9yOgEsSy3cMWKtT8U6qx40xV+e -t0FYmqL7pBE7OFfvCIe7+kthsTqFys/jkRNFvbSo5fjA1m9ubjEJqqfnhuvLaL5O -YHGe1nKQRLi45gmZ1JYvxfZrWUO2BeulNY/mvAFQnRNRRiWfM3Ic4Ya9Wv62wS3p -dYY4HEtDQDyKpOkJ2R31+1FhZYIKJTYR89jxABEBAAEAB/wOApyQMbeMLa1ao/eo -PjSKbXktI4VPGMuLeqbi68f7b+/Y/VPhToz9kPGocp4XaK/ydQoY3f2DDwZgm9VZ -BIQm0wM2XCzVZR631aNoGLSe2OuQOo4JLENd4ItCH+8YAul6vBXreMRyQQZCK2Yc -2A9/FXN+yMiuBEdHILjNT/E5swNm0J85YlXpIW6Jm3aR6OjzfFS/j+7AEDSL5MZX -JotfGjYXuC1MOw5YJZKWkQBz+5IaVceOd9s8TlZFq/eYrN5sqAWh06CBY+Zye3fg -/WiWFUdTgpG81lbAXGxHrjQ5f22saOzkbv0FjdEfx1M9Wcj6OAIRXI7k8EkZJia1 -IYEBBADYRvRE2zyR5M72MfCX1XUOpx/9OZCrVsYoKqp1BORXjt58Szs7UFdeAXE+ -bPzbpcjENiVYVjoeQKCNTU78gzjD+NzkfTdsF7rrvXObo6NpChTCOdQfg56Ll+3y -3nUDKIcFXsYP1NIC2SL0APcpUtDLPIWb0XRnlvBQlakmnyb7bQQA2mnn15LVRK0J -1wYZiSwrRIcE7X+zy6t8iERr+E0jIyQQV4vaOYItCDTP8fzNiiX10Nkt5imRqML3 -NBPs0jInjmYxMmzvjVxyUDv4rGGbiXXeh+W1v/mweMH2pbiItjhyeSVt4U2l6GKI -Ob/K+khx1ftfOTktLTZVMPg6NzPRfRUD/jyLL92V7eoHshreqFIUcBmUdQnMVyz2 -NBhci7RHGn3J84TTMCXBJL8MLUu9PKxfFcZjw1Wet8avX5oPjgl3OiQpjjx69sO5 -S/UWpGaEOrz87j4VRGPb6zmegkZs44sQEfwhDJk6O1eQ5dYzniRC46nzMpETFfIF -U8m8bJrrus4HPEy0HmV4YW1wbGUga2V5IDx0ZXN0QGV4YW1wbGUuY29tPokBOAQT -AQIAIgUCUnBU6QIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQBECJtv74 -2VSWKQf/f0PWmbGxdiBIK5gf2pmaK0aDDM09v7vedysIn/URnj26BMN/YEyDYnZS -BN+iuU6VartvEYlNeYiRAnaG/6gl7DJh2l2X/iuDn0xKT0GjqDpjh7n6964OKAz7 -RHWADXqsr5BWms1EPFtDVnAJfN2A3cxTeA5vUUl41WvCJQa7L8Bw7SezkS0yn8Rn -u4icNKyew7TrFofIydws6LTM1DhHpCB32z6b7HHt85OOzpuUm07HP83S59lxBp6x -x3NH9AH/WPeXiS5QRh1jP6qzUAHoHQpsV2XonmC4JXl+ZFxNyZeJ000ldDFfEHrO -RLg3d5GkZ1pDVDn3HlZ+SKqYilRXCJ0DmARScFTpAQgA1KGTRGcqwla5/VOuHwxw -ABpLYdpsetYoOORjJQvHakG8QBchxsJVniBijD09gFmHYpdSJaeHnvqkeHGO1fJa -E4QxS4AYt/HVoi86RhBLD/Gr0/DWC/0XUV5613PSmWkYCCTgWLaxT9MpPjtGVd4v -L6Iv/d8Go/Wrq55zCl82PTA7ao4PxSSxlforfZOZqsJ/pzjCRkF6Z7co+LO24KSl -Lt4iN2vwJ2VhvOrMFuV91WQeEJWdTX+yx035eU/MFu9u243CE0UGNzWHjYLpgBxl -Pg0W5GFRZM/LYkXAfHAM4/Ic2ex2LQ0RLiH4i0FbzoSwjvz586v2Sagc5nsYMoGu -gQARAQABAAf/W6W+23taJ0SJSuLACJrsRWcP+b/TBQj8cjUidKvEioyFztwJj2lg -zNSplUeqFAHCxGBzpE42uvPOYymTBq08XPAb7S5ruREP4yVXCS7po5gnVyUVpToz -zDscWHQQIFZ3aL84QZSRDVZ3Dt8unEE1dmMCK3rvGkl/8mtLq3tJXgp7/wdsK4G0 -3AuJVQ918XlozNTayGfdCPhWicE7fv5peUlWRWlSuSNmTrHiAbysd2xwXnMq/OdQ -Q+z7ogQHhUvQQ+31msGlcCJQqqWr250/HBrTATrRJNIVvvzCgpw3/6r99MNwlSWV -ZhDotwf745fdzZiwdgJ04nhEj9QmKynKXwQA29p7DmMWMZg61qU481YNWgc5RMjL -ADUS2iC5nr/Y5HuAsGWj9ZkDvRXKSyexkZ+OXi6EonGCGjCNONPaB8JWRO7LssEc -VG+lPp4mwASE38cjfFy7DdEGpxn3eZPDsNwv7vnWhNyGSh8FXKoYXyZiJ6F3zvkU -aWwfaTtxVplfn88EAPeXHLkwl+D8zkk8ILYnsJKEKjcqUwiQ6L7JMEhd+GVo5xR+ -WUDdhnmEkH/QZZt8zTpYL3Hl3JsqQYidq0uzy39qg+cVvD9yJkHP4KMAqe9QcRYR -eQvpopMYt5va7pyaebZbpxfP9M7Y2/5VT59GBO6uHy4CMR1uM4Z50QA0kZCvA/oD -D/9qEaWzwqLtXjN1iRxOv7ioor8ExvA/8HY8xtCsCLuFuo9P44xtYzSCzLdoOYCE -4Lrn7DeE2hXEoq/2VEWoRS4+kU1vUBIJEAxfHk6HozA0apFPqm8ODH44s68VRTce -pGwORxsFhMHw1/m5A1RBZF8UP7VXFxluYuwx6S5NyjSfiQEfBBgBAgAJBQJScFTp -AhsMAAoJEARAibb++NlUojkIAKHwS1VSeW6fgWv7H2qaTjdMeNG7vXUYKUE7KMpQ -UmvdHobMfbO9SEgihwG+WdgPy96RlYx5PuVfeWkPVdVsbrU9BuR+9qdYyGGH4FvP -qAaruT3dFLRFvDj/ta94gDFGCH1LrtGI/t78wjjIEd8QOGIj+8Uo1Z6HKExSsNuG -+8usut6je50a2CsAyoZtrPmybZdkU6eOuM5ZSGDpgfTlFNpeK3sf7CTnYA5NTLPC -wWbyCxUb7EUrch+StmJWsIzS4mClMd6nB4480FwwhGbdFejSF20z64c6hbxuwgfS -nyXklWktEX0d5T7wdAi+UOvNsdoigzUMWpBoo07VOlzjMFU= -=iNqX ------END PGP PRIVATE KEY BLOCK----- diff --git a/playbooks/roles/certs/files/example-public-key.txt b/playbooks/roles/certs/files/example-public-key.txt deleted file mode 100644 index e1c620bcfe6..00000000000 --- a/playbooks/roles/certs/files/example-public-key.txt +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.11 (GNU/Linux) - -mQENBFJwVOkBCAC4heT6+P1sGgITAB5C+hKNr4RACS47K1nxgIiEqiFMIycluDmM -4kdqFInzDK8GHF2W5KijZmYf7LrWIg4+PmnyYAB7cO+eJUDfTE7n7bjGQL3LohJN -FTlRsXKOKGWoBqlytE3D16lQIIp0JkqB9sHO3Y9yOgEsSy3cMWKtT8U6qx40xV+e -t0FYmqL7pBE7OFfvCIe7+kthsTqFys/jkRNFvbSo5fjA1m9ubjEJqqfnhuvLaL5O -YHGe1nKQRLi45gmZ1JYvxfZrWUO2BeulNY/mvAFQnRNRRiWfM3Ic4Ya9Wv62wS3p -dYY4HEtDQDyKpOkJ2R31+1FhZYIKJTYR89jxABEBAAG0HmV4YW1wbGUga2V5IDx0 -ZXN0QGV4YW1wbGUuY29tPokBOAQTAQIAIgUCUnBU6QIbAwYLCQgHAwIGFQgCCQoL -BBYCAwECHgECF4AACgkQBECJtv742VSWKQf/f0PWmbGxdiBIK5gf2pmaK0aDDM09 -v7vedysIn/URnj26BMN/YEyDYnZSBN+iuU6VartvEYlNeYiRAnaG/6gl7DJh2l2X -/iuDn0xKT0GjqDpjh7n6964OKAz7RHWADXqsr5BWms1EPFtDVnAJfN2A3cxTeA5v -UUl41WvCJQa7L8Bw7SezkS0yn8Rnu4icNKyew7TrFofIydws6LTM1DhHpCB32z6b -7HHt85OOzpuUm07HP83S59lxBp6xx3NH9AH/WPeXiS5QRh1jP6qzUAHoHQpsV2Xo -nmC4JXl+ZFxNyZeJ000ldDFfEHrORLg3d5GkZ1pDVDn3HlZ+SKqYilRXCLkBDQRS -cFTpAQgA1KGTRGcqwla5/VOuHwxwABpLYdpsetYoOORjJQvHakG8QBchxsJVniBi -jD09gFmHYpdSJaeHnvqkeHGO1fJaE4QxS4AYt/HVoi86RhBLD/Gr0/DWC/0XUV56 -13PSmWkYCCTgWLaxT9MpPjtGVd4vL6Iv/d8Go/Wrq55zCl82PTA7ao4PxSSxlfor -fZOZqsJ/pzjCRkF6Z7co+LO24KSlLt4iN2vwJ2VhvOrMFuV91WQeEJWdTX+yx035 -eU/MFu9u243CE0UGNzWHjYLpgBxlPg0W5GFRZM/LYkXAfHAM4/Ic2ex2LQ0RLiH4 -i0FbzoSwjvz586v2Sagc5nsYMoGugQARAQABiQEfBBgBAgAJBQJScFTpAhsMAAoJ -EARAibb++NlUojkIAKHwS1VSeW6fgWv7H2qaTjdMeNG7vXUYKUE7KMpQUmvdHobM -fbO9SEgihwG+WdgPy96RlYx5PuVfeWkPVdVsbrU9BuR+9qdYyGGH4FvPqAaruT3d -FLRFvDj/ta94gDFGCH1LrtGI/t78wjjIEd8QOGIj+8Uo1Z6HKExSsNuG+8usut6j -e50a2CsAyoZtrPmybZdkU6eOuM5ZSGDpgfTlFNpeK3sf7CTnYA5NTLPCwWbyCxUb -7EUrch+StmJWsIzS4mClMd6nB4480FwwhGbdFejSF20z64c6hbxuwgfSnyXklWkt -EX0d5T7wdAi+UOvNsdoigzUMWpBoo07VOlzjMFU= -=WP59 ------END PGP PUBLIC KEY BLOCK----- diff --git a/playbooks/roles/certs/meta/main.yml b/playbooks/roles/certs/meta/main.yml deleted file mode 100644 index 837c9ae8171..00000000000 --- a/playbooks/roles/certs/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -dependencies: - - common - - supervisor \ No newline at end of file diff --git a/playbooks/roles/certs/tasks/deploy.yml b/playbooks/roles/certs/tasks/deploy.yml deleted file mode 100644 index 9b2b068ec56..00000000000 --- a/playbooks/roles/certs/tasks/deploy.yml +++ /dev/null @@ -1,126 +0,0 @@ ---- -- name: Create certificate application config and auth file - template: - src: "{{ item.src }}" - dest: "{{ certs_app_dir }}/{{ item.dest }}" - owner: "{{ certs_user }}" - group: "{{ common_web_user }}" - mode: "0640" - with_items: - - { src: 'certs.env.json.j2', dest: 'env.json' } - - { src: 'certs.auth.json.j2', dest: 'auth.json' } - -- name: Copy the boto file - template: - src: "boto.j2" - dest: "{{ certs_app_dir }}/.boto" - owner: "{{ certs_user }}" - group: "{{ common_web_user }}" - mode: 0644 - -- name: Writing supervisor script for certificates - template: - src: certs.conf.j2 - dest: "{{ supervisor_available_dir }}/certs.conf" - owner: "{{ supervisor_user }}" - mode: "0644" - -- name: Enable supervisor script for certificates - file: - src: "{{ supervisor_available_dir }}/certs.conf" - dest: "{{ supervisor_cfg_dir }}/certs.conf" - owner: "{{ supervisor_user }}" - state: link - force: yes - mode: "0644" - when: not disable_edx_services - -# This key is only needed if you are pulling down a private -# certificates repo -- name: Install read-only ssh key for the certs repo - copy: - content: "{{ CERTS_GIT_IDENTITY }}" - dest: "{{ certs_git_identity }}" - force: yes - owner: "{{ certs_user }}" - mode: "0600" - when: CERTS_GIT_IDENTITY != "none" - -- name: "Checkout certificates repo into {{ certs_code_dir }}" - git: - dest: "{{ certs_code_dir }}" - repo: "{{ CERTS_REPO }}" - version: "{{ CERTS_VERSION }}" - accept_hostkey: yes - key_file: "{{ certs_git_identity }}" - become_user: "{{ certs_user }}" - register: certs_checkout - when: CERTS_GIT_IDENTITY != "none" - -- name: Checkout certificates repo into {{ certs_code_dir }} - git: - dest: "{{ certs_code_dir }}" - repo: "{{ CERTS_REPO }}" - version: "{{ CERTS_VERSION }}" - accept_hostkey: yes - become_user: "{{ certs_user }}" - register: certs_checkout - when: CERTS_GIT_IDENTITY == "none" - -- name: Remove read-only ssh key for the certs repo - file: - path: "{{ certs_git_identity }}" - state: absent - when: CERTS_GIT_IDENTITY != "none" - -- name: Install python requirements - pip: - requirements: "{{ certs_requirements_file }}" - virtualenv: "{{ certs_venv_dir }}" - state: present - extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" - virtualenv_python: python3.8 - become_user: "{{ certs_user }}" - - # call supervisorctl update. this reloads - # the supervisorctl config and restarts - # the services if any of the configurations - # have changed. - # -- name: Update supervisor configuration - shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update" - register: supervisor_update - become_user: "{{ supervisor_service_user }}" - changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != "" - when: not disable_edx_services - -- name: Ensure certs has started - supervisorctl: - name: certs - supervisorctl_path: "{{ supervisor_ctl }}" - config: "{{ supervisor_cfg }}" - state: started - become_user: "{{ supervisor_service_user }}" - when: not disable_edx_services - -- name: Create a symlink for venv python - file: - src: "{{ certs_venv_bin }}/{{ item }}" - dest: "{{ COMMON_BIN_DIR }}/{{ item }}.certs" - state: link - with_items: - - python - - pip - -- include: tag_ec2.yml - when: COMMON_TAG_EC2_INSTANCE - tags: - - deploy - -- name: Restart certs - supervisorctl: - name: certs - supervisorctl_path: "{{ supervisor_ctl }}" - config: "{{ supervisor_cfg }}" - state: restarted - when: not disable_edx_services diff --git a/playbooks/roles/certs/tasks/main.yml b/playbooks/roles/certs/tasks/main.yml deleted file mode 100644 index 44b2f797f49..00000000000 --- a/playbooks/roles/certs/tasks/main.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/edx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT -# -# -# -# Tasks for role certs -# -# Overview: -# -# Installs the edX certificate server. -# -# The certificates repo is currently *not* public -# due to sensitive information in it, it may be made -# public in the future. -# -# Dependencies: -# - common -# - supervisor -# -# -# Example play: -# -# - roles: -# - common -# - supervisor -# - certs -# -- name: Create application user - user: - name: "{{ certs_user }}" - home: "{{ certs_app_dir }}" - createhome: no - shell: /bin/false - -- name: Create certs app dirs - file: - path: "{{ item }}" - state: directory - owner: "{{ certs_user }}" - group: "{{ common_web_group }}" - with_items: - - "{{ certs_app_dir }}" - # needed for the ansible 1.5 git module - - "{{ certs_app_dir }}/.ssh" - - "{{ certs_venvs_dir }}" - - "{{ certs_data_dir }}" - -# The certs web root must be owned -# by the web user so the certs service -# can write files there. -- name: Create certs web root - file: - path: "{{ CERTS_WEB_ROOT }}" - state: directory - owner: "{{ common_web_group }}" - group: "{{ certs_user }}" - -- name: Create certs gpg dir - file: - path: "{{ certs_gpg_dir }}" - state: directory - owner: "{{ common_web_user }}" - mode: "0700" - -- name: Copy the private gpg signing key - copy: - src: "{{ CERTS_LOCAL_PRIVATE_KEY }}" - dest: "{{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY | basename }}" - owner: "{{ common_web_user }}" - mode: "0600" - register: certs_gpg_key - no_log: True - -- name: Copy the pgp trust export - copy: - content: "{{ CERTS_OWNER_TRUST }}" - dest: "{{ certs_app_dir }}/trust.export" - owner: "{{ common_web_user }}" - mode: "0600" - -- name: Load the gpg key - shell: "/usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY | basename }}" - become_user: "{{ common_web_user }}" - when: certs_gpg_key.changed - -- name: Import the trust export - shell: "/usr/bin/gpg --homedir {{ certs_gpg_dir }} --import-ownertrust {{ certs_app_dir }}/trust.export" - become_user: "{{ common_web_user }}" - when: certs_gpg_key.changed - -- name: add deadsnakes repo - apt_repository: - repo: ppa:deadsnakes/ppa - when: ansible_distribution_version is version('20.04', '<') - -- name: install python3.8 - apt: - pkg: - - python3.8-dev - - python3.8-distutils - when: ansible_distribution_version is version('20.04', '<') - -- include: deploy.yml - tags: - - deploy diff --git a/playbooks/roles/certs/tasks/tag_ec2.yml b/playbooks/roles/certs/tasks/tag_ec2.yml deleted file mode 100644 index 6a6eef327e6..00000000000 --- a/playbooks/roles/certs/tasks/tag_ec2.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -- name: Get instance information - action: ec2_metadata_facts - -- name: Tag instance - ec2_tag: - resource: "{{ ansible_ec2_instance_id }}" - region: "{{ ansible_ec2_placement_region }}" - tags: - "version:certs" : "{{ CERT_REPO }} {{ certs_checkout.after }}" - when: certs_checkout.after is defined diff --git a/playbooks/roles/certs/templates/boto.j2 b/playbooks/roles/certs/templates/boto.j2 deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/playbooks/roles/certs/templates/certs.auth.json.j2 b/playbooks/roles/certs/templates/certs.auth.json.j2 deleted file mode 100644 index 339bc104354..00000000000 --- a/playbooks/roles/certs/templates/certs.auth.json.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ certs_auth_config | to_nice_json }} diff --git a/playbooks/roles/certs/templates/certs.conf.j2 b/playbooks/roles/certs/templates/certs.conf.j2 deleted file mode 100644 index d0ac6a4061f..00000000000 --- a/playbooks/roles/certs/templates/certs.conf.j2 +++ /dev/null @@ -1,9 +0,0 @@ -[program:certs] -command={{ certs_venv_bin }}/python {{ certs_code_dir }}/certificate_agent.py -priority=999 -environment=SERVICE_VARIANT="certs",HOME="/",BOTO_CONFIG="{{ certs_app_dir }}/.boto" -user={{ common_web_user }} -stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log -stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -killasgroup=true -stopasgroup=true diff --git a/playbooks/roles/certs/templates/certs.env.json.j2 b/playbooks/roles/certs/templates/certs.env.json.j2 deleted file mode 100644 index d53303433d3..00000000000 --- a/playbooks/roles/certs/templates/certs.env.json.j2 +++ /dev/null @@ -1 +0,0 @@ -{{ certs_env_config | to_nice_json }} diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index 02514bbde10..2bbe1320001 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -64,7 +64,6 @@ repos_to_cmd["xqueue"]="$edx_ansible_cmd xqueue.yml -e 'XQUEUE_VERSION=$2' -e 'e repos_to_cmd["credentials"]="$edx_ansible_cmd credentials.yml -e 'credentials_version=$2'" repos_to_cmd["cs_comments_service"]="$edx_ansible_cmd forum.yml -e 'FORUM_VERSION=$2'" repos_to_cmd["configuration"]="$edx_ansible_cmd edx_ansible.yml -e 'CONFIGURATION_VERSION=$2'" -repos_to_cmd["read-only-certificate-code"]="$edx_ansible_cmd certs.yml -e 'CERTS_VERSION=$2'" repos_to_cmd["edx-analytics-data-api"]="$edx_ansible_cmd analyticsapi.yml -e 'ANALYTICS_API_VERSION=$2'" repos_to_cmd["edx-ora2"]="$edx_ansible_cmd ora2.yml -e 'ora2_version=$2'" repos_to_cmd["insights"]="$edx_ansible_cmd insights.yml -e 'INSIGHTS_VERSION=$2'" diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 702561eb5c1..5fd3433f49b 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -189,7 +189,6 @@ COMMON_ENABLE_BASIC_AUTH: False REDIRECT_NGINX_PORT: "{{ EDXAPP_LMS_NGINX_PORT }}" REDIRECT_SSL_NGINX_PORT: "{{ EDXAPP_LMS_SSL_NGINX_PORT }}" -CERTS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}" ECOMMERCE_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}" EDXAPP_CMS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}" EDXAPP_LMS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}" @@ -200,7 +199,6 @@ XQUEUE_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}" NGINX_CREATE_HTPASSWD_FILE: > {{ - CERTS_ENABLE_BASIC_AUTH|bool or ECOMMERCE_ENABLE_BASIC_AUTH|bool or EDXAPP_CMS_ENABLE_BASIC_AUTH|bool or EDXAPP_LMS_ENABLE_BASIC_AUTH|bool or diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/certs.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/certs.j2 deleted file mode 100644 index 7dbaa45c859..00000000000 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/certs.j2 +++ /dev/null @@ -1,11 +0,0 @@ -server { - listen {{ CERTS_NGINX_PORT }} default_server; - - location / { - root {{ CERTS_WEB_ROOT }}; - {% if CERTS_ENABLE_BASIC_AUTH|bool %} - {% include "basic-auth.j2" %} - {% endif %} - try_files $uri $uri/valid.html =404; - } -} diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index e01ddc83f95..ed3af4009c5 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -4,8 +4,6 @@ #EDXAPP_LMS_BASE: ${deploy_host} #EDXAPP_CMS_BASE: studio-${deploy_host} #EDXAPP_SITE_NAME: ${deploy_host} -#CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090" -#CERTS_VERIFY_URL: "http://${deploy_host}:18090" #edx_internal: True #COMMON_USER_INFO: # - name: ${github_username} @@ -68,7 +66,6 @@ #EDXAPP_WORKERS: # lms: 2 # cms: 2 -#CERTS_BUCKET: "verify-test.example.org" #migrate_db: "yes" #rabbitmq_ip: "127.0.0.1" #rabbitmq_refresh: True @@ -145,9 +142,6 @@ #AWS_S3_LOGS_FROM_EMAIL: devops@example.com #EDX_ANSIBLE_DUMP_VARS: true #CONFIGURATION_VERSION: release -#CERTS_AWS_KEY: 'AWS SECRET KEY HERE' -#CERTS_AWS_ID: 'AWS KEY ID HERE' -#CERTS_REPO: "git@github.com:/edx/certificates" #EDXAPP_INSTALL_PRIVATE_REQUIREMENTS: true #EDXAPP_USE_GIT_IDENTITY: true #_local_git_identity: | diff --git a/util/install/native.sh b/util/install/native.sh index cd93446eea8..d5337a499a2 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -111,7 +111,6 @@ sudo -H pip3 install --upgrade virtualenv==20.2.0 ## VERSION_VARS=( EDX_PLATFORM_VERSION - CERTS_VERSION FORUM_VERSION XQUEUE_VERSION CONFIGURATION_VERSION diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 6ca9115a0cb..56aac3cf8b1 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -250,7 +250,6 @@ cat << EOF > $extra_vars_file EDX_PLATFORM_VERSION: $edxapp_version FORUM_VERSION: $forum_version XQUEUE_VERSION: $xqueue_version -CERTS_VERSION: $certs_version CONFIGURATION_VERSION: $configuration_version DEMO_VERSION: $demo_version THEMES_VERSION: $themes_version @@ -398,8 +397,6 @@ EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host} EDXAPP_LMS_BASE: ${deploy_host} EDXAPP_CMS_BASE: studio-${deploy_host} EDXAPP_SITE_NAME: ${deploy_host} -CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090" -CERTS_VERIFY_URL: "http://${deploy_host}:18090" edx_internal: True COMMON_USER_INFO: - name: ${github_username} diff --git a/vagrant/base/analyticstack/Vagrantfile b/vagrant/base/analyticstack/Vagrantfile index 2f837288d6f..8fa50e401b9 100644 --- a/vagrant/base/analyticstack/Vagrantfile +++ b/vagrant/base/analyticstack/Vagrantfile @@ -20,7 +20,6 @@ end VERSION_VARS = [ 'EDX_PLATFORM_VERSION', 'CONFIGURATION_VERSION', - 'CERTS_VERSION', 'FORUM_VERSION', 'XQUEUE_VERSION', 'DEMO_VERSION', diff --git a/vagrant/release/analyticstack/Vagrantfile b/vagrant/release/analyticstack/Vagrantfile index 8cf9c233f49..f80a61c60ff 100644 --- a/vagrant/release/analyticstack/Vagrantfile +++ b/vagrant/release/analyticstack/Vagrantfile @@ -15,7 +15,6 @@ CPU_COUNT = 2 VERSION_VARS = [ 'EDX_PLATFORM_VERSION', 'CONFIGURATION_VERSION', - 'CERTS_VERSION', 'FORUM_VERSION', 'XQUEUE_VERSION', 'DEMO_VERSION', From 9a25463e88c870daba15d75f721eac7915b95fc8 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 20 Aug 2021 14:44:58 +0500 Subject: [PATCH 135/664] fix: regex_replace double replacement with python 3.8 --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- playbooks/roles/hermes/defaults/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index a2872ece431..85d3c547ca6 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -690,8 +690,8 @@ EDXAPP_RECALCULATE_GRADES_ROUTING_KEY: 'edx.lms.core.default' EDXAPP_POLICY_CHANGE_GRADES_ROUTING_KEY: 'edx.lms.core.default' EDXAPP_BULK_EMAIL_ROUTING_KEY_SMALL_JOBS: 'edx.lms.core.default' EDXAPP_PROGRAM_CERTIFICATES_ROUTING_KEY: 'edx.lms.core.default' -EDXAPP_LMS_CELERY_QUEUES: "{{ edxapp_workers|selectattr('service_variant', 'equalto', 'lms')|map(attribute='queue')|map('regex_replace', '(.*)', 'edx.lms.core.\\1')|list }}" -EDXAPP_CMS_CELERY_QUEUES: "{{ edxapp_workers|selectattr('service_variant', 'equalto', 'cms')|map(attribute='queue')|map('regex_replace', '(.*)', 'edx.cms.core.\\1')|list }}" +EDXAPP_LMS_CELERY_QUEUES: "{{ edxapp_workers|selectattr('service_variant', 'equalto', 'lms')|map(attribute='queue')|map('regex_replace', '^(.*)$', 'edx.lms.core.\\1')|list }}" +EDXAPP_CMS_CELERY_QUEUES: "{{ edxapp_workers|selectattr('service_variant', 'equalto', 'cms')|map(attribute='queue')|map('regex_replace', '^(.*)$', 'edx.cms.core.\\1')|list }}" EDXAPP_DEFAULT_CACHE_VERSION: "1" EDXAPP_OAUTH_ENFORCE_SECURE: True diff --git a/playbooks/roles/hermes/defaults/main.yml b/playbooks/roles/hermes/defaults/main.yml index 005703dd4fe..de8dec1899a 100644 --- a/playbooks/roles/hermes/defaults/main.yml +++ b/playbooks/roles/hermes/defaults/main.yml @@ -73,7 +73,7 @@ HERMES_SERVICE_CONFIG: - url: '{{ HERMES_REMOTE_FILE_PATH }}' filename: '{{ HERMES_LOCAL_FILE_PATH }}' command: 'sudo {{ HERMES_COPY_COMMAND }} && sudo {{ HERMES_RELOAD_COMMAND }}' - secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','(.*)','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" + secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','^(.*)$','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" # These are dropped into sudoers for the user that runs this program, care should be taken to ensure they are safe # to run. By default we assume the 1 service per box and restart supervisor model. If you did something custom with From 66c4204751d8e07379bf9e64d4fbfba3d46ef175 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 23 Aug 2021 15:02:08 +0500 Subject: [PATCH 136/664] fix regex replace hermes --- playbooks/edxapp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/edxapp.yml b/playbooks/edxapp.yml index 886ae30db0e..2364f0f585a 100644 --- a/playbooks/edxapp.yml +++ b/playbooks/edxapp.yml @@ -45,8 +45,8 @@ - url: '{{ HERMES_REMOTE_FILE_LOCATION }}/{{ COMMON_ENVIRONMENT }}/lms.yml' filename: '{{ hermes_download_dir }}/lms.yml' command: "sudo /bin/cp {{ hermes_download_dir }}/lms.yml {{ COMMON_CFG_DIR }}/lms.yml && sudo /edx/app/edxapp/reload_lms_config.sh" - secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','(.*)','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" + secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','^(.*)$','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" - url: '{{ HERMES_REMOTE_FILE_LOCATION }}/{{ COMMON_ENVIRONMENT }}/studio.yml' filename: '{{ hermes_download_dir }}/studio.yml' command: "sudo /bin/cp {{ hermes_download_dir }}/studio.yml {{ COMMON_CFG_DIR }}/studio.yml && sudo /edx/app/edxapp/reload_cms_config.sh" - secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','(.*)','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" + secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','^(.*)$','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" From bd1f52aff62869f2d1214827fca70665e5e245d3 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Mon, 23 Aug 2021 20:18:28 +0930 Subject: [PATCH 137/664] fix: removes certs from list of nginx sites (#6524) cf https://github.com/edx/configuration/pull/6501#issuecomment-902496617 --- playbooks/openedx_native.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 6bf4a1f5b7c..3342c9f9edb 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -74,7 +74,6 @@ when: COMMON_ENABLE_AWS_ROLE - role: nginx nginx_sites: - - certs - cms - lms - forum From b6e167d9ed4cf230d0ea2eea309e3a287fa14245 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 24 Aug 2021 06:38:43 +0000 Subject: [PATCH 138/664] python requirements update --- .../roles/aws/templates/requirements.txt.j2 | 22 +++++----- requirements.txt | 41 +++++++++---------- requirements/aws.in | 2 +- requirements/pip-tools.txt | 13 ++++-- util/elasticsearch/requirements.txt | 8 +--- util/jenkins/requirements-cloudflare.txt | 4 +- util/jenkins/requirements.txt | 34 +++++++-------- util/pingdom/requirements.txt | 6 +-- util/vpc-tools/requirements.txt | 12 +++--- 9 files changed, 70 insertions(+), 72 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index cae4e7d7a76..03d408ef620 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # -awscli==1.18.223 - # via -r requirements/aws.in -boto3==1.16.63 +awscli==1.20.27 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -botocore==1.19.63 +boto3==1.18.27 + # via -r requirements/aws.in +botocore==1.21.27 # via # awscli # boto3 @@ -25,25 +25,25 @@ jmespath==0.10.0 # botocore pyasn1==0.4.8 # via rsa -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # botocore # s3cmd -python-magic==0.4.22 +python-magic==0.4.24 # via s3cmd pyyaml==5.3.1 # via # -r requirements/aws.in # awscli -rsa==4.5 +rsa==4.7.2 # via awscli s3cmd==2.1.0 # via -r requirements/aws.in -s3transfer==0.3.4 +s3transfer==0.5.0 # via # awscli # boto3 -six==1.15.0 +six==1.16.0 # via python-dateutil -urllib3==1.26.4 +urllib3==1.26.6 # via botocore diff --git a/requirements.txt b/requirements.txt index 9f45941645e..8e63c9026a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,44 +1,44 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # -ansible==2.8.19 +ansible==2.8.20 # via -r requirements/base.in -awscli==1.18.223 +awscli==1.20.27 # via -r requirements/base.in bcrypt==3.1.7 # via # -c requirements/constraints.txt # paramiko -boto3==1.16.63 - # via -r requirements/base.in boto==2.49.0 # via -r requirements/base.in -botocore==1.19.63 +boto3==1.18.27 + # via -r requirements/base.in +botocore==1.21.27 # via # awscli # boto3 # s3transfer -certifi==2020.12.5 +certifi==2021.5.30 # via requests -cffi==1.14.5 +cffi==1.14.6 # via # bcrypt # cryptography # pynacl -chardet==4.0.0 +charset-normalizer==2.0.4 # via requests colorama==0.4.3 # via awscli -cryptography==3.2.1 +cryptography==3.4.7 # via # ansible # paramiko datadog==0.8.0 # via -r requirements/base.in -decorator==4.4.2 +decorator==5.0.9 # via # datadog # networkx @@ -48,7 +48,7 @@ docutils==0.15.2 # via awscli ecdsa==0.13.3 # via -r requirements/base.in -idna==2.10 +idna==3.2 # via requests jinja2==2.8 # via @@ -58,7 +58,7 @@ jmespath==0.10.0 # via # boto3 # botocore -markupsafe==1.1.1 +markupsafe==2.0.1 # via jinja2 mysqlclient==1.4.6 # via -r requirements/base.in @@ -82,31 +82,30 @@ pymongo==3.9.0 # via -r requirements/base.in pynacl==1.4.0 # via paramiko -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via botocore -pyyaml==5.3.1 +pyyaml==5.4.1 # via # -r requirements/base.in # ansible # awscli -requests==2.25.1 +requests==2.26.0 # via # -r requirements/base.in # datadog -rsa==4.5 +rsa==4.7.2 # via awscli -s3transfer==0.3.4 +s3transfer==0.5.0 # via # awscli # boto3 -six==1.15.0 +six==1.16.0 # via # bcrypt - # cryptography # pathlib2 # pynacl # python-dateutil -urllib3==1.26.4 +urllib3==1.26.6 # via # botocore # requests diff --git a/requirements/aws.in b/requirements/aws.in index e457993d02a..f76449ccdef 100644 --- a/requirements/aws.in +++ b/requirements/aws.in @@ -5,4 +5,4 @@ boto boto3 futures ; python_version == "2.7" # via s3transfer s3cmd -pyyaml +pyyaml==5.3.1 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index aa6ffb82da2..3a172bbf62a 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,13 +1,20 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # -click==7.1.2 +click==8.0.1 # via pip-tools -pip-tools==5.5.0 +pep517==0.11.0 + # via pip-tools +pip-tools==6.2.0 # via -r requirements/pip-tools.in +tomli==1.2.1 + # via pep517 +wheel==0.37.0 + # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip +# setuptools diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 754191b2b89..5999dcee7b6 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade @@ -8,11 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -importlib-metadata==2.1.1 - # via jsonpickle jsonpickle==2.0.0 # via deepdiff -urllib3==1.26.4 +urllib3==1.26.6 # via elasticsearch -zipp==1.2.0 - # via importlib-metadata diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 47272aa7795..5252f53cd28 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # -certifi==2020.12.5 +certifi==2021.5.30 # via requests chardet==3.0.4 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 644e294b69c..9e6341a9f1b 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade @@ -8,28 +8,28 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.19.55 +awscli==1.20.27 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in billiard==3.3.0.23 # via celery -boto3==1.17.55 - # via -r requirements/jenkins.in boto==2.49.0 # via -r requirements/jenkins.in -botocore==1.20.55 +boto3==1.18.27 + # via -r requirements/jenkins.in +botocore==1.21.27 # via # awscli # boto3 # s3transfer celery==3.1.25 # via -r requirements/jenkins.in -certifi==2020.12.5 +certifi==2021.5.30 # via # opsgenie-sdk # requests -chardet==4.0.0 +charset-normalizer==2.0.4 # via requests click==6.7 # via -r requirements/jenkins.in @@ -37,11 +37,7 @@ colorama==0.4.3 # via awscli docutils==0.15.2 # via awscli -futures==3.3.0 ; python_version == "2.7" - # via - # -r requirements/jenkins.in - # s3transfer -idna==2.10 +idna==3.2 # via requests jmespath==0.10.0 # via @@ -55,12 +51,12 @@ pyasn1==0.4.8 # via rsa pymysql==0.9.3 # via -r requirements/jenkins.in -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # botocore # opsgenie-sdk # s3cmd -python-magic==0.4.22 +python-magic==0.4.24 # via s3cmd pytz==2021.1 # via @@ -72,23 +68,23 @@ pyyaml==5.4.1 # awscli redis==2.10.6 # via -r requirements/jenkins.in -requests==2.25.1 +requests==2.26.0 # via opsgenie-sdk -rsa==4.5 +rsa==4.7.2 # via awscli s3cmd==2.1.0 # via -r requirements/jenkins.in -s3transfer==0.4.1 +s3transfer==0.5.0 # via # awscli # boto3 -six==1.15.0 +six==1.16.0 # via # opsgenie-sdk # python-dateutil splunk-sdk==1.6.6 # via -r requirements/jenkins.in -urllib3==1.26.4 +urllib3==1.26.6 # via # botocore # opsgenie-sdk diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 2751140fcdb..4329c25e6ec 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # -certifi==2020.12.5 +certifi==2021.5.30 # via requests chardet==3.0.4 # via requests @@ -12,7 +12,7 @@ click==6.7 # via -r requirements/pingdom.in idna==2.7 # via requests -pyyaml==5.3.1 +pyyaml==5.4.1 # via -r requirements/pingdom.in requests==2.20.0 # via -r requirements/pingdom.in diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index e5b55226d8b..b42dce25ee8 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make upgrade # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2020.12.5 +certifi==2021.5.30 # via requests -chardet==4.0.0 +charset-normalizer==2.0.4 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==2.10 +idna==3.2 # via requests -requests==2.25.1 +requests==2.26.0 # via -r requirements/vpc-tools.in -urllib3==1.26.4 +urllib3==1.26.6 # via requests From 2352b729a6b2a8b7b431061d8b5a007abbd73103 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 24 Aug 2021 13:36:34 +0500 Subject: [PATCH 139/664] remove python3.5 checks --- .github/workflows/playbook-test.yml | 2 +- .github/workflows/syntax-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playbook-test.yml b/.github/workflows/playbook-test.yml index 6d9f82aed67..d1d102c45d3 100644 --- a/.github/workflows/playbook-test.yml +++ b/.github/workflows/playbook-test.yml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.5, 3.8] + python-version: [3.8] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/syntax-test.yml b/.github/workflows/syntax-test.yml index 88a7102bcb0..d53dbce9ac8 100644 --- a/.github/workflows/syntax-test.yml +++ b/.github/workflows/syntax-test.yml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.5, 3.8] + python-version: [3.8] steps: - uses: actions/checkout@v2 From 65f9c1af2b670003c01b1882d2ffeaa77bc05eea Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 24 Aug 2021 09:42:11 +0000 Subject: [PATCH 140/664] update python requirements --- requirements/cloudflare.in | 3 ++- requirements/pingdom.in | 2 +- util/check_rds_configs/requirements.txt | 10 +--------- util/cloudflare/by_origin_purger/requirements.txt | 7 +------ util/jenkins/requirements-cloudflare.txt | 10 ++++++---- util/pingdom/requirements.txt | 8 ++++---- 6 files changed, 15 insertions(+), 25 deletions(-) mode change 100644 => 120000 util/check_rds_configs/requirements.txt mode change 100644 => 120000 util/cloudflare/by_origin_purger/requirements.txt diff --git a/requirements/cloudflare.in b/requirements/cloudflare.in index d4e6108020c..4704073c2c8 100644 --- a/requirements/cloudflare.in +++ b/requirements/cloudflare.in @@ -1,3 +1,4 @@ # Needed for CloudFlare cache hit rate job in util/jenkins -requests==2.20.0 +requests +click diff --git a/requirements/pingdom.in b/requirements/pingdom.in index e9ff57f0095..4112c810fa6 100644 --- a/requirements/pingdom.in +++ b/requirements/pingdom.in @@ -2,5 +2,5 @@ click==6.7 PyYAML -requests==2.20.0 +requests six==1.14.0 diff --git a/util/check_rds_configs/requirements.txt b/util/check_rds_configs/requirements.txt deleted file mode 100644 index 9a09b17b262..00000000000 --- a/util/check_rds_configs/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -boto3==1.9.96 -botocore==1.12.214 -Click==7.0 -docutils==0.15.2 -jmespath==0.9.4 -python-dateutil==2.8.0 -s3transfer==0.2.1 -six==1.12.0 -urllib3==1.25.3 diff --git a/util/check_rds_configs/requirements.txt b/util/check_rds_configs/requirements.txt new file mode 120000 index 00000000000..68c7b6c4342 --- /dev/null +++ b/util/check_rds_configs/requirements.txt @@ -0,0 +1 @@ +../jenkins/requirements.txt \ No newline at end of file diff --git a/util/cloudflare/by_origin_purger/requirements.txt b/util/cloudflare/by_origin_purger/requirements.txt deleted file mode 100644 index 9b31d71a232..00000000000 --- a/util/cloudflare/by_origin_purger/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -certifi==2018.10.15 -chardet==3.0.4 -Click==7.0 -idna==2.7 -requests==2.20.0 -urllib3==1.24.2 diff --git a/util/cloudflare/by_origin_purger/requirements.txt b/util/cloudflare/by_origin_purger/requirements.txt new file mode 120000 index 00000000000..8a8582a57da --- /dev/null +++ b/util/cloudflare/by_origin_purger/requirements.txt @@ -0,0 +1 @@ +../../jenkins/requirements-cloudflare.txt \ No newline at end of file diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 5252f53cd28..7ffec39e4e0 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -6,11 +6,13 @@ # certifi==2021.5.30 # via requests -chardet==3.0.4 +charset-normalizer==2.0.4 # via requests -idna==2.7 +click==8.0.1 + # via -r requirements/cloudflare.in +idna==3.2 # via requests -requests==2.20.0 +requests==2.26.0 # via -r requirements/cloudflare.in -urllib3==1.24.3 +urllib3==1.26.6 # via requests diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 4329c25e6ec..070854d0955 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -6,17 +6,17 @@ # certifi==2021.5.30 # via requests -chardet==3.0.4 +charset-normalizer==2.0.4 # via requests click==6.7 # via -r requirements/pingdom.in -idna==2.7 +idna==3.2 # via requests pyyaml==5.4.1 # via -r requirements/pingdom.in -requests==2.20.0 +requests==2.26.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.24.3 +urllib3==1.26.6 # via requests From d23afd8ed2e2bc1ed727ae14c4b7bc7a0499b042 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 25 Aug 2021 14:17:11 +0500 Subject: [PATCH 141/664] update hermes role to use python3.8 --- playbooks/roles/hermes/tasks/main.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/playbooks/roles/hermes/tasks/main.yml b/playbooks/roles/hermes/tasks/main.yml index e2e4148b23e..b9bf1ff2a97 100644 --- a/playbooks/roles/hermes/tasks/main.yml +++ b/playbooks/roles/hermes/tasks/main.yml @@ -20,17 +20,8 @@ # # -# The deadsnakes PPA is required to install python3.6 on Xenial. -# Bionic comes with python3.6 installed. -- name: add deadsnakes repository - apt_repository: - repo: "ppa:fkrull/deadsnakes" - when: ansible_distribution_release == 'xenial' - tags: - - install - - install:system-requirements - -- name: install python3.6 +# Install python3.8 on Bionic.Focal comes with python3.8 installed. +- name: install python3.8 apt: name: "{{ item }}" update_cache: yes @@ -39,15 +30,15 @@ retries: 10 delay: 5 with_items: - - python3.6 + - python3.8 - python3-pip - when: ansible_distribution_release == 'xenial' or ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'bionic' tags: - install - install:system-requirements - name: build virtualenv with python3 - command: "virtualenv --python=/usr/bin/python3.6 {{ hermes_venv_dir }}" + command: "virtualenv --python=/usr/bin/python3.8 {{ hermes_venv_dir }}" args: creates: "{{ hermes_venv_dir }}/bin/pip" become_user: "{{ hermes_user }}" From 2505cda7fd2c403e5e011ae22b69b8578c57de9b Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 25 Aug 2021 15:52:52 +0500 Subject: [PATCH 142/664] Revert "feat: update hermes role to use python3.8" --- playbooks/roles/hermes/tasks/main.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/hermes/tasks/main.yml b/playbooks/roles/hermes/tasks/main.yml index b9bf1ff2a97..e2e4148b23e 100644 --- a/playbooks/roles/hermes/tasks/main.yml +++ b/playbooks/roles/hermes/tasks/main.yml @@ -20,8 +20,17 @@ # # -# Install python3.8 on Bionic.Focal comes with python3.8 installed. -- name: install python3.8 +# The deadsnakes PPA is required to install python3.6 on Xenial. +# Bionic comes with python3.6 installed. +- name: add deadsnakes repository + apt_repository: + repo: "ppa:fkrull/deadsnakes" + when: ansible_distribution_release == 'xenial' + tags: + - install + - install:system-requirements + +- name: install python3.6 apt: name: "{{ item }}" update_cache: yes @@ -30,15 +39,15 @@ retries: 10 delay: 5 with_items: - - python3.8 + - python3.6 - python3-pip - when: ansible_distribution_release == 'bionic' + when: ansible_distribution_release == 'xenial' or ansible_distribution_release == 'focal' tags: - install - install:system-requirements - name: build virtualenv with python3 - command: "virtualenv --python=/usr/bin/python3.8 {{ hermes_venv_dir }}" + command: "virtualenv --python=/usr/bin/python3.6 {{ hermes_venv_dir }}" args: creates: "{{ hermes_venv_dir }}/bin/pip" become_user: "{{ hermes_user }}" From 009cd96aee8164b38252af06a2e82f91fa7c26ce Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Wed, 25 Aug 2021 16:43:50 +0500 Subject: [PATCH 143/664] Revert "Revert "feat: update hermes role to use python3.8"" --- playbooks/roles/hermes/tasks/main.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/playbooks/roles/hermes/tasks/main.yml b/playbooks/roles/hermes/tasks/main.yml index e2e4148b23e..b9bf1ff2a97 100644 --- a/playbooks/roles/hermes/tasks/main.yml +++ b/playbooks/roles/hermes/tasks/main.yml @@ -20,17 +20,8 @@ # # -# The deadsnakes PPA is required to install python3.6 on Xenial. -# Bionic comes with python3.6 installed. -- name: add deadsnakes repository - apt_repository: - repo: "ppa:fkrull/deadsnakes" - when: ansible_distribution_release == 'xenial' - tags: - - install - - install:system-requirements - -- name: install python3.6 +# Install python3.8 on Bionic.Focal comes with python3.8 installed. +- name: install python3.8 apt: name: "{{ item }}" update_cache: yes @@ -39,15 +30,15 @@ retries: 10 delay: 5 with_items: - - python3.6 + - python3.8 - python3-pip - when: ansible_distribution_release == 'xenial' or ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'bionic' tags: - install - install:system-requirements - name: build virtualenv with python3 - command: "virtualenv --python=/usr/bin/python3.6 {{ hermes_venv_dir }}" + command: "virtualenv --python=/usr/bin/python3.8 {{ hermes_venv_dir }}" args: creates: "{{ hermes_venv_dir }}/bin/pip" become_user: "{{ hermes_user }}" From a490bcdcbc5e675595bcd93dcae0d40ae9b57036 Mon Sep 17 00:00:00 2001 From: Arbab Khalil Date: Fri, 27 Aug 2021 12:54:44 +0500 Subject: [PATCH 144/664] edX | PSRE-974 | Moving create_data_czar in 'configuration/util' from sysadmin repo (#6516) * PSRE-974 | Moving create_data_czar in 'configuration/util' from sysadmin repo --- .../assign_czar_org_groups.py | 58 ++++++++++++ util/create_data_czar/create_data_czar.py | 51 +++++++++++ .../create_org_data_czar_policy.py | 88 +++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100755 util/create_data_czar/assign_czar_org_groups.py create mode 100755 util/create_data_czar/create_data_czar.py create mode 100755 util/create_data_czar/create_org_data_czar_policy.py diff --git a/util/create_data_czar/assign_czar_org_groups.py b/util/create_data_czar/assign_czar_org_groups.py new file mode 100755 index 00000000000..f5a30b340c6 --- /dev/null +++ b/util/create_data_czar/assign_czar_org_groups.py @@ -0,0 +1,58 @@ +""" +assign_czar_org_groups.py + +Assigns data czars to the iam groups for their org based on the configuration specificed in the analytics-exporter +repository, https://github.com/edx/analytics-exporter/blob/master/config/default.yaml. + +The file needs to be available locally as the repo is private. + +Assumes that a group for the org has already been created using the create_org_data_czar_polcy.py script. + +Assumes that the data czars email is their IAM user name. + +Assumes that org names are consistent in s3 and the yaml config file and IAM. + +""" + +import argparse +import boto +import yaml +import sys + + + +parser = argparse.ArgumentParser() +parser.add_argument('-f', '--file', help='Path to the Analytics YAML file containing ' + 'the organization meta-data which is located ') +parser.add_argument('-p', '--profile', help='The IAM profile to use when ' + 'adding user to groups') +args = parser.parse_args() + + +org_group_name_template = "edx-course-data-{org}" + +with open(args.file) as config: + data = yaml.load(config) + + +iam_connection = boto.connect_iam(profile_name=args.profile) + +for group, group_info in data['organizations'].items(): + print(f"Adding {group_info['recipients']} to group {group}.") + + # Add to the group providing general permissions for all data czars. + try: + for user in group_info['recipients']: + iam_connection.add_user_to_group('analytics-edx-course-data-s3-ro', user) + except Exception as e: + print(e) + + # Add to the org specific group + try: + pass + for user in group_info['recipients']: + iam_connection.add_user_to_group(org_group_name_template.format(org=group), user) + except Exception as e: + print(e) + +sys.exit(0) diff --git a/util/create_data_czar/create_data_czar.py b/util/create_data_czar/create_data_czar.py new file mode 100755 index 00000000000..6a75679ae0f --- /dev/null +++ b/util/create_data_czar/create_data_czar.py @@ -0,0 +1,51 @@ +import boto +import argparse +import gnupg + +# Assumes you have GPG already installed +# Assumes that the Data Czars already have your public key +# Asumes that .boto is configured with edX Prod account + +# Parser +parser = argparse.ArgumentParser(description="Username of Data Czar.") +parser.add_argument('-u', '--user', help='Email of Data Czar', required=True) +parser.add_argument('-f', '--file', help='Public Key file', required=True) +parser.add_argument('--credentials-only', help='Only create new credentials', default=False, action='store_true') +parser.add_argument('-o', '--orgs', nargs='*', help='Name of the org(s) as list, User need to be a member', default=None) +args = parser.parse_args() + +# Import Data Czar GPG Key +gpg = gnupg.GPG() +key_data = open(args.file).read() +import_result = gpg.import_keys(key_data) + +# Connect to AWS and create account +iam = boto.connect_iam() + +if not args.credentials_only: + user_response = iam.create_user(args.user) + +key_response = iam.create_access_key(args.user) + +# Add user to group edx-s3bucket-course-data-readonly +iam.add_user_to_group('edx-s3bucket-course-data-readonly', args.user) + +# Add user to it's respective Org +if args.orgs: + for org in args.orgs: + user_org = 'edx-course-data-' + org.lower() + iam.add_user_to_group(user_org, args.user) + +# Create AWS Cred String +key = key_response.create_access_key_response.create_access_key_result.access_key +credstring = str('AWS_ACCESS_KEY_ID = ' + key.access_key_id + '\n' + 'AWS_SECRET_ACCESS_KEY = ' + key.secret_access_key) + +# Encrypt file +encrypted_data = gpg.encrypt(credstring, args.user, always_trust=True) +encrypted_string = str(encrypted_data) +gpgfile = open(args.user + '-credentials.txt.gpg', 'w+') +gpgfile.write(encrypted_string) + +print('ok: ', encrypted_data.ok) +print('status: ', encrypted_data.status) +print('stderr: ', encrypted_data.stderr) diff --git a/util/create_data_czar/create_org_data_czar_policy.py b/util/create_data_czar/create_org_data_czar_policy.py new file mode 100755 index 00000000000..a1894565346 --- /dev/null +++ b/util/create_data_czar/create_org_data_czar_policy.py @@ -0,0 +1,88 @@ +""" +create_org_data_czar_policy.py + +Creates an IAM group for an edX org and applies an S3 policy to that group +that allows for read-only access to the group. + +""" + +import argparse +import boto +from string import Template +import sys + +template = Template("""{ + "Version":"2012-10-17", + "Statement": [ + { + "Sid": "AllowListingOfOrgFolder", + "Action": ["s3:ListBucket"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::edx-course-data"], + "Condition":{"StringLike":{"s3:prefix":["$org","$org/*"]}} + }, + { + "Sid": "AllowGetBucketLocation", + "Action": ["s3:GetBucketLocation"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::edx-course-data"] + }, + { + "Sid": "AllowGetS3ActionInOrgFolder", + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::edx-course-data/$org/*"] + } + ] +}""") + + +def add_org_group(org, iam_connection): + group_name = "edx-course-data-{org}".format(org=org) + + try: + iam_connection.create_group(group_name) + except boto.exception.BotoServerError as bse: + if bse.status == 409: + pass + else: + print(bse) + + try: + iam_connection.put_group_policy( + group_name,group_name,template.substitute(org=org)) + except boto.exception.BotoServerError as bse: + if bse.status == 409: + pass + else: + print(bse) + print(template.substitute(org=org)) + + +parser = argparse.ArgumentParser() +group = parser.add_mutually_exclusive_group() +group.add_argument('-o', '--org', help='Name of the org for which to create an IAM ' + 'role and policy, this should have the same ' + 'name as the S3 bucket') +group.add_argument('-f', '--file', help='The path to a file containing one org name ' + 'per line.') + +parser.add_argument('-p', '--profile', help='The IAM profile to use when creating ' + 'the group and policy.') + +args = parser.parse_args() + +iam_connection = boto.connect_iam(profile_name=args.profile) + +if args.org: + add_org_group(args.org.rstrip('\n').lower(), iam_connection) +elif args.file: + with open(args.file) as file: + for line in file: + org = line.rstrip('\n').lower() + add_org_group(org, iam_connection) +else: + parser.print_usage() + sys.exit(1) + +sys.exit(0) From 7b9c1e21925f6086372026254fb85ea7f5c162e3 Mon Sep 17 00:00:00 2001 From: albemarle <45690905+albemarle@users.noreply.github.com> Date: Fri, 27 Aug 2021 10:09:27 -0400 Subject: [PATCH 145/664] WS-2100: Prospectus node 12 -> node 16 --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 2ae1486bc3e..6f67cacbbf9 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '12.21.0' +PROSPECTUS_NODE_VERSION: '16.7.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From 52df7ed3dcb550768a141db30644fa26e6b10d3f Mon Sep 17 00:00:00 2001 From: albemarle <45690905+albemarle@users.noreply.github.com> Date: Fri, 27 Aug 2021 10:55:47 -0400 Subject: [PATCH 146/664] Revert "WS-2100: Prospectus node 12 -> node 16" --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 6f67cacbbf9..2ae1486bc3e 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '16.7.0' +PROSPECTUS_NODE_VERSION: '12.21.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From d4b4d9e5a2b98cb3b27225e4c8e84dbb4cdf3769 Mon Sep 17 00:00:00 2001 From: albemarle <45690905+albemarle@users.noreply.github.com> Date: Fri, 27 Aug 2021 10:09:27 -0400 Subject: [PATCH 147/664] WS-2100: Prospectus node 12 -> node 16 --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 2ae1486bc3e..6f67cacbbf9 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '12.21.0' +PROSPECTUS_NODE_VERSION: '16.7.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From 656b3ef3e1adbc65e0f907a5a2f4394b60003e5c Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 31 Aug 2021 12:14:53 +0500 Subject: [PATCH 148/664] update mongo client version on tools gp --- playbooks/roles/ad_hoc_reporting/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/ad_hoc_reporting/defaults/main.yml b/playbooks/roles/ad_hoc_reporting/defaults/main.yml index 447157a0376..56de3d06f22 100644 --- a/playbooks/roles/ad_hoc_reporting/defaults/main.yml +++ b/playbooks/roles/ad_hoc_reporting/defaults/main.yml @@ -43,8 +43,8 @@ ad_hoc_reporting_pip_pkgs: MONGODB_APT_KEY: "7F0CEB10" MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com" -MONGO_VERSION_MAJOR_MINOR: "4.0" -MONGO_VERSION_PATCH: "26" +MONGO_VERSION_MAJOR_MINOR: "4.2" +MONGO_VERSION_PATCH: "14" MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" From 54aec9ae26f391c67876dcaeab3e597d251fe29b Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 27 Aug 2021 06:57:44 +0500 Subject: [PATCH 149/664] Updated tools-jenkins playbook to isntall helm --- .../roles/tools_jenkins/defaults/main.yml | 2 ++ playbooks/roles/tools_jenkins/tasks/main.yml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/playbooks/roles/tools_jenkins/defaults/main.yml b/playbooks/roles/tools_jenkins/defaults/main.yml index f5b90cd4eea..31ba8a978a2 100644 --- a/playbooks/roles/tools_jenkins/defaults/main.yml +++ b/playbooks/roles/tools_jenkins/defaults/main.yml @@ -68,6 +68,8 @@ jenkins_tools_bundled_plugins: - "ssh-credentials" - "ssh-slaves" +jenkins_tools_helm_url: "https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz" + jenkins_tools_debian_pkgs: - nginx - git diff --git a/playbooks/roles/tools_jenkins/tasks/main.yml b/playbooks/roles/tools_jenkins/tasks/main.yml index 17a8508ded8..0962555522d 100644 --- a/playbooks/roles/tools_jenkins/tasks/main.yml +++ b/playbooks/roles/tools_jenkins/tasks/main.yml @@ -23,3 +23,22 @@ tags: - install - install:system-requirements + +- name: download helm package + get_url: + url: "{{ jenkins_tools_helm_url }}" + dest: "/tmp/helm.tar.gz" + mode: 0644 + +- name: extract helm archive + unarchive: + src: "/tmp/helm.tar.gz" + dest: "/tmp/" + copy: false + +- name: copy helm binary to executable path + copy: + src: /tmp/linux-amd64/helm + dest: /usr/local/bin/helm + remote_src: yes + mode: 0755 From e3f8b677d0881d0158bca08dbf3e5df88ad22dc6 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Wed, 11 Aug 2021 20:22:57 +0500 Subject: [PATCH 150/664] Added helm checker script --- .../helm_update_checker.py | 164 ++++++++++++++++++ .../helm_update_checker/requirements.txt | 1 + 2 files changed, 165 insertions(+) create mode 100644 util/jenkins/helm_update_checker/helm_update_checker.py create mode 120000 util/jenkins/helm_update_checker/requirements.txt diff --git a/util/jenkins/helm_update_checker/helm_update_checker.py b/util/jenkins/helm_update_checker/helm_update_checker.py new file mode 100644 index 00000000000..b83593de8db --- /dev/null +++ b/util/jenkins/helm_update_checker/helm_update_checker.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +import subprocess +import yaml +import sys +import logging +import click +import os +import json +import boto3 +import backoff +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger(__name__) +logging.basicConfig() +global_list = [] +MAX_TRIES = 5 + + +class SESBotoWrapper: + def __init__(self, **kwargs): + self.client = boto3.client("ses", **kwargs) + + @backoff.on_exception(backoff.expo, + ClientError, + max_tries=MAX_TRIES) + def send_email(self, *args, **kwargs): + return self.client.send_email(*args, **kwargs) + + +def parse_yaml(file_name): + with open(file_name, 'r') as stream: + try: + charts_name = yaml.safe_load(stream) + if "dependencies" in charts_name: + app_list = charts_name["dependencies"] + for key in app_list: + add_helm(key["repository"], key["name"]) + update_helm() + for key in app_list: + repo_name = get_repo_name(key["repository"]) + check_version(charts_name["name"], key["name"], repo_name, key["version"]) + except yaml.YAMLError as exc: + LOGGER.error("error in configuration file: %s" % str(exc)) + sys.exit(1) + except KeyError as e: + print('I got a KeyError - reason "%s"' % str(e)) + + +def add_helm(repo_url, repo_name): + try: + cmd_add = 'helm repo add ' + repo_name + " " + repo_url + subprocess.check_output(cmd_add, shell=True) + except subprocess.CalledProcessError as e: + print(e.output) + + +def update_helm(): + cmd_update = 'helm repo update' + subprocess.check_output(cmd_update, shell=True) + + +def get_repo_name(repo_url): + try: + cmd_list = 'helm repo list -o json' + output = subprocess.check_output(cmd_list, shell=True).strip() + output_list = json.loads(output.decode()) + for dic in output_list: + if dic["url"] == repo_url: + return dic['name'] + except subprocess.CalledProcessError as e: + print(e.output) + + +def check_version(chart_name, app_name, repo_name, app_version): + cmd = 'helm show chart ' + repo_name + "/" + app_name + ' | grep version | tail -1' + output = subprocess.check_output(cmd, shell=True) + latest_version = output.decode().split(": ")[-1].rstrip() + if not compare_version(app_version, latest_version): + temp_dict = { + chart_name+"/"+app_name: { + "current_version": app_version, + "latest_version": latest_version + }, + } + global_list.append(temp_dict) + + +def compare_version(current_version, latest_version): + if current_version == latest_version: + return True + return False + + +def find(name, path): + for root, dirs, files in os.walk(path): + if name in files: + parse_yaml(os.path.join(root, name)) + + +def send_an_email(to_addr, from_addr, app_list, region): + ses_client = SESBotoWrapper(region_name=region) + + message = """ +

Hello,

+

Updates are available for the following helm charts

+ + + + + + + """ + for apps in app_list: + for data in apps: + message += """ + + + + """.format( + AppName=data, + CurrentVersion=apps[data]["current_version"], + LatestVersion=apps[data]["latest_version"] + ) + + message += """
App NameCurrent versionLatest Version
{AppName}{CurrentVersion}{LatestVersion}
""" + print(("Sending the following as email to {}".format(to_addr))) + print(message) + ses_client.send_email( + Source=from_addr, + Destination={ + 'ToAddresses': [ + to_addr + ] + }, + Message={ + 'Subject': { + 'Data': 'Updates available for helms charts', + 'Charset': 'utf-8' + }, + 'Body': { + 'Html':{ + 'Data': message, + 'Charset': 'utf-8' + } + } + } + ) + + +@click.command() +@click.option('--file-name', required=True, help='Filename which have helm chart details.') +@click.option('--file-path', required=True, help='File path where helm chart file exists.') +@click.option('--region', multiple=True, help='Default AWS region') +@click.option('--recipient', multiple=True, help='Recipient Email address') +@click.option('--sender', multiple=True, help='Sender email address') +def controller(file_name, file_path, region, recipient, sender): + find(file_name, file_path) + if len(global_list) > 0: + send_an_email(recipient[0], sender[0], global_list, region[0]) + + +if __name__ == "__main__": + controller() diff --git a/util/jenkins/helm_update_checker/requirements.txt b/util/jenkins/helm_update_checker/requirements.txt new file mode 120000 index 00000000000..dc833dd4bef --- /dev/null +++ b/util/jenkins/helm_update_checker/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file From f30790e4046dd7b97e9d311c9148e096193f3f1b Mon Sep 17 00:00:00 2001 From: jfavellar90 Date: Tue, 3 Aug 2021 15:59:56 -0500 Subject: [PATCH 151/664] refactor: enforcing read committed isolation level for edxapp MySQL database --- playbooks/roles/edxapp/defaults/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 85d3c547ca6..b9fe413e56d 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -91,7 +91,14 @@ EDXAPP_MYSQL_USER_ADMIN: 'root' EDXAPP_MYSQL_PASSWORD: 'password' EDXAPP_MYSQL_PASSWORD_READ_ONLY: 'password' EDXAPP_MYSQL_PASSWORD_ADMIN: 'password' -EDXAPP_MYSQL_OPTIONS: {} +# From Django 2.0 the default isolation level used for the MySQL database backend is 'read commited' +# (refer to https://github.com/django/django/pull/7978). However, this isolation level is enforced +# from the Django database configuration options to prevent possible inconsistencies or malfunctions. +# Changing the isolation level can lead to unexpected behaviors, so please proceed only if you +# what you're doing. Refer to https://docs.djangoproject.com/en/2.2/ref/databases/#mysql-isolation-level +# to get further information. +EDXAPP_MYSQL_OPTIONS: + isolation_level: "read committed" EDXAPP_MYSQL_ATOMIC_REQUESTS: True EDXAPP_MYSQL_REPLICA_DB_NAME: "{{ EDXAPP_MYSQL_DB_NAME }}" EDXAPP_MYSQL_REPLICA_USER: "{{ EDXAPP_MYSQL_USER }}" From c174c005effd69b1a6e9e1a7b86749c4632857d3 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Wed, 1 Sep 2021 01:13:50 +0500 Subject: [PATCH 152/664] fix: add check to switch bwtween py35 and py38 requirements --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index ffe6c0f9d31..2d7017f2074 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1690,7 +1690,7 @@ base_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/base.txt" django_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/django.txt" openstack_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/openstack.txt" -sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/py38.txt" +sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/{% if edxapp_sandbox_python_version == 'python3.5' %}py35.txt{% else %}py38.txt{% endif %}" # The Python requirements files in the order they should be installed. This order should # match the order of PYTHON_REQ_FILES in edx-platform/pavelib/prereqs.py. From 5f27265926e6f6791980fcf4e8803083864ad36b Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 1 Sep 2021 09:05:37 -0400 Subject: [PATCH 153/664] fix: Rename PROFILE_IMAGE_SECRET_KEY to PROFILE_IMAGE_HASH_SEED. We made this change in edx-platform to clarify what this setting was used for, but we did not update it in configuration. This change should fix this. --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- playbooks/sample_vars/passwords.yml | 2 +- playbooks/sample_vars/server_vars.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index b9fe413e56d..4b88a693e88 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -796,7 +796,7 @@ EDXAPP_PROFILE_IMAGE_BACKEND: EDXAPP_PROFILE_IMAGE_MAX_AGE: 31536000 # used to salt hashed usernames for profile image filenames -EDXAPP_PROFILE_IMAGE_SECRET_KEY: placeholder_secret_key +EDXAPP_PROFILE_IMAGE_HASH_SEED: placeholder_secret_key # In bytes EDXAPP_PROFILE_IMAGE_MAX_BYTES: 1048576 @@ -1515,7 +1515,7 @@ lms_auth_config: GOOGLE_ANALYTICS_LINKEDIN: "{{ EDXAPP_GOOGLE_ANALYTICS_LINKEDIN }}" GOOGLE_ANALYTICS_TRACKING_ID: "{{ EDXAPP_GOOGLE_ANALYTICS_TRACKING_ID }}" TRACKING_SEGMENTIO_WEBHOOK_SECRET: "{{ EDXAPP_TRACKING_SEGMENTIO_WEBHOOK_SECRET }}" - PROFILE_IMAGE_SECRET_KEY: "{{ EDXAPP_PROFILE_IMAGE_SECRET_KEY }}" + PROFILE_IMAGE_HASH_SEED: "{{ EDXAPP_PROFILE_IMAGE_HASH_SEED }}" MODULESTORE: default: ENGINE: 'xmodule.modulestore.mixed.MixedModuleStore' diff --git a/playbooks/sample_vars/passwords.yml b/playbooks/sample_vars/passwords.yml index 56436d30872..ba574030254 100644 --- a/playbooks/sample_vars/passwords.yml +++ b/playbooks/sample_vars/passwords.yml @@ -67,7 +67,7 @@ DISCOVERY_SECRET_KEY: !!null #SECRET_KEY ECOMMERCE_SECRET_KEY: !!null #SECRET_KEY EDX_NOTES_API_SECRET_KEY: !!null #SECRET_KEY EDXAPP_EDXAPP_SECRET_KEY: !!null #SECRET_KEY -EDXAPP_PROFILE_IMAGE_SECRET_KEY: !!null #SECRET_KEY +EDXAPP_PROFILE_IMAGE_HASH_SEED: !!null #SECRET_KEY INSIGHTS_SECRET_KEY: !!null #SECRET_KEY MONGO_USERS: diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index ed3af4009c5..8c8d5da8f1c 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -131,7 +131,7 @@ # custom_domain: yourcloudfrontdomain.cloudfront.net # headers: # Cache-Control: max-age-{{ EDXAPP_PROFILE_IMAGE_MAX_AGE }} -#EDXAPP_PROFILE_IMAGE_SECRET_KEY: "SECRET KEY HERE" +#EDXAPP_PROFILE_IMAGE_HASH_SEED: "SECRET KEY HERE" # ##TODO: remove once ansible_provision.sh stops sucking or is burned to the ground #EDXAPP_PROFILE_IMAGE_MAX_AGE: 31536000 From 7441437d1002f57502ea261f1f7415d70d71f2e6 Mon Sep 17 00:00:00 2001 From: Arbab Khalil Date: Wed, 1 Sep 2021 18:49:43 +0500 Subject: [PATCH 154/664] feat: PSRE-974 | Update Data Czar scripts from boto to boto3 (#6537) * PSRE-974 | Update create data czar scripts from boto to boto3 --- .../roles/aws/templates/requirements.txt.j2 | 6 ++--- requirements.txt | 8 +++---- requirements/jenkins.in | 1 + util/create_data_czar/create_data_czar.py | 16 +++++++------- .../create_org_data_czar_policy.py | 22 +++++++++---------- util/jenkins/requirements.txt | 8 ++++--- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 03d408ef620..65fe45fb372 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.20.27 +awscli==1.20.31 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.18.27 +boto3==1.18.31 # via -r requirements/aws.in -botocore==1.21.27 +botocore==1.21.31 # via # awscli # boto3 diff --git a/requirements.txt b/requirements.txt index 8e63c9026a4..3096740f5e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.8.20 # via -r requirements/base.in -awscli==1.20.27 +awscli==1.20.31 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,9 +14,9 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.18.27 +boto3==1.18.31 # via -r requirements/base.in -botocore==1.21.27 +botocore==1.21.31 # via # awscli # boto3 @@ -32,7 +32,7 @@ charset-normalizer==2.0.4 # via requests colorama==0.4.3 # via awscli -cryptography==3.4.7 +cryptography==3.4.8 # via # ansible # paramiko diff --git a/requirements/jenkins.in b/requirements/jenkins.in index cee5a2e27c6..c155d37fa1a 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -11,5 +11,6 @@ celery==3.1.25 click==6.7 opsgenie-sdk==0.3.1 PyMySQL==0.9.3 +python-gnupg redis==2.10.6 splunk-sdk==1.6.6 diff --git a/util/create_data_czar/create_data_czar.py b/util/create_data_czar/create_data_czar.py index 6a75679ae0f..a187f466e17 100755 --- a/util/create_data_czar/create_data_czar.py +++ b/util/create_data_czar/create_data_czar.py @@ -1,4 +1,4 @@ -import boto +import boto3 import argparse import gnupg @@ -20,25 +20,25 @@ import_result = gpg.import_keys(key_data) # Connect to AWS and create account -iam = boto.connect_iam() +iam = boto3.client('iam') if not args.credentials_only: - user_response = iam.create_user(args.user) + user_response = iam.create_user(UserName=args.user) -key_response = iam.create_access_key(args.user) +key_response = iam.create_access_key(UserName=args.user) # Add user to group edx-s3bucket-course-data-readonly -iam.add_user_to_group('edx-s3bucket-course-data-readonly', args.user) +iam.add_user_to_group(GroupName='edx-s3bucket-course-data-readonly', UserName=args.user) # Add user to it's respective Org if args.orgs: for org in args.orgs: user_org = 'edx-course-data-' + org.lower() - iam.add_user_to_group(user_org, args.user) + iam.add_user_to_group(GroupName=user_org, UserName=args.user) # Create AWS Cred String -key = key_response.create_access_key_response.create_access_key_result.access_key -credstring = str('AWS_ACCESS_KEY_ID = ' + key.access_key_id + '\n' + 'AWS_SECRET_ACCESS_KEY = ' + key.secret_access_key) +key = key_response['AccessKey'] +credstring = str(f'AWS_ACCESS_KEY_ID = {key["AccessKeyId"]} \nAWS_SECRET_ACCESS_KEY = {key["SecretAccessKey"]}') # Encrypt file encrypted_data = gpg.encrypt(credstring, args.user, always_trust=True) diff --git a/util/create_data_czar/create_org_data_czar_policy.py b/util/create_data_czar/create_org_data_czar_policy.py index a1894565346..34794681c71 100755 --- a/util/create_data_czar/create_org_data_czar_policy.py +++ b/util/create_data_czar/create_org_data_czar_policy.py @@ -7,7 +7,8 @@ """ import argparse -import boto +import boto3 +from botocore.exceptions import ClientError from string import Template import sys @@ -41,18 +42,21 @@ def add_org_group(org, iam_connection): group_name = "edx-course-data-{org}".format(org=org) try: - iam_connection.create_group(group_name) - except boto.exception.BotoServerError as bse: - if bse.status == 409: + iam_connection.create_group(GroupName=group_name) + except ClientError as bse: + if bse.response['ResponseMetadata']['HTTPStatusCode'] == 409: pass else: print(bse) try: iam_connection.put_group_policy( - group_name,group_name,template.substitute(org=org)) + GroupName=group_name, + PolicyName=group_name, + PolicyDocument=template.substitute(org=org) + ) except boto.exception.BotoServerError as bse: - if bse.status == 409: + if bse.response['ResponseMetadata']['HTTPStatusCode'] == 409: pass else: print(bse) @@ -67,13 +71,9 @@ def add_org_group(org, iam_connection): group.add_argument('-f', '--file', help='The path to a file containing one org name ' 'per line.') -parser.add_argument('-p', '--profile', help='The IAM profile to use when creating ' - 'the group and policy.') - args = parser.parse_args() -iam_connection = boto.connect_iam(profile_name=args.profile) - +iam_connection = boto3.client('iam') if args.org: add_org_group(args.org.rstrip('\n').lower(), iam_connection) elif args.file: diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 9e6341a9f1b..e7f44aac5ae 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.20.27 +awscli==1.20.31 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,9 +16,9 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.18.27 +boto3==1.18.31 # via -r requirements/jenkins.in -botocore==1.21.27 +botocore==1.21.31 # via # awscli # boto3 @@ -56,6 +56,8 @@ python-dateutil==2.8.2 # botocore # opsgenie-sdk # s3cmd +python-gnupg==0.4.7 + # via -r requirements/jenkins.in python-magic==0.4.24 # via s3cmd pytz==2021.1 From ab20dcc4ae57cdd745f476136edcde7df17d8981 Mon Sep 17 00:00:00 2001 From: Manjinder Singh <49171515+jinder1s@users.noreply.github.com> Date: Wed, 1 Sep 2021 12:18:21 -0400 Subject: [PATCH 155/664] feat: Studio uses oauth2 to login in our sandbox environments (#6532) Settings to enable oauth2 loging through lms for studio in our sandbox environments Other settings that were needed to be added to django settings: SOCIAL_AUTH_EDX_OAUTH2_ISSUER SOCIAL_AUTH_EDX_OAUTH2_KEY SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL SOCIAL_AUTH_EDX_OAUTH2_SECRET SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT LOGIN_URL: /login/ --- playbooks/roles/edxapp/defaults/main.yml | 15 +++++++++++++++ .../roles/oauth_client_setup/defaults/main.yml | 10 ++++++++++ util/jenkins/ansible-provision.sh | 1 + 3 files changed, 26 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index b9fe413e56d..1dd8efc87b0 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -235,6 +235,12 @@ EDXAPP_THIRD_PARTY_AUTH_BACKENDS: - common.djangoapps.third_party_auth.saml.SAMLAuthBackend - common.djangoapps.third_party_auth.lti.LTIAuthBackend +EDXAPP_CMS_SERVICE_NAME: 'edxapp-cms' +EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_KEY: 'edxapp-cms-sso-key' +EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET: 'edxapp-cms-sso-secret' +EDXAPP_CMS_BACKEND_SERVICE_EDX_OAUTH2_KEY: 'edxapp-cms-backend-service-key' +EDXAPP_CMS_BACKEND_SERVICE_EDX_OAUTH2_SECRET: 'edxapp-cms-backend-service-secret' + EDXAPP_ENABLE_MOBILE_REST_API: false EDXAPP_ENABLE_BULK_ENROLLMENT_VIEW: false @@ -947,6 +953,11 @@ DISCOVERY_SERVICE_USER_NAME: "discovery_worker" ECOMMERCE_SERVICE_USER_EMAIL: "ecommerce_worker@example.com" ECOMMERCE_SERVICE_USER_NAME: "ecommerce_worker" +# Configuration needed for LMS to communicate with the Studio service +EDXAPP_CMS_SERVICE_USER_EMAIL: "edxapp_cms_worker@example.com" +EDXAPP_CMS_SERVICE_USER_NAME: "edxapp_cms_worker" + + # Configuration needed for LMS to communicate with the Credentials service CREDENTIALS_SERVICE_USER_EMAIL: "credentials_worker@example.com" CREDENTIALS_SERVICE_USER_NAME: "credentials_worker" @@ -1786,6 +1797,10 @@ SERVICE_WORKER_USERS: username: "{{ REGISTRAR_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ EDXAPP_CMS_SERVICE_USER_EMAIL }}" + username: "{{ EDXAPP_CMS_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ LICENSE_MANAGER_SERVICE_USER_EMAIL }}" username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME }}" is_staff: true diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 8bedb458abb..5d0e583479e 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -28,6 +28,16 @@ oauth_client_setup_oauth2_clients: logout_uri: "{{ ECOMMERCE_LOGOUT_URL | default('None') }}", username: "{{ ECOMMERCE_SERVICE_USER_NAME | default('None') }}", } + - { + name: "{{ EDXAPP_CMS_SERVICE_NAME | default('None') }}", + url_root: "{{ EDXAPP_CMS_URL_ROOT | default('None') }}", + sso_id: "{{ EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_KEY | default('None') }}", + sso_secret: "{{ EDXAPP_CMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET | default('None') }}", + backend_service_id: "{{ EDXAPP_CMS_BACKEND_SERVICE_EDX_OAUTH2_KEY | default('None') }}", + backend_service_secret: "{{ EDXAPP_CMS_BACKEND_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", + logout_uri: "{{ EDXAPP_CMS_LOGOUT_URL | default('None') }}", + username: "{{ EDXAPP_CMS_SERVICE_USER_NAME | default('None') }}", + } - { name: "{{ INSIGHTS_OAUTH2_APP_CLIENT_NAME | default('None') }}", url_root: "{{ INSIGHTS_BASE_URL | default('None') }}", diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 56aac3cf8b1..616e17f94f3 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -396,6 +396,7 @@ if [[ $edx_internal == "true" ]]; then EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host} EDXAPP_LMS_BASE: ${deploy_host} EDXAPP_CMS_BASE: studio-${deploy_host} +EDXAPP_CMS_URL_ROOT: "https://{{ EDXAPP_CMS_BASE }}" EDXAPP_SITE_NAME: ${deploy_host} edx_internal: True COMMON_USER_INFO: From 7f9c3a4913d1c4c27912bb7e083c285862b45bb1 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 8 Sep 2021 14:56:56 -0400 Subject: [PATCH 156/664] feat: remove unused single course url feature flag from prospectus configuration (#6544) --- playbooks/roles/prospectus/defaults/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 6f67cacbbf9..2681efde7d4 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -49,8 +49,6 @@ prospectus_env_vars: PATH: "{{ prospectus_nodeenv_bin }}:{{ prospectus_venv_dir }}/bin:{{ ansible_env.PATH }}" NODE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" ACTIVE_ENV: "{{ PROSPECTUS_ENVIRONMENT }}" - USE_COURSE_URL_SLUGS: "{{ PROSPECTUS_USE_COURSE_URL_SLUGS }}" - GATSBY_USE_COURSE_URL_SLUGS: "{{ PROSPECTUS_USE_COURSE_URL_SLUGS }}" GATSBY_CONTENTFUL_SPACE_ID: "{{ PROSPECTUS_CONTENTFUL_SPACE_ID }}" GATSBY_CONTENTFUL_ACCESS_TOKEN: "{{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }}" GATSBY_CSRF_TOKEN_API_PATH: '/csrf/api/v1/token' From d85d945a3c0a3ba505fe342a714f194d8919b15d Mon Sep 17 00:00:00 2001 From: Arbab Khalil Date: Mon, 13 Sep 2021 18:55:18 +0500 Subject: [PATCH 157/664] feat: Add user tagging in create_data_czar.py | PSRE-1038 (#6546) --- util/create_data_czar/create_data_czar.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/create_data_czar/create_data_czar.py b/util/create_data_czar/create_data_czar.py index a187f466e17..bf3c102e6c3 100755 --- a/util/create_data_czar/create_data_czar.py +++ b/util/create_data_czar/create_data_czar.py @@ -12,6 +12,7 @@ parser.add_argument('-f', '--file', help='Public Key file', required=True) parser.add_argument('--credentials-only', help='Only create new credentials', default=False, action='store_true') parser.add_argument('-o', '--orgs', nargs='*', help='Name of the org(s) as list, User need to be a member', default=None) +parser.add_argument('-c', '--creator', help='Name of the creator', default=None) args = parser.parse_args() # Import Data Czar GPG Key @@ -24,6 +25,8 @@ if not args.credentials_only: user_response = iam.create_user(UserName=args.user) + if args.creator: + tag_response = iam.tag_user(UserName=args.user, Tags=[{'Key': 'Creator', 'Value': args.creator}]) key_response = iam.create_access_key(UserName=args.user) From c5df224e403ce2ea59dc20e28930ae35b8d451fc Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Thu, 16 Sep 2021 14:15:17 +0500 Subject: [PATCH 158/664] Feat: Add group and user management tasks for other IDAs (#6547) * Add group and user management tasks for other IDAs --- playbooks/manage_edxapp_users_and_groups.yml | 60 +++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 170421e99a9..0768991473c 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -115,6 +115,7 @@ ignore_user_creation_errors: no deployment_settings: "{{ EDXAPP_SETTINGS | default('production') }}" group_environment: "" # By default, create groups for all envs (for backwards compatibility). + service: "" # Used to display the service name during execution with *-ida tags. Set using ansible-playbook -e. vars_files: - roles/common_vars/defaults/main.yml tasks: @@ -147,7 +148,7 @@ - name: Manage active LMS/CMS users tags: - - manage-active-users + - manage-active-users-edxapp shell: > . {{env_path}} && {{ python_path }} {{ manage_path }} lms --settings={{ deployment_settings }} manage_user {{ item.username | quote }} {{ item.email | quote }} @@ -168,7 +169,7 @@ - name: Manage inactive LMS/CMS users tags: - - manage-inactive-users + - manage-inactive-users-edxapp shell: > . {{env_path}} && {{ python_path }} {{ manage_path }} lms --settings={{ deployment_settings }} manage_user {{ item.username | quote }} {{ item.email | quote }} @@ -186,3 +187,58 @@ until: manage_users_result is not failed become: true become_user: "{{ common_web_user }}" + + - name: Manage {{ service }} groups + tags: + - manage-groups-ida + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + manage_group {{ item.name | quote }} + {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + with_items: "{{ django_groups }}" + when: (not group_environment) or group_environment in item.environments + become: true + become_user: "{{ common_web_user }}" + + - name: Manage active {{ service }} users + tags: + - manage-active-users-ida + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + manage_user {{ item.username | quote }} {{ item.email | quote }} + {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + {% if item.get('superuser') %}--superuser{% endif %} + {% if item.get('staff') %}--staff{% endif %} + {% if item.get('unusable_password') %}--unusable-password{% endif %} + {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} + with_items: "{{ django_users }}" + when: not item.get('unusable_password') + register: manage_users_result + failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) + retries: 3 + until: manage_users_result is not failed + become: true + become_user: "{{ common_web_user }}" + + - name: Manage inactive {{ service }} users + tags: + - manage-inactive-users-ida + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + manage_user {{ item.username | quote }} {{ item.email | quote }} + {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + {% if item.get('superuser') %}--superuser{% endif %} + {% if item.get('staff') %}--staff{% endif %} + {% if item.get('unusable_password') %}--unusable-password{% endif %} + {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} + with_items: "{{ django_users }}" + when: item.get('unusable_password') + register: manage_users_result + failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) + retries: 3 + until: manage_users_result is not failed + become: true + become_user: "{{ common_web_user }}" From 0ddb2e1bf376fad4cc94c1df59ef0609d7f29176 Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Thu, 16 Sep 2021 14:39:46 +0500 Subject: [PATCH 159/664] Remove --settings from IDA tasks for user and group management --- playbooks/manage_edxapp_users_and_groups.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 0768991473c..a0dd9c0abf2 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -192,7 +192,7 @@ tags: - manage-groups-ida shell: > - . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + . {{env_path}} && {{ python_path }} {{ manage_path }} manage_group {{ item.name | quote }} {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} @@ -205,7 +205,7 @@ tags: - manage-active-users-ida shell: > - . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + . {{env_path}} && {{ python_path }} {{ manage_path }} manage_user {{ item.username | quote }} {{ item.email | quote }} {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} @@ -226,7 +226,7 @@ tags: - manage-inactive-users-ida shell: > - . {{env_path}} && {{ python_path }} {{ manage_path }} --settings={{ deployment_settings }} + . {{env_path}} && {{ python_path }} {{ manage_path }} manage_user {{ item.username | quote }} {{ item.email | quote }} {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} From 0c815170fef59855f2ced35eccb8a52b3adce1c9 Mon Sep 17 00:00:00 2001 From: "M. Zulqarnain" Date: Thu, 16 Sep 2021 14:59:42 +0500 Subject: [PATCH 160/664] feat: update xblock-invideo-quiz --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 6cbd06124d8..59c4963a832 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -567,7 +567,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - name: git+https://github.com/edx/xblock-qualtrics-survey.git@d3d2773bcf453d14a0626f813911dc308ddce572#egg=xblock_qualtrics_survey extra_args: -e - - name: git+https://github.com/edx/xblock-in-video-quiz.git@c1cc11f87285cd885d76604145433dae87434a6d#egg=invideoquiz-xblock + - name: git+https://github.com/edx/xblock-in-video-quiz.git@8e820b59df2ad0f2f0531c909199d76d536c6100#egg=invideoquiz-xblock extra_args: -e - name: git+https://github.com/edx/xblock-submit-and-compare@28751ed01c4d4ae5f10f9ddee710e76f8b8ae640#egg=xblock-submit-and-compare extra_args: -e From 3fd6b28f3cba99496eae88b21f79be2702ad4ea7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 17 Sep 2021 14:51:53 +0500 Subject: [PATCH 161/664] fix: update script exit status with gpg STDERR --- util/create_data_czar/create_data_czar.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/create_data_czar/create_data_czar.py b/util/create_data_czar/create_data_czar.py index bf3c102e6c3..54f3d30e7b2 100755 --- a/util/create_data_czar/create_data_czar.py +++ b/util/create_data_czar/create_data_czar.py @@ -1,6 +1,7 @@ import boto3 import argparse import gnupg +import sys # Assumes you have GPG already installed # Assumes that the Data Czars already have your public key @@ -52,3 +53,6 @@ print('ok: ', encrypted_data.ok) print('status: ', encrypted_data.status) print('stderr: ', encrypted_data.stderr) + +if encrypted_data.stderr: + sys.exit(1) From 72165b7d40c20e321218785438203da028222ed6 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 17 Sep 2021 15:19:51 +0500 Subject: [PATCH 162/664] fix data czar script failure --- util/create_data_czar/create_data_czar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/create_data_czar/create_data_czar.py b/util/create_data_czar/create_data_czar.py index 54f3d30e7b2..74898975b14 100755 --- a/util/create_data_czar/create_data_czar.py +++ b/util/create_data_czar/create_data_czar.py @@ -54,5 +54,5 @@ print('status: ', encrypted_data.status) print('stderr: ', encrypted_data.stderr) -if encrypted_data.stderr: +if 'error' in encrypted_data.stderr: sys.exit(1) From c563fec7a1febfe7118ada43aca6c8820b7abfc2 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 20 Sep 2021 11:53:47 +0500 Subject: [PATCH 163/664] fix: It was removed accidentally. Adding it back to run django related packages. --- playbooks/roles/edxapp/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 6cbd06124d8..2534ba9e030 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1715,6 +1715,7 @@ sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/{% i edxapp_requirements_files: - "{{ custom_requirements_file }}" - "{{ base_requirements_file }}" + - "{{ django_requirements_file }}" # All edxapp requirements files potentially containing Github URLs. When using a custom # Github mirror, occurrences of "github.com" are replaced by the custom mirror in these From 6d485bd71c54caad05100880d7ae9f080fa37598 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 23 Sep 2021 13:52:44 -0400 Subject: [PATCH 164/664] fix: sandbox termination date reporting via message of the day i.e. the message that gets spit out when you ssh in to such a sandbox appears to've been broken by the 20.04 upgrade which moved system python into 3.whatever --- playbooks/roles/edx-sandbox/tasks/main.yml | 2 +- .../etc/update-motd.d/{temiate_motd.j2 => terminate_motd.j2} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename playbooks/roles/edx-sandbox/templates/etc/update-motd.d/{temiate_motd.j2 => terminate_motd.j2} (91%) diff --git a/playbooks/roles/edx-sandbox/tasks/main.yml b/playbooks/roles/edx-sandbox/tasks/main.yml index 61fc47d62a8..c83ac00f808 100644 --- a/playbooks/roles/edx-sandbox/tasks/main.yml +++ b/playbooks/roles/edx-sandbox/tasks/main.yml @@ -25,7 +25,7 @@ - name: update the termination date and time as motd template: dest: "/etc/update-motd.d/999-terminate-sandbox" - src: "etc/update-motd.d/temiate_motd.j2" + src: "etc/update-motd.d/terminate_motd.j2" mode: 0755 owner: root group: root diff --git a/playbooks/roles/edx-sandbox/templates/etc/update-motd.d/temiate_motd.j2 b/playbooks/roles/edx-sandbox/templates/etc/update-motd.d/terminate_motd.j2 similarity index 91% rename from playbooks/roles/edx-sandbox/templates/etc/update-motd.d/temiate_motd.j2 rename to playbooks/roles/edx-sandbox/templates/etc/update-motd.d/terminate_motd.j2 index 59f2d6cb90d..5bb84f9ccb0 100644 --- a/playbooks/roles/edx-sandbox/templates/etc/update-motd.d/temiate_motd.j2 +++ b/playbooks/roles/edx-sandbox/templates/etc/update-motd.d/terminate_motd.j2 @@ -10,7 +10,7 @@ instance_id = get_instance_metadata()['instance-id'] reservations = ec2.get_all_instances(instance_ids=[instance_id]) instance = reservations[0].instances[0] -if instance.tags.has_key('instance_termination_time'): +if 'instance_termination_time' in instance.tags: terminate_time = datetime.strptime(str(instance.tags['instance_termination_time']), "%m-%d-%Y %H:%M:%S") else: terminate_time = datetime.strptime(instance.launch_time, "%Y-%m-%dT%H:%M:%S.%fZ") + timedelta(days=7) From f56c5ff6370cd1b71f8c06c17e1366f3fa5fc949 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 27 Sep 2021 11:57:59 -0400 Subject: [PATCH 165/664] fix: remove programtypes created by disco migrations this is done on the creation of a sandbox because these migration-created program types will end up conflicting with the ones we load via fixtures from production data. We want to load these via only one method, since it simplifies each piece of the puzzle. --- playbooks/roles/discovery/defaults/main.yml | 2 ++ playbooks/roles/discovery/meta/main.yml | 1 + util/jenkins/ansible-provision.sh | 7 +++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 6c4387c66ee..5002c787fc7 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -189,6 +189,8 @@ discovery_service_config_overrides: # See edx_django_service_automated_users for an example of what this should be DISCOVERY_AUTOMATED_USERS: {} +DISCOVERY_POST_MIGRATE_COMMANDS: [] + DISCOVERY_CSRF_COOKIE_SECURE: false DISCOVERY_CORS_ORIGIN_WHITELIST: [] diff --git a/playbooks/roles/discovery/meta/main.yml b/playbooks/roles/discovery/meta/main.yml index f0b0446f819..6a2ef4d7838 100644 --- a/playbooks/roles/discovery/meta/main.yml +++ b/playbooks/roles/discovery/meta/main.yml @@ -49,6 +49,7 @@ dependencies: edx_django_service_session_expire_at_browser_close: '{{ DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' edx_django_service_node_version: '{{ DISCOVERY_NODE_VERSION }}' edx_django_service_automated_users: '{{ DISCOVERY_AUTOMATED_USERS }}' + edx_django_service_post_migrate_commands: '{{ DISCOVERY_POST_MIGRATE_COMMANDS }}' edx_django_service_enable_newrelic_distributed_tracing: '{{ DISCOVERY_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}' edx_django_service_decrypt_config_enabled: '{{ DISCOVERY_DECRYPT_CONFIG_ENABLED }}' edx_django_service_copy_config_enabled: '{{ DISCOVERY_COPY_CONFIG_ENABLED }}' diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 616e17f94f3..68b3c459e10 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -543,8 +543,11 @@ video_pipeline_integration=${video_pipeline:-false} # ansible overrides for master's integration environment setup if [[ $registrar == "true" ]]; then cat << EOF >> $extra_vars_file -COMMON_ENABLE_SPLUNKFORWARDER: true, -EDXAPP_ENABLE_ENROLLMENT_RESET: true, +COMMON_ENABLE_SPLUNKFORWARDER: true +EDXAPP_ENABLE_ENROLLMENT_RESET: true +DISCOVERY_POST_MIGRATE_COMMANDS: + - command: "./manage.py remove_program_types_from_migrations" + when: true EOF fi From a461ef90893f88b495d057bb3ff17afea9ecab4b Mon Sep 17 00:00:00 2001 From: Florian Haas Date: Fri, 24 Sep 2021 09:13:51 +0200 Subject: [PATCH 166/664] Make nginx listen on both the IPv4 and IPv6 wildcard addresses The nginx role always assumes that Open edX services listen on the wildcard address (we can only ever configure ports, not listen addresses). However, if nginx is configured with a "listen" directive that contains just a port number, it will listen only on the IPv4 wildcard address, 0.0.0.0. For IPv6-enabled hosts, that means that Open edX services will not be accessible through the system's IPv6 address or AAAA hostname. Add a variable, NGINX_ENABLE_IPV6, and set its default to True. Conditionally add a listen directive for the IPv6 wildcard address, "[::]". There is a shorthand available for this, which is to use only a listen directive for the IPv6 wildcard and then the "ipv6only=off" option. We cannot use that here because that doesn't work with multiple vhosts, causing nginx to quit with: nginx: [emerg] duplicate listen options for [::]:80 in /etc/nginx/sites-enabled/lms:40 nginx: configuration file /etc/nginx/nginx.conf test failed So, instead inject the additional listen directive specifically for the IPv6 wildcard address. See also: * https://nginx.org/en/docs/http/ngx_http_core_module.html#listen * https://serverfault.com/questions/638367/do-you-need-separate-ipv4-and-ipv6-listen-directives-in-nginx --- CHANGELOG.md | 6 ++++++ playbooks/roles/nginx/defaults/main.yml | 2 ++ .../edx/app/nginx/sites-available/analytics_api.j2 | 3 +++ .../nginx/templates/edx/app/nginx/sites-available/cms.j2 | 6 ++++++ .../templates/edx/app/nginx/sites-available/conductor.j2 | 3 +++ .../edx/app/nginx/sites-available/edx_notes_api.j2 | 6 ++++++ .../templates/edx/app/nginx/sites-available/forum.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/gh_mirror.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/gitreload.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/grafana.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/graphite.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/insights.j2 | 6 ++++++ .../templates/edx/app/nginx/sites-available/jenkins.j2 | 3 +++ .../templates/edx/app/nginx/sites-available/kibana.j2 | 9 +++++++++ .../edx/app/nginx/sites-available/learner_portal.j2 | 6 ++++++ .../nginx/templates/edx/app/nginx/sites-available/lms.j2 | 6 ++++++ .../edx/app/nginx/sites-available/nginx_redirect.j2 | 6 ++++++ .../edx/app/nginx/sites-available/program_console.j2 | 6 ++++++ .../edx/app/nginx/sites-available/prospectus.j2 | 6 ++++++ .../templates/edx/app/nginx/sites-available/xqueue.j2 | 6 ++++++ 20 files changed, 95 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aa3c18e9d6..0c792fac1e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-09-28 + - Role nginx + - Add `NGINX_ENABLE_IPV6` configuration variable to make nginx + services listen on the IPv6 wildcard address (in addition to + the IPv4 one, where services always listen). Defaults to true. + - 2021-09-19 - Remove configuration for edx-certificates, as that repo and service are no longer used. diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 5fd3433f49b..e91862c8d2c 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -36,6 +36,8 @@ PROSPECTUS_PREVIEW_NGINX_USERS: password: "{{ PROSPECTUS_PREVIEW_HTPASSWD_PASS }}" state: present +NGINX_ENABLE_IPV6: True + NGINX_ENABLE_SSL: False NGINX_REDIRECT_TO_HTTPS: False # Disable handling IP disclosure for private IP addresses. This is needed by ELB to run the health checks while using `NGINX_ENABLE_SSL`. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics_api.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics_api.j2 index 7900e0a22ed..ccafcc823aa 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics_api.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/analytics_api.j2 @@ -6,6 +6,9 @@ upstream analytics_api_app_server { server { listen {{ ANALYTICS_API_NGINX_PORT }} default_server; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ ANALYTICS_API_NGINX_PORT }} default_server; + {% endif %} # Nginx does not support nested condition or or conditions so # there is an unfortunate mix of conditonals here. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 index 63857c1d010..c0c94614a45 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 @@ -54,10 +54,16 @@ error_page {{ k }} {{ v }}; {% include "empty_json.j2" %} listen {{ EDXAPP_CMS_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ EDXAPP_CMS_NGINX_PORT }} {{ default_site }}; + {% endif %} {% if NGINX_ENABLE_SSL %} listen {{ EDXAPP_CMS_SSL_NGINX_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ EDXAPP_CMS_SSL_NGINX_PORT }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 index 7a6bd391244..816313e21f5 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 @@ -26,6 +26,9 @@ server { {% endif %} listen {{ CONDUCTOR_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ CONDUCTOR_NGINX_PORT }} {{ default_site }}; + {% endif %} # Redirects using the client port instead of the port the service is running # on. This prevents redirects to the local 8000 port. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2 index 7a1bf487d49..5f94acdbb98 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2 @@ -12,10 +12,16 @@ upstream {{ edx_notes_api_service_name }}_app_server { server { listen {{ edx_notes_api_nginx_port }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ edx_notes_api_nginx_port }} {{ default_site }}; + {% endif %} {% if NGINX_ENABLE_SSL %} listen {{ edx_notes_api_ssl_nginx_port }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ edx_notes_api_ssl_nginx_port }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2 index 12132f04865..718a6e60211 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/forum.j2 @@ -33,6 +33,9 @@ server { server_name forum.*; listen {{ FORUM_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ FORUM_NGINX_PORT }} {{ default_site }}; + {% endif %} client_max_body_size {{ NGINX_FORUM_CLIENT_MAX_BODY_SIZE }}; proxy_read_timeout {{ NGINX_FORUM_PROXY_READ_TIMEOUT }}; keepalive_timeout 5; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gh_mirror.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gh_mirror.j2 index 4449ae02e7d..845992f8704 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gh_mirror.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gh_mirror.j2 @@ -1,5 +1,8 @@ server { listen {{ gh_mirror_nginx_port }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]: {{ gh_mirror_nginx_port }}; + {% endif %} server_name {{ gh_mirror_server_name }}; location ~ (/.*) { root {{ gh_mirror_data_dir }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gitreload.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gitreload.j2 index a9a0fd126b2..86ab6d4e26e 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gitreload.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gitreload.j2 @@ -6,6 +6,9 @@ upstream gitreload_app_server { server { listen {{ GITRELOAD_NGINX_PORT }} default_server; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ GITRELOAD_NGINX_PORT }} default_server; + {% endif %} location / { auth_basic "Restricted"; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/grafana.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/grafana.j2 index 78d0394b3d2..33cb6a1a38c 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/grafana.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/grafana.j2 @@ -15,6 +15,9 @@ upstream grafana_app_server { server { server_name grafana.*; listen {{ GRAFANA_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ GRAFANA_NGINX_PORT }} {{ default_site }}; + {% endif %} client_max_body_size 1M; keepalive_timeout 5; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2 index 0ef811c3e00..cd94520db48 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2 @@ -22,6 +22,9 @@ upstream graphite_app_server { server { server_name graphite.*; listen {{ GRAPHITE_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ GRAPHITE_NGINX_PORT }} {{ default_site }}; + {% endif %} client_max_body_size 1M; keepalive_timeout 5; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2 index ac5a6fc0b6c..8bef747fd9b 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2 @@ -19,9 +19,15 @@ map $http_origin $cors_origin { server { listen {{ INSIGHTS_NGINX_PORT }} default_server; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ INSIGHTS_NGINX_PORT }} default_server; + {% endif %} {% if NGINX_ENABLE_SSL %} listen {{ INSIGHTS_NGINX_SSL_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ INSIGHTS_NGINX_SSL_PORT }} ssl; + {% endif %} {% include "common-settings.j2" %} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 index d783c034c70..b130b5e71b8 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 @@ -1,5 +1,8 @@ server { listen {{ jenkins_nginx_port }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ jenkins_nginx_port }}; + {% endif %} server_name {{ jenkins_server_name }}; location / { diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2 index 6ef6ede2ec5..c90fafc6ba1 100755 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/kibana.j2 @@ -14,13 +14,22 @@ server { {% if NGINX_ENABLE_SSL %} listen {{ KIBANA_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ KIBANA_NGINX_PORT }} {{ default_site }}; + {% endif %} listen {{ KIBANA_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ KIBANA_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; {% else %} listen {{ KIBANA_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ KIBANA_NGINX_PORT }} {{ default_site }}; + {% endif %} {% endif %} {% include "handle-tls-redirect-and-ip-disclosure.j2" %} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/learner_portal.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/learner_portal.j2 index 41bbd35afbb..84c79da639e 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/learner_portal.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/learner_portal.j2 @@ -6,6 +6,9 @@ server { listen {{ LEARNER_PORTAL_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ LEARNER_PORTAL_NGINX_PORT }} {{ default_site }}; + {% endif %} server_name ~^((stage|prod)-)?learner-portal.*; location / { root /edx/app/learner_portal/learner_portal/dist; @@ -15,6 +18,9 @@ server { server { listen {{ LEARNER_PORTAL_SSL_NGINX_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ LEARNER_PORTAL_SSL_NGINX_PORT }} ssl; + {% endif %} server_name ~^((stage|prod)-)?learner-portal.*; ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 index d514348e984..e40dedd8bdc 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 @@ -87,9 +87,15 @@ error_page {{ k }} {{ v }}; {% include "empty_json.j2" %} listen {{ EDXAPP_LMS_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ EDXAPP_LMS_NGINX_PORT }} {{ default_site }}; + {% endif %} {% if NGINX_ENABLE_SSL %} listen {{ EDXAPP_LMS_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ EDXAPP_LMS_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/nginx_redirect.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/nginx_redirect.j2 index d5f4e50be1c..b01fbc123d0 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/nginx_redirect.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/nginx_redirect.j2 @@ -6,9 +6,15 @@ server { listen {{ REDIRECT_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ REDIRECT_NGINX_PORT }} {{ default_site }}; + {% endif %} {% if "ssl" in item.value and item.value['ssl'] == true -%} listen {{ REDIRECT_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ REDIRECT_SSL_NGINX_PORT }} {{ default_site }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; {% endif -%} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/program_console.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/program_console.j2 index 3f835c01deb..775ecedf8a4 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/program_console.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/program_console.j2 @@ -6,6 +6,9 @@ server { listen {{ PROGRAM_CONSOLE_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ PROGRAM_CONSOLE_NGINX_PORT }} {{ default_site }}; + {% endif %} server_name ~^((stage|prod)-)?program-console.*; {% include "handle-tls-redirect-and-ip-disclosure.j2" %} @@ -18,6 +21,9 @@ server { server { listen {{ PROGRAM_CONSOLE_SSL_NGINX_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ PROGRAM_CONSOLE_SSL_NGINX_PORT }} ssl; + {% endif %} server_name ~^((stage|prod)-)?program-console.*; ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 6837119a850..7243dce5e77 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -14,6 +14,9 @@ server { {% if NGINX_ENABLE_SSL %} listen {{ prospectus_ssl_nginx_port }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ prospectus_ssl_nginx_port }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; @@ -30,6 +33,9 @@ server { {% endif %} listen {{ PROSPECTUS_NGINX_PORT }} {{ default_site }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ PROSPECTUS_NGINX_PORT }} {{ default_site }}; + {% endif %} root {{ PROSPECTUS_DATA_DIR }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/xqueue.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/xqueue.j2 index e521127c1eb..28d96272dd4 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/xqueue.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/xqueue.j2 @@ -6,9 +6,15 @@ upstream xqueue_app_server { server { listen {{ XQUEUE_NGINX_PORT }} default_server; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ XQUEUE_NGINX_PORT }} default_server; + {% endif %} {% if NGINX_ENABLE_SSL %} listen {{ XQUEUE_NGINX_SSL_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ XQUEUE_NGINX_SSL_PORT }} ssl; + {% endif %} {% include "common-settings.j2" %} From 968d34ae2a179fac67023b23183a68133d86dd8b Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 29 Sep 2021 12:24:07 -0400 Subject: [PATCH 167/664] fix: restore admin access to default user on master's sandboxes JIRA:EDUCATOR-5825 --- util/jenkins/ansible-provision.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 68b3c459e10..9063a1b3aec 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -548,6 +548,19 @@ EDXAPP_ENABLE_ENROLLMENT_RESET: true DISCOVERY_POST_MIGRATE_COMMANDS: - command: "./manage.py remove_program_types_from_migrations" when: true + - command: > + ./manage.py create_superuser + --username="admin" + --email="admin@example.com" + --no-input + when: true +registrar_post_migrate_commands: + - command: > + ./manage.py create_superuser + --username="admin" + --email="admin@example.com" + --no-input + when: true EOF fi From 1b1728e37fc2107077e9d018391a10f73c62824a Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 29 Sep 2021 15:12:35 -0400 Subject: [PATCH 168/664] fix: bad management command name for admin user creation for master's integration environments --- util/jenkins/ansible-provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 9063a1b3aec..8124c0667df 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -549,14 +549,14 @@ DISCOVERY_POST_MIGRATE_COMMANDS: - command: "./manage.py remove_program_types_from_migrations" when: true - command: > - ./manage.py create_superuser + ./manage.py createsuperuser --username="admin" --email="admin@example.com" --no-input when: true registrar_post_migrate_commands: - command: > - ./manage.py create_superuser + ./manage.py createsuperuser --username="admin" --email="admin@example.com" --no-input From dbca0188463920769e3e7619dda744e2806956b5 Mon Sep 17 00:00:00 2001 From: "zia.fazal@arbisoft.com" Date: Thu, 30 Sep 2021 12:00:29 +0500 Subject: [PATCH 169/664] edx-event-routing-backends library transforms edX events according to caliper and xAPI specifications and route them to specifies LRS Fixed typo added version --- playbooks/roles/edxapp/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 2534ba9e030..3a0acc60827 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -583,6 +583,8 @@ EDXAPP_PRIVATE_REQUIREMENTS: # "Pathways" learning context plugin for the LabXchange project - name: git+https://github.com/open-craft/lx-pathway-plugin.git@337abf249b7c5ecc1e78a44d2e639e1ab65f2085#egg=lx-pathway-plugin extra_args: -e + # Caliper and xAPI event routing plugin + - name: edx-event-routing-backends==4.1.1 # List of custom middlewares that should be used in edxapp to process # incoming HTTP resquests. Should be a list of plain strings that fully From 0ec714a18e45d0ee467585e3633bd6bef871f261 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 27 Sep 2021 17:29:02 +0500 Subject: [PATCH 170/664] feat: Added kustomize checker script --- .../kustomize_update_checker.py | 129 ++++++++++++++++++ .../kustomize_update_checker/requirements.txt | 1 + 2 files changed, 130 insertions(+) create mode 100644 util/jenkins/kustomize_update_checker/kustomize_update_checker.py create mode 120000 util/jenkins/kustomize_update_checker/requirements.txt diff --git a/util/jenkins/kustomize_update_checker/kustomize_update_checker.py b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py new file mode 100644 index 00000000000..88910255049 --- /dev/null +++ b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +import subprocess +import yaml +import sys +import logging +import click +import boto3 +import backoff +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger(__name__) +logging.basicConfig() +global_list = [] +MAX_TRIES = 5 + + +class SESBotoWrapper: + def __init__(self, **kwargs): + self.client = boto3.client("ses", **kwargs) + + @backoff.on_exception(backoff.expo, + ClientError, + max_tries=MAX_TRIES) + def send_email(self, *args, **kwargs): + return self.client.send_email(*args, **kwargs) + + +def parse_yaml(file_name): + with open(file_name, 'r') as stream: + try: + apps_details = yaml.safe_load(stream) + for item in apps_details: + registery = apps_details[item]["registery"] + repo = apps_details[item]["repo"] + url = "https://" + registery + "/v1/repositories/" + repo + "/tags" + sed_filter = "sed -e 's/[][]//g' -e 's/\"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' | tail -1" + cmd = "wget -q " + url + " -O - | " + sed_filter + latest_version = subprocess.check_output(cmd, shell=True).strip().decode("utf-8") + check_version(item, apps_details[item]["version"], latest_version) + + except yaml.YAMLError as exc: + LOGGER.error("error in configuration file: %s" % str(exc)) + sys.exit(1) + except KeyError as e: + print('I got a KeyError - reason "%s"' % str(e)) + + +def check_version(app_name, app_version, latest_version): + if not compare_version(app_version, latest_version): + temp_dict = { + app_name: { + "current_version": app_version, + "latest_version": latest_version + }, + } + global_list.append(temp_dict) + + +def compare_version(current_version, latest_version): + if current_version == latest_version: + return True + return False + + +def send_an_email(to_addr, from_addr, app_list, region): + ses_client = SESBotoWrapper(region_name=region) + + message = """ +

Hello,

+

Updates are available for the following kustomize based apps

+ + + + + + + """ + for apps in app_list: + for data in apps: + message += """ + + + + """.format( + AppName=data, + CurrentVersion=apps[data]["current_version"], + LatestVersion=apps[data]["latest_version"] + ) + + message += """
App NameCurrent versionLatest Version
{AppName}{CurrentVersion}{LatestVersion}
""" + print(("Sending the following as email to {}".format(to_addr))) + print(message) + ses_client.send_email( + Source=from_addr, + Destination={ + 'ToAddresses': [ + to_addr + ] + }, + Message={ + 'Subject': { + 'Data': 'Updates available for kustomize based apps', + 'Charset': 'utf-8' + }, + 'Body': { + 'Html':{ + 'Data': message, + 'Charset': 'utf-8' + } + } + } + ) + + +@click.command() +@click.option('--file-name', required=True, help='Filename which have kustomize based apps details.') +@click.option('--file-path', required=True, help='File path where kustomize based apps file exists.') +@click.option('--region', multiple=True, help='Default AWS region') +@click.option('--recipient', multiple=True, help='Recipient Email address') +@click.option('--sender', multiple=True, help='Sender email address') +def controller(file_name, file_path, region, recipient, sender): + parse_yaml(file_name) + if len(global_list) > 0: + send_an_email(recipient[0], sender[0], global_list, region[0]) + + +if __name__ == "__main__": + controller() diff --git a/util/jenkins/kustomize_update_checker/requirements.txt b/util/jenkins/kustomize_update_checker/requirements.txt new file mode 120000 index 00000000000..dc833dd4bef --- /dev/null +++ b/util/jenkins/kustomize_update_checker/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file From ee117fac8c905b29486754cde8c4921292c50578 Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Thu, 30 Sep 2021 16:47:28 -0400 Subject: [PATCH 171/664] Adding JSON log format for Nginx logs In order to simplify log processing it is useful to have a JSON format for the Nginx logs. This adds a detailed set of information about each Nginx request that can be used to generate a rich set of answers about usage and performance of HTTP traffic. --- .../nginx/templates/etc/nginx/nginx.conf.j2 | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index db39d4a5667..c54faf12ae0 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -51,6 +51,45 @@ http { 'request_time=$request_time request_id=$request_id upstream_response_time=$upstream_response_time ' 'upstream_connect_time=$upstream_connect_time upstream_header_time=$upstream_header_time'; + log_format json_analytics escape=json '{' + '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution + '"connection": "$connection", ' # connection serial number + '"connection_requests": "$connection_requests", ' # number of requests made in connection + '"pid": "$pid", ' # process pid + '"request_id": "$request_id", ' # the unique request id + '"request_length": "$request_length", ' # request length (including headers and body) + '"remote_addr": "$remote_addr", ' # client IP + '"remote_user": "$remote_user", ' # client HTTP username + '"remote_port": "$remote_port", ' # client port + '"time_local": "$time_local", ' + '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format + '"request": "$request", ' # full path no arguments of the request + '"request_uri": "$request_uri", ' # full path and arguments of the request + '"args": "$args", ' # args + '"status": "$status", ' # response status code + '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client + '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client + '"http_referer": "$http_referer", ' # HTTP referer + '"http_user_agent": "$http_user_agent", ' # user agent + '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for + '"http_host": "$http_host", ' # the request Host: header + '"server_name": "$server_name", ' # the name of the vhost serving the request + '"request_time": "$request_time", ' # request processing time in seconds with msec resolution + '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests + '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS + '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers + '"upstream_response_time": "$upstream_response_time", ' # time spent receiving upstream body + '"upstream_response_length": "$upstream_response_length", ' # upstream response length + '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable + '"ssl_protocol": "$ssl_protocol", ' # TLS protocol + '"ssl_cipher": "$ssl_cipher", ' # TLS cipher + '"scheme": "$scheme", ' # http or https + '"request_method": "$request_method", ' # request method + '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0 + '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise + '"gzip_ratio": "$gzip_ratio"' + '}'; + access_log {{ nginx_log_dir }}/access.log {{ NGINX_LOG_FORMAT_NAME }}; error_log {{ nginx_log_dir }}/error.log; @@ -85,4 +124,3 @@ http { include {{ nginx_conf_dir }}/*.conf; include {{ nginx_sites_enabled_dir }}/*; } - From 62a9cd4ea59a4dd6221d1a68a424cfd7a22388b6 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Sun, 3 Oct 2021 23:05:38 +0500 Subject: [PATCH 172/664] fix: upgrading the hash with latest master. --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 59c4963a832..32f679f230e 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -567,7 +567,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - name: git+https://github.com/edx/xblock-qualtrics-survey.git@d3d2773bcf453d14a0626f813911dc308ddce572#egg=xblock_qualtrics_survey extra_args: -e - - name: git+https://github.com/edx/xblock-in-video-quiz.git@8e820b59df2ad0f2f0531c909199d76d536c6100#egg=invideoquiz-xblock + - name: git+https://github.com/edx/xblock-in-video-quiz.git@96ef746309019b589ba9ccf18b71f3f8de660cde#egg=invideoquiz-xblock extra_args: -e - name: git+https://github.com/edx/xblock-submit-and-compare@28751ed01c4d4ae5f10f9ddee710e76f8b8ae640#egg=xblock-submit-and-compare extra_args: -e From bb1f69aefb578461cf685c297e2784e44f66e052 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Sun, 3 Oct 2021 23:14:32 +0500 Subject: [PATCH 173/664] chore: upgrading the edx-zoom latest version. --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 3a0acc60827..53658cd455f 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -563,7 +563,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: extra_args: -e - name: git+https://github.com/open-craft/xblock-activetable.git@013003aa3ce28f0ae03b8227dc3a6daa4e19997d#egg=activetable-xblock extra_args: -e - - name: edx-zoom==1.8 + - name: edx-zoom==2.0.1 # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - name: git+https://github.com/edx/xblock-qualtrics-survey.git@d3d2773bcf453d14a0626f813911dc308ddce572#egg=xblock_qualtrics_survey extra_args: -e From a757580ddcd1cb43138e71639adfb20a4533dba5 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 4 Oct 2021 10:35:33 +0500 Subject: [PATCH 174/664] update sandbox AMI --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 8124c0667df..f03d7db95fd 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -132,7 +132,7 @@ if [[ -z $ami ]]; then elif [[ $server_type == "ubuntu_18.04" ]]; then ami="ami-07ebfd5b3428b6f4d" elif [[ $server_type == "ubuntu_20.04" || $server_type == "full_edx_installation_from_scratch" ]]; then - ami="ami-0dd76f917833aac4b" + ami="ami-089b5711e63812c2a" # Ansible will always use Python3 interpreter on Ubuntu 20.04 hosts to execute modules extra_var_arg+=' -e ansible_python_interpreter=auto' fi From b6a4c5696140d25164f4b096d97d327979ac1303 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Thu, 7 Oct 2021 12:38:36 -0400 Subject: [PATCH 175/664] feat: Add apt retry logic to neo4j role. --- playbooks/roles/neo4j/tasks/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/playbooks/roles/neo4j/tasks/main.yml b/playbooks/roles/neo4j/tasks/main.yml index 06d51e94628..69b9941841c 100644 --- a/playbooks/roles/neo4j/tasks/main.yml +++ b/playbooks/roles/neo4j/tasks/main.yml @@ -28,6 +28,10 @@ tags: - install - install:system-requirements + retries: 10 + delay: 10 + register: neo4j_gpg_key + until: neo4j_gpg_key is succeeded - name: add neo4j apt repository apt_repository: @@ -36,6 +40,10 @@ tags: - install - install:system-requirements + retries: 10 + delay: 10 + register: neo4j_apt_repository + until: neo4j_apt_repository is succeeded - name: remove deprecated config file file: @@ -52,6 +60,10 @@ tags: - install - install:base + retries: 10 + delay: 10 + register: neo4j_apt_pkg + until: neo4j_apt_pkg is succeeded - name: enable or disable authentication lineinfile: From a092d86a4151f5e35f1f046957debd1ddc350a67 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Thu, 7 Oct 2021 12:46:39 -0400 Subject: [PATCH 176/664] chore: Update DevOps to SRE in several places. --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- docker/build/edxapp/lms.yml | 2 +- docker/build/edxapp/studio.yml | 2 +- documentation/0001-ansible-code-conventions.rst | 2 +- playbooks/roles/edxapp/defaults/main.yml | 2 +- playbooks/sample_vars/server_vars.yml | 4 ++-- util/packer/stop-automatic-updates.sh | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6c7b9de1a19..7e8124403bc 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,10 +11,10 @@ Configuration Pull Request Make sure that the following steps are done before merging: - - [ ] A DevOps team member has approved the PR if it is code shared across multiple services and you don't own all of the services. + - [ ] A SRE team member has approved the PR if it is code shared across multiple services and you don't own all of the services. - [ ] Are you adding any new default values that need to be overridden when this change goes live? If so: - [ ] Update the appropriate internal repo (be sure to update for all our environments) - - [ ] If you are updating a secure value rather than an internal one, file a DEVOPS ticket with details. + - [ ] If you are updating a secure value rather than an internal one, file a SRE ticket with details. - [ ] Add an entry to the CHANGELOG. - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)? Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes). - [ ] Think about how this change will affect Open edX operators. Have you updated the wiki page for the next Open edX release? diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 397cce5db7f..3c48a6cdefc 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -494,7 +494,7 @@ RETIREMENT_STATES: - COMPLETE SECRET_KEY: DUMMY KEY ONLY FOR TO DEVSTACK SEGMENT_KEY: null -SERVER_EMAIL: devops@example.com +SERVER_EMAIL: sre@example.com SESSION_COOKIE_DOMAIN: '' SESSION_COOKIE_NAME: sessionid SESSION_COOKIE_SECURE: false diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 1058cc847f0..26cf0483a1b 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -411,7 +411,7 @@ RETIREMENT_STATES: - COMPLETE SECRET_KEY: DUMMY KEY ONLY FOR TO DEVSTACK SEGMENT_KEY: null -SERVER_EMAIL: devops@example.com +SERVER_EMAIL: sre@example.com SESSION_COOKIE_DOMAIN: '' SESSION_COOKIE_NAME: sessionid SESSION_COOKIE_SECURE: false diff --git a/documentation/0001-ansible-code-conventions.rst b/documentation/0001-ansible-code-conventions.rst index 8f1ba4d1f1d..3d61c0fc51d 100644 --- a/documentation/0001-ansible-code-conventions.rst +++ b/documentation/0001-ansible-code-conventions.rst @@ -140,4 +140,4 @@ Directory structure for the secure repository: -Secure vars are set in files under the ``ansible/vars`` directory. These files will be passed in when the relevant ansible-playbook commands are run. If you need a secure variable defined, give it a name and use it in your playbooks like any other variable. The value should be set in the secure vars files of the relevant deployment (edx, edge, etc.). If you don't have access to this repository, you'll need to submit a ticket to the devops team to make the secure change. +Secure vars are set in files under the ``ansible/vars`` directory. These files will be passed in when the relevant ansible-playbook commands are run. If you need a secure variable defined, give it a name and use it in your playbooks like any other variable. The value should be set in the secure vars files of the relevant deployment (edx, edge, etc.). If you don't have access to this repository, you'll need to submit a ticket to the SRE team to make the secure change. diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 12b260b6309..79854210394 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -414,7 +414,7 @@ EDXAPP_CONTACT_EMAIL: 'info@example.com' EDXAPP_BUGS_EMAIL: 'bugs@example.com' EDXAPP_DEFAULT_FROM_EMAIL: 'registration@example.com' EDXAPP_DEFAULT_FEEDBACK_EMAIL: 'feedback@example.com' -EDXAPP_DEFAULT_SERVER_EMAIL: 'devops@example.com' +EDXAPP_DEFAULT_SERVER_EMAIL: 'sre@example.com' EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL: 'no-reply@example.com' EDXAPP_BULK_EMAIL_LOG_SENT_EMAILS: false EDXAPP_UNIVERSITY_EMAIL: 'university@example.com' diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index 8c8d5da8f1c..ff14f08a23f 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -138,8 +138,8 @@ # ## send logs to s3 #AWS_S3_LOGS: true -#AWS_S3_LOGS_NOTIFY_EMAIL: devops+logs@example.com -#AWS_S3_LOGS_FROM_EMAIL: devops@example.com +#AWS_S3_LOGS_NOTIFY_EMAIL: SRE+logs@example.com +#AWS_S3_LOGS_FROM_EMAIL: sre@example.com #EDX_ANSIBLE_DUMP_VARS: true #CONFIGURATION_VERSION: release #EDXAPP_INSTALL_PRIVATE_REQUIREMENTS: true diff --git a/util/packer/stop-automatic-updates.sh b/util/packer/stop-automatic-updates.sh index 2b0301ab9e0..610cb391c51 100644 --- a/util/packer/stop-automatic-updates.sh +++ b/util/packer/stop-automatic-updates.sh @@ -3,7 +3,7 @@ # Kill the apt services holding a dpkg lock, so that the ansible-bootstrap # script can run without conflicts. -# NOTE: this is a temporary fix. Instead, we should be doing what devops does, +# NOTE: this is a temporary fix. Instead, we should be doing what SRE does, # and first run the security+common roles on a vanilla AMI, which will disable # unattended-updates and set up users. Then we can feel free to run the # ansible bootstrap without any problems. From ad0ed51e87f0184427cec3ffb3d74977d0c62e5e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 7 Oct 2021 13:48:24 -0400 Subject: [PATCH 177/664] build: use the organization commitlint check --- .github/workflows/commitlint.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/commitlint.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 00000000000..e2b066153f2 --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,10 @@ +# Run commitlint on the commit messages in a pull request. + +name: Lint Commit Messages + +on: + - pull_request + +jobs: + commitlint: + uses: edx/.github/.github/workflows/commitlint.yml@master From f52f6e9fc1f1343f5b932bcc05511874c57fe25a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 8 Oct 2021 15:32:17 +0500 Subject: [PATCH 178/664] chore: update pyyaml --- util/check_for_key_collisions/requirements.txt | 2 +- util/helm_values_to_rst_table_util/requirements3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/check_for_key_collisions/requirements.txt b/util/check_for_key_collisions/requirements.txt index edb74a8e7be..28655b36eb1 100644 --- a/util/check_for_key_collisions/requirements.txt +++ b/util/check_for_key_collisions/requirements.txt @@ -1,2 +1,2 @@ Click==7.0 -PyYAML==5.1 +PyYAML==5.4.1 diff --git a/util/helm_values_to_rst_table_util/requirements3.txt b/util/helm_values_to_rst_table_util/requirements3.txt index dcae9c809c1..28655b36eb1 100644 --- a/util/helm_values_to_rst_table_util/requirements3.txt +++ b/util/helm_values_to_rst_table_util/requirements3.txt @@ -1,2 +1,2 @@ Click==7.0 -PyYAML==5.1.2 +PyYAML==5.4.1 From d44c0f45640d670bd7eca1833f7cf5d39f8de12e Mon Sep 17 00:00:00 2001 From: awaisali Date: Fri, 8 Oct 2021 17:06:19 +0500 Subject: [PATCH 179/664] fix: update build jenkins plugins --- playbooks/roles/aws/defaults/main.yml | 2 +- playbooks/roles/aws/tasks/main.yml | 1 - .../roles/jenkins_build/defaults/main.yml | 40 +++++++++++++++---- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/aws/defaults/main.yml b/playbooks/roles/aws/defaults/main.yml index 38781b83f55..1e8f2624984 100644 --- a/playbooks/roles/aws/defaults/main.yml +++ b/playbooks/roles/aws/defaults/main.yml @@ -54,4 +54,4 @@ aws_redhat_pkgs: [] # This issues started happening after the ec2_facts role was replaced with # the new version `ec2_metadata_facts` that fails when the server is not # on AWS, unlike its older counterpart -AWS_GATHER_FACTS: true +AWS_GATHER_FACTS: true \ No newline at end of file diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml index cd8f8f53c10..292c0b1a011 100644 --- a/playbooks/roles/aws/tasks/main.yml +++ b/playbooks/roles/aws/tasks/main.yml @@ -46,7 +46,6 @@ # # End dealing with Jumbo frames issue in mixed MTU deployements in AWS # - - name: Install system packages apt: name: "{{ aws_debian_pkgs }}" diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 38de36ed1c2..3ad50f62cf4 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.263.4 +BUILD_JENKINS_VERSION: jenkins_2.289.3 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: @@ -51,6 +51,9 @@ build_jenkins_plugins_list: - name: 'badge' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'bootstrap5-api' + version: '5.1.1-1' + group: 'io.jenkins.plugins' - name: 'bouncycastle-api' version: '2.18' group: 'org.jenkins-ci.plugins' @@ -63,17 +66,23 @@ build_jenkins_plugins_list: - name: 'build-user-vars-plugin' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'checks-api' + version: '1.7.2' + group: 'io.jenkins.plugins' + - name: 'cloudbees-folder' + version: '6.16' + group: 'org.jenkins-ci.plugins' - name: 'cobertura' version: '1.16' group: 'org.jenkins-ci.plugins' - name: 'code-coverage-api' - version: '1.1.6' + version: '1.4.1' group: 'io.jenkins.plugins' - name: 'copyartifact' version: '1.44' group: 'org.jenkins-ci.plugins' - name: 'credentials' - version: '2.4.1' + version: '2.6.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' version: '1.23' @@ -81,6 +90,9 @@ build_jenkins_plugins_list: - name: 'cvs' version: '2.17' group: 'org.jenkins-ci.plugins' + - name: 'display-url-api' + version: '2.3.5' + group: 'org.jenkins-ci.plugins' - name: 'docker-commons' version: '1.8' group: 'org.jenkins-ci.plugins' @@ -94,7 +106,7 @@ build_jenkins_plugins_list: version: '2.3.0' group: 'org.jenkins-ci.plugins' - name: 'echarts-api' - version: '5.0.1-1' + version: '5.1.2-11' group: 'io.jenkins.plugins' - name: 'exclusive-execution' version: '0.8' @@ -102,6 +114,9 @@ build_jenkins_plugins_list: - name: 'external-monitor-job' version: '1.4' group: 'org.jenkins-ci.plugins' + - name: 'font-awesome-api' + version: '5.15.4-1' + group: 'io.jenkins.plugins' - name: 'flexible-publish' version: '0.15.2' group: 'org.jenkins-ci.plugins' @@ -139,7 +154,7 @@ build_jenkins_plugins_list: version: '1.21' group: 'org.jenkins-ci.plugins' - name: 'jackson2-api' - version: '2.12.1' + version: '2.12.4' group: 'org.jenkins-ci.plugins' - name: 'javadoc' version: '1.3' @@ -154,7 +169,7 @@ build_jenkins_plugins_list: version: '1.77' group: 'org.jenkins-ci.plugins' - name: 'jquery3-api' - version: '3.5.1-1' + version: '3.6.0-2' group: 'io.jenkins.plugins' - name: 'junit' version: '1.30' @@ -202,7 +217,10 @@ build_jenkins_plugins_list: version: '2.0.2' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' - version: '2.0.0' + version: '2.4.0' + group: 'io.jenkins.plugins' + - name: 'popper2-api' + version: '2.10.1-1' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' @@ -232,7 +250,7 @@ build_jenkins_plugins_list: version: '2.21' group: 'org.jenkins-ci.plugins' - name: 'snakeyaml-api' - version: '1.27.0' + version: '1.29.1' group: 'io.jenkins.plugins' - name: 'splunk-devops' version: '1.8.1' @@ -300,6 +318,12 @@ build_jenkins_plugins_list: - name: 'workflow-job' version: '2.40' group: 'org.jenkins-ci.plugins.workflow' + - name: 'workflow-multibranch' + version: '2.24' + group: 'org.jenkins-ci.plugins.workflow' + - name: 'workflow-scm-step' + version: '2.13' + group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-support' version: '3.8' group: 'org.jenkins-ci.plugins.workflow' From b60c14dd0145e9f599b1cf1e1c5c047b5e322208 Mon Sep 17 00:00:00 2001 From: Syed Awais Ali Date: Fri, 8 Oct 2021 17:13:40 +0500 Subject: [PATCH 180/664] chore: add new line to end of file --- playbooks/roles/aws/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws/defaults/main.yml b/playbooks/roles/aws/defaults/main.yml index 1e8f2624984..38781b83f55 100644 --- a/playbooks/roles/aws/defaults/main.yml +++ b/playbooks/roles/aws/defaults/main.yml @@ -54,4 +54,4 @@ aws_redhat_pkgs: [] # This issues started happening after the ec2_facts role was replaced with # the new version `ec2_metadata_facts` that fails when the server is not # on AWS, unlike its older counterpart -AWS_GATHER_FACTS: true \ No newline at end of file +AWS_GATHER_FACTS: true From 3950e414f723adc8e78bbe2737ca99f8d357e66b Mon Sep 17 00:00:00 2001 From: awaisali Date: Fri, 8 Oct 2021 19:34:20 +0500 Subject: [PATCH 181/664] fix: reverting changes as it did not sent notifications --- playbooks/roles/aws/tasks/main.yml | 1 + .../roles/jenkins_build/defaults/main.yml | 40 ++++--------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml index 292c0b1a011..cd8f8f53c10 100644 --- a/playbooks/roles/aws/tasks/main.yml +++ b/playbooks/roles/aws/tasks/main.yml @@ -46,6 +46,7 @@ # # End dealing with Jumbo frames issue in mixed MTU deployements in AWS # + - name: Install system packages apt: name: "{{ aws_debian_pkgs }}" diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 3ad50f62cf4..38de36ed1c2 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.289.3 +BUILD_JENKINS_VERSION: jenkins_2.263.4 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: @@ -51,9 +51,6 @@ build_jenkins_plugins_list: - name: 'badge' version: '1.5' group: 'org.jenkins-ci.plugins' - - name: 'bootstrap5-api' - version: '5.1.1-1' - group: 'io.jenkins.plugins' - name: 'bouncycastle-api' version: '2.18' group: 'org.jenkins-ci.plugins' @@ -66,23 +63,17 @@ build_jenkins_plugins_list: - name: 'build-user-vars-plugin' version: '1.5' group: 'org.jenkins-ci.plugins' - - name: 'checks-api' - version: '1.7.2' - group: 'io.jenkins.plugins' - - name: 'cloudbees-folder' - version: '6.16' - group: 'org.jenkins-ci.plugins' - name: 'cobertura' version: '1.16' group: 'org.jenkins-ci.plugins' - name: 'code-coverage-api' - version: '1.4.1' + version: '1.1.6' group: 'io.jenkins.plugins' - name: 'copyartifact' version: '1.44' group: 'org.jenkins-ci.plugins' - name: 'credentials' - version: '2.6.1' + version: '2.4.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' version: '1.23' @@ -90,9 +81,6 @@ build_jenkins_plugins_list: - name: 'cvs' version: '2.17' group: 'org.jenkins-ci.plugins' - - name: 'display-url-api' - version: '2.3.5' - group: 'org.jenkins-ci.plugins' - name: 'docker-commons' version: '1.8' group: 'org.jenkins-ci.plugins' @@ -106,7 +94,7 @@ build_jenkins_plugins_list: version: '2.3.0' group: 'org.jenkins-ci.plugins' - name: 'echarts-api' - version: '5.1.2-11' + version: '5.0.1-1' group: 'io.jenkins.plugins' - name: 'exclusive-execution' version: '0.8' @@ -114,9 +102,6 @@ build_jenkins_plugins_list: - name: 'external-monitor-job' version: '1.4' group: 'org.jenkins-ci.plugins' - - name: 'font-awesome-api' - version: '5.15.4-1' - group: 'io.jenkins.plugins' - name: 'flexible-publish' version: '0.15.2' group: 'org.jenkins-ci.plugins' @@ -154,7 +139,7 @@ build_jenkins_plugins_list: version: '1.21' group: 'org.jenkins-ci.plugins' - name: 'jackson2-api' - version: '2.12.4' + version: '2.12.1' group: 'org.jenkins-ci.plugins' - name: 'javadoc' version: '1.3' @@ -169,7 +154,7 @@ build_jenkins_plugins_list: version: '1.77' group: 'org.jenkins-ci.plugins' - name: 'jquery3-api' - version: '3.6.0-2' + version: '3.5.1-1' group: 'io.jenkins.plugins' - name: 'junit' version: '1.30' @@ -217,10 +202,7 @@ build_jenkins_plugins_list: version: '2.0.2' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' - version: '2.4.0' - group: 'io.jenkins.plugins' - - name: 'popper2-api' - version: '2.10.1-1' + version: '2.0.0' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' @@ -250,7 +232,7 @@ build_jenkins_plugins_list: version: '2.21' group: 'org.jenkins-ci.plugins' - name: 'snakeyaml-api' - version: '1.29.1' + version: '1.27.0' group: 'io.jenkins.plugins' - name: 'splunk-devops' version: '1.8.1' @@ -318,12 +300,6 @@ build_jenkins_plugins_list: - name: 'workflow-job' version: '2.40' group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-multibranch' - version: '2.24' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-scm-step' - version: '2.13' - group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-support' version: '3.8' group: 'org.jenkins-ci.plugins.workflow' From e3909737c52b2dbc63228aaacd1661a33a234ea0 Mon Sep 17 00:00:00 2001 From: Jawayria <39649635+Jawayria@users.noreply.github.com> Date: Wed, 16 Jun 2021 14:14:54 +0500 Subject: [PATCH 182/664] feat: Added upgrade-requirements workflow --- .../workflows/upgrade-python-requirements.yml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/upgrade-python-requirements.yml diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml new file mode 100644 index 00000000000..384cbf6f033 --- /dev/null +++ b/.github/workflows/upgrade-python-requirements.yml @@ -0,0 +1,69 @@ +name: Upgrade Requirements + +on: + schedule: + # will start the job at 01:15 UTC every Monday + - cron: "15 1 * * 1" + workflow_dispatch: + inputs: + branch: + description: "Target branch to create requirements PR against" + required: true + default: 'master' + +jobs: + upgrade_requirements: + runs-on: ubuntu-20.04 + + strategy: + matrix: + python-version: ["3.8"] + + steps: + - name: setup target branch + run: echo "target_branch=$(if ['${{ github.event.inputs.branch }}' = '']; then echo 'master'; else echo '${{ github.event.inputs.branch }}'; fi)" >> $GITHUB_ENV + + - uses: actions/checkout@v1 + with: + ref: ${{ env.target_branch }} + + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: make upgrade + run: | + cd $GITHUB_WORKSPACE + make upgrade + + - name: setup testeng-ci + run: | + git clone https://github.com/edx/testeng-ci.git + cd $GITHUB_WORKSPACE/testeng-ci + ls + pip install -r requirements/base.txt + - name: create pull request + env: + GITHUB_TOKEN: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} + GITHUB_USER_EMAIL: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} + run: | + cd $GITHUB_WORKSPACE/testeng-ci + python -m jenkins.pull_request_creator --repo-root=$GITHUB_WORKSPACE \ + --target-branch="${{ env.target_branch }}" --base-branch-name="upgrade-python-requirements" \ + --commit-message="chore: Updating Python Requirements" --pr-title="Python Requirements Update" \ + --pr-body="Python requirements update.Please review the [changelogs](https://openedx.atlassian.net/wiki/spaces/TE/pages/1001521320/Python+Package+Changelogs) for the upgraded packages." \ + --user-reviewers="" --team-reviewers="devops" --delete-old-pull-requests + + - name: Send failure notification + if: ${{ failure() }} + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.EDX_SMTP_USERNAME}} + password: ${{secrets.EDX_SMTP_PASSWORD}} + subject: Upgrade python requirements workflow failed in ${{github.repository}} + to: devops@edx.org + from: github-actions + body: Upgrade python requirements workflow in ${{github.repository}} failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From 5f2b8f0d03e9a241d83c4cc465d3e64029606662 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 13 Oct 2021 11:43:39 -0400 Subject: [PATCH 183/664] perf: speed up designer runbooks by not installing py3 pre 3.8 --- playbooks/roles/designer/meta/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/designer/meta/main.yml b/playbooks/roles/designer/meta/main.yml index d4124ce7c4a..09d6106ae24 100644 --- a/playbooks/roles/designer/meta/main.yml +++ b/playbooks/roles/designer/meta/main.yml @@ -13,6 +13,7 @@ dependencies: - role: edx_django_service edx_django_service_use_python38: '{{ DESIGNER_USE_PYTHON38 }}' + edx_django_service_use_python3: '{{ not DESIGNER_USE_PYTHON38 }}' edx_django_service_version: '{{ DESIGNER_VERSION }}' edx_django_service_name: '{{ designer_service_name }}' edx_django_service_config_overrides: '{{ designer_service_config_overrides }}' From d915467663bf19a95f580898985efeb39b723fca Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 13 Oct 2021 14:50:23 -0400 Subject: [PATCH 184/664] fix: Warnings due to duplicate registers on tasks --- playbooks/roles/newrelic_infrastructure/tasks/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/playbooks/roles/newrelic_infrastructure/tasks/main.yml b/playbooks/roles/newrelic_infrastructure/tasks/main.yml index 2a54446dfbb..fed66ad3f53 100644 --- a/playbooks/roles/newrelic_infrastructure/tasks/main.yml +++ b/playbooks/roles/newrelic_infrastructure/tasks/main.yml @@ -41,8 +41,8 @@ when: ansible_distribution == 'Ubuntu' retries: 10 delay: 10 - register: result - until: result is succeeded + register: nr_apt_key + until: nr_apt_key is succeeded # For focal, use the bionic repo for now. - name: Install apt repository for New Relic Infrastructure if neither bionic nor focal @@ -85,8 +85,6 @@ when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal' retries: 10 delay: 10 - register: result - until: result is succeeded register: nr_apt_repo_focal until: nr_apt_repo_focal is succeeded @@ -118,8 +116,6 @@ when: ansible_distribution == 'Amazon' retries: 10 delay: 10 - register: result - until: result is succeeded register: nr_yum_repo until: nr_yum_repo is succeeded @@ -135,8 +131,6 @@ when: ansible_distribution == 'Amazon' retries: 10 delay: 10 - register: result - until: result is succeeded register: nr_yum_pkg until: nr_yum_pkg is succeeded From fefffb36b50566d25c0d79920cb90ff2d2b065b3 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 14 Oct 2021 10:12:08 -0400 Subject: [PATCH 185/664] fix: don't advertise the building of a service not supported JIRA:EDUCATOR-5920 --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index f03d7db95fd..7e929437099 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -482,7 +482,7 @@ ENTERPRISE_CATALOG_URL_ROOT: "https://enterprise-catalog-${deploy_host}" EOF fi -encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api insights discovery credentials registrar edx_notes_api license_manager) +encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api discovery credentials registrar edx_notes_api license_manager) for app in ${encrypted_config_apps[@]}; do eval app_decrypt_and_copy_config_enabled=\${${app}_decrypt_and_copy_config_enabled} From 974e7ae36a70523dad5ec8ddbd7f92e04e3fb166 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 14 Oct 2021 10:16:36 -0400 Subject: [PATCH 186/664] style: separate parameter controlling integration env. sandbox build JIRA:EDUCATOR-5920 --- util/jenkins/ansible-provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index f03d7db95fd..08268463e12 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -541,7 +541,7 @@ veda_encode_worker=${video_encode_worker:-false} video_pipeline_integration=${video_pipeline:-false} # ansible overrides for master's integration environment setup -if [[ $registrar == "true" ]]; then +if [[ $masters_integration_environment == "true" ]]; then cat << EOF >> $extra_vars_file COMMON_ENABLE_SPLUNKFORWARDER: true EDXAPP_ENABLE_ENROLLMENT_RESET: true @@ -609,7 +609,7 @@ fi run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu # master's integration environment setup -if [[ $registrar == "true" ]]; then +if [[ $masters_integration_environment == "true" ]]; then # vars specific to master's integration environment cat << EOF >> $extra_vars_file username: $registrar_user_email From 379cc5b1f12e19aa34649bd7c963cd32fd230f08 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Oct 2021 13:25:35 -0400 Subject: [PATCH 187/664] feat: Update to the latest problem builder This includes fixes and supports for running in Django 3.2 --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 79854210394..3d964968650 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -537,7 +537,7 @@ EDXAPP_EXTRA_REQUIREMENTS: [] # - name: git+https://git.myproject.org/MyProject#egg=MyProject EDXAPP_PRIVATE_REQUIREMENTS: # For Harvard courses: - - name: xblock-problem-builder==4.1.11 + - name: xblock-problem-builder==5.0.0 # Oppia XBlock - name: git+https://github.com/oppia/xblock.git@3b5c17c5832b4f8ef132c6bbf48da8a86df43b3d#egg=oppia-xblock extra_args: -e From b8489d00030df9bcb9e06f9a7ca69b1095f1ea52 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Oct 2021 13:32:10 -0400 Subject: [PATCH 188/664] feat: Update the default versions of various xblocks Many of these were out of date compared to what is running on edx.org. Update the default versions to pick up the latest changes and bug fixes. --- playbooks/roles/edxapp/defaults/main.yml | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 3d964968650..6c5d46fc13a 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -539,19 +539,19 @@ EDXAPP_PRIVATE_REQUIREMENTS: # For Harvard courses: - name: xblock-problem-builder==5.0.0 # Oppia XBlock - - name: git+https://github.com/oppia/xblock.git@3b5c17c5832b4f8ef132c6bbf48da8a86df43b3d#egg=oppia-xblock + - name: git+https://github.com/oppia/xblock.git@1030adb3590ad2d32c93443cc8690db0985d76b6#egg=oppia-xblock extra_args: -e # This repository contains schoolyourself-xblock, which is used in # edX's "AlgebraX" and "GeometryX" courses. - - name: git+https://github.com/edx/schoolyourself-xblock.git@c3f7fecefa8e164a0f02ef6264d00e65e9c5a6de#egg=schoolyourself-xblock + - name: git+https://github.com/edx/schoolyourself-xblock.git@2093048720cfb36cc05b3143cd6f2585c7c64d85#egg=schoolyourself-xblock extra_args: -e # Prototype XBlocks from edX learning sciences limited roll-outs and user testing. # Concept XBlock, in particular, is nowhere near finished and an early prototype. # Profile XBlock is there so we can play with XBlock arguments in the platform, but isn't ready for use outside of # edX. - - name: git+https://github.com/edx/ConceptXBlock.git@a8e3cff03c3b46dad093db069b12c6fba1aed327#egg=concept-xblock + - name: git+https://github.com/edx/ConceptXBlock.git@a45a6560c92b6d8b62be1f939ff1d00dfff84e70#egg=concept-xblock extra_args: -e - - name: git+https://github.com/edx/AudioXBlock.git@f668a4e231c5cdc338eef6bd148e57a02b48200b#egg=audio-xblock + - name: git+https://github.com/edx/AudioXBlock.git@20538c6e9bb704801a71ecbb6981f794556dfc45#egg=audio-xblock extra_args: -e - name: git+https://github.com/edx/AnimationXBlock.git@c950ffdda2f69effda93bf03df8646f61d3ffada#egg=animation-xblock extra_args: -e @@ -559,29 +559,29 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/ubc/ubcpi.git@1.0.0#egg=ubcpi-xblock extra_args: -e # Vector Drawing and ActiveTable XBlocks (Davidson) - - name: git+https://github.com/open-craft/xblock-vectordraw.git@76976425356dfc7f13570f354c0c438db84c2840#egg=vectordraw-xblock==0.3.0 + - name: git+https://github.com/open-craft/xblock-vectordraw.git@0b931ae5d6314dbda5b58ab6c865aea1bc121267#egg=vectordraw-xblock==0.3.0 extra_args: -e - - name: git+https://github.com/open-craft/xblock-activetable.git@013003aa3ce28f0ae03b8227dc3a6daa4e19997d#egg=activetable-xblock + - name: git+https://github.com/open-craft/xblock-activetable.git@d3fb772435c382b59293e4e688a6a3096c4f6fd7#egg=activetable-xblock extra_args: -e - name: edx-zoom==2.0.1 # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - - name: git+https://github.com/edx/xblock-qualtrics-survey.git@d3d2773bcf453d14a0626f813911dc308ddce572#egg=xblock_qualtrics_survey + - name: git+https://github.com/edx/xblock-qualtrics-survey.git@02d87f567ec2af4579642cf795bb851507d6edf9#egg=xblock_qualtrics_survey extra_args: -e - - name: git+https://github.com/edx/xblock-in-video-quiz.git@96ef746309019b589ba9ccf18b71f3f8de660cde#egg=invideoquiz-xblock + - name: git+https://github.com/edx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock extra_args: -e - - name: git+https://github.com/edx/xblock-submit-and-compare@28751ed01c4d4ae5f10f9ddee710e76f8b8ae640#egg=xblock-submit-and-compare + - name: git+https://github.com/edx/xblock-submit-and-compare@8f0b3279f36e25aff8cce3f0b1793b9f19dc8729#egg=xblock-submit-and-compare extra_args: -e - - name: git+https://github.com/edx/xblock-free-text-response@19ed5bb4b9aece19e9b1c62995a09f73f893f536#egg=xblock-free-text-response + - name: git+https://github.com/edx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response extra_args: -e - - name: git+https://github.com/edx/xblock-sql-grader@56b13ea9719d5c6e910ec2edf98b35dda0ba9c57#egg=xblock-sql-grader + - name: git+https://github.com/edx/xblock-sql-grader@5257b4a1aa07aa572fd3865647d91d0628f0cbd3#egg=xblock-sql-grader extra_args: -e - - name: git+https://github.com/edx/xblock-image-modal@20262ae713bf307f7b13afe73c8a32c5c56b4fd6#egg=xblock-image-modal + - name: git+https://github.com/edx/xblock-image-modal@bef91413447e15570863ab08316a04c0b546b268#egg=xblock-image-modal extra_args: -e # XBlocks associated with the LabXchange project - - name: git+https://github.com/open-craft/labxchange-xblocks.git@5352e93cb26c789aecbf92fabf210179483616a2#egg=labxchange-xblocks + - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e # "Pathways" learning context plugin for the LabXchange project - - name: git+https://github.com/open-craft/lx-pathway-plugin.git@337abf249b7c5ecc1e78a44d2e639e1ab65f2085#egg=lx-pathway-plugin + - name: git+https://github.com/open-craft/lx-pathway-plugin.git@ba1d470217cd5908cbd8b56075628bd4eacf7b39#egg=lx-pathway-plugin extra_args: -e # Caliper and xAPI event routing plugin - name: edx-event-routing-backends==4.1.1 From ad5341cf2557a1b752009e392c65c13608a34623 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 15 Oct 2021 14:34:48 +0500 Subject: [PATCH 189/664] chore: Update build plugins --- .../roles/jenkins_build/defaults/main.yml | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 38de36ed1c2..93b7d8469c6 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -51,6 +51,9 @@ build_jenkins_plugins_list: - name: 'badge' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'bootstrap5-api' + version: '5.1.1-1' + group: 'io.jenkins.plugins' - name: 'bouncycastle-api' version: '2.18' group: 'org.jenkins-ci.plugins' @@ -63,17 +66,23 @@ build_jenkins_plugins_list: - name: 'build-user-vars-plugin' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'checks-api' + version: '1.7.2' + group: 'io.jenkins.plugins' + - name: 'cloudbees-folder' + version: '6.16' + group: 'org.jenkins-ci.plugins' - name: 'cobertura' version: '1.16' group: 'org.jenkins-ci.plugins' - name: 'code-coverage-api' - version: '1.1.6' + version: '2.0.2' group: 'io.jenkins.plugins' - name: 'copyartifact' version: '1.44' group: 'org.jenkins-ci.plugins' - name: 'credentials' - version: '2.4.1' + version: '2.6.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' version: '1.23' @@ -81,6 +90,12 @@ build_jenkins_plugins_list: - name: 'cvs' version: '2.17' group: 'org.jenkins-ci.plugins' + - name: 'data-tables-api' + version: '1.11.2-1' + group: 'io.jenkins.plugins' + - name: 'display-url-api' + version: '2.3.5' + group: 'org.jenkins-ci.plugins' - name: 'docker-commons' version: '1.8' group: 'org.jenkins-ci.plugins' @@ -94,7 +109,7 @@ build_jenkins_plugins_list: version: '2.3.0' group: 'org.jenkins-ci.plugins' - name: 'echarts-api' - version: '5.0.1-1' + version: '5.2.1-1' group: 'io.jenkins.plugins' - name: 'exclusive-execution' version: '0.8' @@ -105,6 +120,12 @@ build_jenkins_plugins_list: - name: 'flexible-publish' version: '0.15.2' group: 'org.jenkins-ci.plugins' + - name: 'font-awesome-api' + version: '5.15.4-1' + group: 'io.jenkins.plugins' + - name: 'forensics-api' + version: '1.4.0' + group: 'io.jenkins.plugins' - name: 'ghprb' version: '1.42.1' group: 'org.jenkins-ci.plugins' @@ -139,7 +160,7 @@ build_jenkins_plugins_list: version: '1.21' group: 'org.jenkins-ci.plugins' - name: 'jackson2-api' - version: '2.12.1' + version: '2.13.1' group: 'org.jenkins-ci.plugins' - name: 'javadoc' version: '1.3' @@ -154,7 +175,7 @@ build_jenkins_plugins_list: version: '1.77' group: 'org.jenkins-ci.plugins' - name: 'jquery3-api' - version: '3.5.1-1' + version: '3.6.0-2' group: 'io.jenkins.plugins' - name: 'junit' version: '1.30' @@ -202,7 +223,10 @@ build_jenkins_plugins_list: version: '2.0.2' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' - version: '2.0.0' + version: '2.5.0' + group: 'io.jenkins.plugins' + - name: 'popper2-api' + version: '2.10.1-1' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' @@ -223,7 +247,7 @@ build_jenkins_plugins_list: version: '1.0' group: 'org.jenkins-ci.plugins' - name: 'script-security' - version: '1.76' + version: '1.78' group: 'org.jenkins-ci.plugins' - name: 'shiningpanda' version: '0.23' @@ -232,7 +256,7 @@ build_jenkins_plugins_list: version: '2.21' group: 'org.jenkins-ci.plugins' - name: 'snakeyaml-api' - version: '1.27.0' + version: '1.29.1' group: 'io.jenkins.plugins' - name: 'splunk-devops' version: '1.8.1' @@ -286,19 +310,25 @@ build_jenkins_plugins_list: version: '2.6' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-api' - version: '2.41' + version: '2.47' group: 'org.jenkins-ci.plugins.workflow' - name: 'windows-slaves' version: '1.3.1' group: 'org.jenkins-ci.plugins' - name: 'workflow-cps' - version: '2.90' + version: '2.94' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-cps-global-lib' version: '2.15' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-job' - version: '2.40' + version: '2.42' + group: 'org.jenkins-ci.plugins.workflow' + - name: 'workflow-multibranch' + version: '2.24' + group: 'org.jenkins-ci.plugins.workflow' + - name: 'workflow-scm-step' + version: '2.13' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-support' version: '3.8' From 530c9ca4ce2b673c4b450f397efd48a53a6f6cc7 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Fri, 15 Oct 2021 22:33:53 +0500 Subject: [PATCH 190/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 10 +++++----- requirements.txt | 20 +++++++++---------- requirements/pip-tools.txt | 4 ++-- util/elasticsearch/requirements.txt | 2 +- util/jenkins/requirements-cloudflare.txt | 10 +++++----- util/jenkins/requirements.txt | 18 ++++++++--------- util/pingdom/requirements.txt | 10 +++++----- util/vpc-tools/requirements.txt | 8 ++++---- 8 files changed, 41 insertions(+), 41 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 65fe45fb372..31d2b2bfeda 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.20.31 +awscli==1.20.62 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.18.31 +boto3==1.18.62 # via -r requirements/aws.in -botocore==1.21.31 +botocore==1.21.62 # via # awscli # boto3 @@ -37,7 +37,7 @@ pyyaml==5.3.1 # awscli rsa==4.7.2 # via awscli -s3cmd==2.1.0 +s3cmd==2.2.0 # via -r requirements/aws.in s3transfer==0.5.0 # via @@ -45,5 +45,5 @@ s3transfer==0.5.0 # boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.6 +urllib3==1.26.7 # via botocore diff --git a/requirements.txt b/requirements.txt index 3096740f5e9..49b92583e16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.8.20 # via -r requirements/base.in -awscli==1.20.31 +awscli==1.20.62 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,31 +14,31 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.18.31 +boto3==1.18.62 # via -r requirements/base.in -botocore==1.21.31 +botocore==1.21.62 # via # awscli # boto3 # s3transfer -certifi==2021.5.30 +certifi==2021.10.8 # via requests -cffi==1.14.6 +cffi==1.15.0 # via # bcrypt # cryptography # pynacl -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 # via requests colorama==0.4.3 # via awscli -cryptography==3.4.8 +cryptography==35.0.0 # via # ansible # paramiko datadog==0.8.0 # via -r requirements/base.in -decorator==5.0.9 +decorator==5.1.0 # via # datadog # networkx @@ -48,7 +48,7 @@ docutils==0.15.2 # via awscli ecdsa==0.13.3 # via -r requirements/base.in -idna==3.2 +idna==3.3 # via requests jinja2==2.8 # via @@ -105,7 +105,7 @@ six==1.16.0 # pathlib2 # pynacl # python-dateutil -urllib3==1.26.6 +urllib3==1.26.7 # via # botocore # requests diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 3a172bbf62a..6eb6efdb478 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,11 +4,11 @@ # # make upgrade # -click==8.0.1 +click==8.0.3 # via pip-tools pep517==0.11.0 # via pip-tools -pip-tools==6.2.0 +pip-tools==6.4.0 # via -r requirements/pip-tools.in tomli==1.2.1 # via pep517 diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 5999dcee7b6..952c857ee9e 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -10,5 +10,5 @@ elasticsearch==0.4.5 # via -r requirements/elasticsearch.in jsonpickle==2.0.0 # via deepdiff -urllib3==1.26.6 +urllib3==1.26.7 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 7ffec39e4e0..1102b13ce93 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,15 +4,15 @@ # # make upgrade # -certifi==2021.5.30 +certifi==2021.10.8 # via requests -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 # via requests -click==8.0.1 +click==8.0.3 # via -r requirements/cloudflare.in -idna==3.2 +idna==3.3 # via requests requests==2.26.0 # via -r requirements/cloudflare.in -urllib3==1.26.6 +urllib3==1.26.7 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index e7f44aac5ae..c1031b8ba7c 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.20.31 +awscli==1.20.62 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,20 +16,20 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.18.31 +boto3==1.18.62 # via -r requirements/jenkins.in -botocore==1.21.31 +botocore==1.21.62 # via # awscli # boto3 # s3transfer celery==3.1.25 # via -r requirements/jenkins.in -certifi==2021.5.30 +certifi==2021.10.8 # via # opsgenie-sdk # requests -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 # via requests click==6.7 # via -r requirements/jenkins.in @@ -37,7 +37,7 @@ colorama==0.4.3 # via awscli docutils==0.15.2 # via awscli -idna==3.2 +idna==3.3 # via requests jmespath==0.10.0 # via @@ -60,7 +60,7 @@ python-gnupg==0.4.7 # via -r requirements/jenkins.in python-magic==0.4.24 # via s3cmd -pytz==2021.1 +pytz==2021.3 # via # celery # opsgenie-sdk @@ -74,7 +74,7 @@ requests==2.26.0 # via opsgenie-sdk rsa==4.7.2 # via awscli -s3cmd==2.1.0 +s3cmd==2.2.0 # via -r requirements/jenkins.in s3transfer==0.5.0 # via @@ -86,7 +86,7 @@ six==1.16.0 # python-dateutil splunk-sdk==1.6.6 # via -r requirements/jenkins.in -urllib3==1.26.6 +urllib3==1.26.7 # via # botocore # opsgenie-sdk diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 070854d0955..b3dbf7a6f70 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,19 +4,19 @@ # # make upgrade # -certifi==2021.5.30 +certifi==2021.10.8 # via requests -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 # via requests click==6.7 # via -r requirements/pingdom.in -idna==3.2 +idna==3.3 # via requests -pyyaml==5.4.1 +pyyaml==6.0 # via -r requirements/pingdom.in requests==2.26.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.26.6 +urllib3==1.26.7 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index b42dce25ee8..92b14065b77 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,15 +6,15 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2021.5.30 +certifi==2021.10.8 # via requests -charset-normalizer==2.0.4 +charset-normalizer==2.0.7 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==3.2 +idna==3.3 # via requests requests==2.26.0 # via -r requirements/vpc-tools.in -urllib3==1.26.6 +urllib3==1.26.7 # via requests From 3bec388bf7e604eb1d4c6c64378ded829e2f197f Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Mon, 18 Oct 2021 06:42:40 +0500 Subject: [PATCH 191/664] chore: Updating Python Requirements --- playbooks/roles/aws/templates/requirements.txt.j2 | 6 +++--- requirements.txt | 6 +++--- requirements/pip-tools.txt | 2 +- util/jenkins/requirements.txt | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 31d2b2bfeda..93d02af498c 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.20.62 +awscli==1.20.63 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.18.62 +boto3==1.18.63 # via -r requirements/aws.in -botocore==1.21.62 +botocore==1.21.63 # via # awscli # boto3 diff --git a/requirements.txt b/requirements.txt index 49b92583e16..46db7919a7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.8.20 # via -r requirements/base.in -awscli==1.20.62 +awscli==1.20.63 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,9 +14,9 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.18.62 +boto3==1.18.63 # via -r requirements/base.in -botocore==1.21.62 +botocore==1.21.63 # via # awscli # boto3 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 6eb6efdb478..cbc229e3b6d 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -6,7 +6,7 @@ # click==8.0.3 # via pip-tools -pep517==0.11.0 +pep517==0.11.1 # via pip-tools pip-tools==6.4.0 # via -r requirements/pip-tools.in diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index c1031b8ba7c..f50a12fcd1d 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.20.62 +awscli==1.20.63 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,9 +16,9 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.18.62 +boto3==1.18.63 # via -r requirements/jenkins.in -botocore==1.21.62 +botocore==1.21.63 # via # awscli # boto3 From a5fc26fc0c05a43b8361cc566f4d1ac1e3818879 Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Wed, 20 Oct 2021 15:34:23 +0500 Subject: [PATCH 192/664] build: remove sandbox python35 requirements path --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 6c5d46fc13a..2863b481bdd 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1710,7 +1710,7 @@ base_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/base.txt" django_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/django.txt" openstack_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/openstack.txt" -sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/{% if edxapp_sandbox_python_version == 'python3.5' %}py35.txt{% else %}py38.txt{% endif %}" +sandbox_base_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/py38.txt" # The Python requirements files in the order they should be installed. This order should # match the order of PYTHON_REQ_FILES in edx-platform/pavelib/prereqs.py. From db14b77d2e3f3175e4746666d92b19025abfceb7 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 23 Aug 2021 14:55:23 -0400 Subject: [PATCH 193/664] fix: bring neo4j role in line with reality The Neo4j ansible role seems to have drifted from what edX has deployed. These changes bring it closer to what is running on coursegraph.edx.org, and also make it possbile to get CourseGraph working out-of-the-box on an Ubuntu 18.04 sandbox: * Change Neo4j version from 3.2.2 to 3.3.1. * Expose Bolt on 0.0.0.0:7687 with optional encryption. * Enable `dbms.allow_upgrade`, which is the new name of the `dbms.allow_format_migration` key. * Remove http->https redirection logic when NGINX_ENABLE_SSL is false (this is necessary on sandboxes, and also just makes sense... we shouldn't redirect to https if https is disabled). TNL-8386 --- CHANGELOG.md | 8 +++++++ playbooks/roles/neo4j/defaults/main.yml | 5 +++- playbooks/roles/neo4j/tasks/main.yml | 24 +++++++++++++++++-- .../app/nginx/sites-available/coursegraph.j2 | 5 ++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c792fac1e6..60bd8cd0f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-10-20 + - Role neo4j + - Bring Neo4j role closer in with what we really deploy: + - Change Neo4j version from 3.2.2 to 3.3.1. + - Expose Bolt on 0.0.0.0:7687 with optional encryption. + - Enable `dbms.allow_upgrade`, which is the new name of the `dbms.allow_format_migration` key. + - Remove http->https redirection logic when NGINX_ENABLE_SSL is false. + - 2021-09-28 - Role nginx - Add `NGINX_ENABLE_IPV6` configuration variable to make nginx diff --git a/playbooks/roles/neo4j/defaults/main.yml b/playbooks/roles/neo4j/defaults/main.yml index fcc781113e3..76bec8cf8db 100644 --- a/playbooks/roles/neo4j/defaults/main.yml +++ b/playbooks/roles/neo4j/defaults/main.yml @@ -22,8 +22,9 @@ NEO4J_AUTH_ENABLED: "true" neo4j_gpg_key_url: https://debian.neo4j.org/neotechnology.gpg.key neo4j_apt_repository: "deb http://debian.neo4j.org/repo stable/" neo4j_defaults_file: "/etc/default/neo4j" -NEO4J_VERSION: "3.2.2" +NEO4J_VERSION: "3.3.1" neo4j_server_config_file: "/etc/neo4j/neo4j.conf" +neo4j_bolt_port: 7687 # default in package is 7687 neo4j_https_port: 7473 # default in package is 7473 neo4j_http_port: 7474 # default in package is 7474 neo4j_listen_address: "0.0.0.0" @@ -32,6 +33,8 @@ neo4j_page_cache_size: "6000m" neo4j_log_dir: "/var/log/neo4j" # Properties file settings +neo4j_bolt_settings_key: "dbms.connector.bolt.listen_address" +neo4j_bolt_tls_key: "dbms.connector.bolt.tls_level" neo4j_https_settings_key: "dbms.connector.https.listen_address" neo4j_http_settings_key: "dbms.connector.http.listen_address" diff --git a/playbooks/roles/neo4j/tasks/main.yml b/playbooks/roles/neo4j/tasks/main.yml index 69b9941841c..470ee556eea 100644 --- a/playbooks/roles/neo4j/tasks/main.yml +++ b/playbooks/roles/neo4j/tasks/main.yml @@ -98,8 +98,28 @@ - name: allow format migration (when updating neo4j versions) lineinfile: dest: "{{ neo4j_server_config_file }}" - regexp: "dbms.allow_format_migration=" - line: "dbms.allow_format_migration=true" + regexp: "dbms.allow_upgrade=" + line: "dbms.allow_upgrade=true" + tags: + - install + - install:configuration + +- name: set to listen on specific port for bolt + lineinfile: + create: yes + dest: "{{ neo4j_server_config_file }}" + regexp: "{{ neo4j_bolt_settings_key }}=" + line: "{{ neo4j_bolt_settings_key }}={{ neo4j_listen_address }}:{{ neo4j_bolt_port }}" + tags: + - install + - install:configuration + +- name: allow both encrypted and unencrypted bolt connections + lineinfile: + create: yes + dest: "{{ neo4j_server_config_file }}" + regexp: "{{ neo4j_bolt_tls_key }}=" + line: "{{ neo4j_bolt_tls_key }}=OPTIONAL" tags: - install - install:configuration diff --git a/playbooks/roles/neo4j/templates/edx/app/nginx/sites-available/coursegraph.j2 b/playbooks/roles/neo4j/templates/edx/app/nginx/sites-available/coursegraph.j2 index 2318938e106..70ead75ebaf 100644 --- a/playbooks/roles/neo4j/templates/edx/app/nginx/sites-available/coursegraph.j2 +++ b/playbooks/roles/neo4j/templates/edx/app/nginx/sites-available/coursegraph.j2 @@ -39,6 +39,8 @@ server { proxy_pass http://127.0.0.1:{{ neo4j_http_port }}; } + {% if NGINX_ENABLE_SSL %} + # Forward to HTTPS if we're an HTTP request... if ($http_x_forwarded_proto = "http") { set $do_redirect "true"; @@ -48,4 +50,7 @@ server { if ($do_redirect = "true") { rewrite ^ https://$host$request_uri? permanent; } + + {% endif %} + } From 38c8f7a76e36a13b42b77955cc80b4df72d52696 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 9 Aug 2021 17:46:07 -0400 Subject: [PATCH 194/664] fix: upgrade py2neo from 3.3.1 to 3.5.28 After we upgraded edxapp from python3.5 to python3.8, the dump_to_neo4j management command (which we use to dump modulestore data to into coursegraph) stopped working. Our working hypothesis is that the py2neo version that edx-platform pins (3.1.2) does not support python3.8. In order upgrade to a py2neo that python3.8 supports *and* that hasn't been deleted from Neo4j's documentation, we must first upgrade the version of neo4j itself that we install. By upgrading to 3.5.28, we upgrade as far into the 3.x series as possible without having to deal with potential breaking changes in 4.x. According to Neo4j's docs, this should be a painless upgrade, although it will trigger an automatic db migration when neo4j starts up again. It should allow us to upgrade to any py2neo version series between 4.x (the next major version) to 2021.x (the latest). TNL-8386 --- CHANGELOG.md | 4 ++++ playbooks/roles/neo4j/defaults/main.yml | 15 ++++++++++++--- playbooks/roles/neo4j/tasks/main.yml | 10 ++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60bd8cd0f5e..0731d5930ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). - 2021-10-20 + - Role neo4j + - Upgrade Neo4j from 3.2.2 to 3.5.28. + + - 2021-08-26 - Role neo4j - Bring Neo4j role closer in with what we really deploy: - Change Neo4j version from 3.2.2 to 3.3.1. diff --git a/playbooks/roles/neo4j/defaults/main.yml b/playbooks/roles/neo4j/defaults/main.yml index 76bec8cf8db..3c0924f38b2 100644 --- a/playbooks/roles/neo4j/defaults/main.yml +++ b/playbooks/roles/neo4j/defaults/main.yml @@ -19,10 +19,19 @@ NEO4J_SERVER_NAME: "localhost" NEO4J_AUTH_ENABLED: "true" -neo4j_gpg_key_url: https://debian.neo4j.org/neotechnology.gpg.key -neo4j_apt_repository: "deb http://debian.neo4j.org/repo stable/" +# When updating this version, please update the corresponding +# neo4j Docker image tag used by the Devstack coursegraph service +# (see github.com/edx/devstack/tree/master/docker-compose.yml). +# Note that the corresponding docker image tag does not include the +# epoch prefix ('1:') -- it's just 'Major.Minor.Patch'. +NEO4J_VERSION: "1:3.5.28" + +# If upgrading to a Major.Minor series other than 3.5, you'll need +# to change the '3.5' repository component below accordingly. +neo4j_apt_repository: "deb https://debian.neo4j.com stable 3.5" + +neo4j_gpg_key_url: https://debian.neo4j.com/neotechnology.gpg.key neo4j_defaults_file: "/etc/default/neo4j" -NEO4J_VERSION: "3.3.1" neo4j_server_config_file: "/etc/neo4j/neo4j.conf" neo4j_bolt_port: 7687 # default in package is 7687 neo4j_https_port: 7473 # default in package is 7473 diff --git a/playbooks/roles/neo4j/tasks/main.yml b/playbooks/roles/neo4j/tasks/main.yml index 470ee556eea..f5f1d213d01 100644 --- a/playbooks/roles/neo4j/tasks/main.yml +++ b/playbooks/roles/neo4j/tasks/main.yml @@ -65,6 +65,16 @@ register: neo4j_apt_pkg until: neo4j_apt_pkg is succeeded +# For what it's worth: We purposely do not prefix these line-replacement +# regex with ^ or suffix them with $. That's because we cannot be +# confident whether these lines initially (i) exist in file commented-out, +# or (ii) exist in the file with a value already set. So, we purposefully +# leave the regexes without beginning- or end-of-line matches so that +# they can handle both scenario (i) and (ii). +# In the future, it'd be good to get rid of these tasks, and instead +# just include j2-templated configuration files to wholesale replace +# what's on the box. + - name: enable or disable authentication lineinfile: dest: "{{ neo4j_server_config_file }}" From 758c1be2f99c6079fb2e8ecc2788c57a5debeddb Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 22 Oct 2021 15:26:34 +0500 Subject: [PATCH 195/664] chore: enable systemd boot-start splunkforwarder --- playbooks/roles/splunkforwarder/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/splunkforwarder/tasks/main.yml b/playbooks/roles/splunkforwarder/tasks/main.yml index 8c646292d3f..668e72faf78 100644 --- a/playbooks/roles/splunkforwarder/tasks/main.yml +++ b/playbooks/roles/splunkforwarder/tasks/main.yml @@ -69,7 +69,7 @@ when: download_rpm.changed or download_deb.changed - name: Create boot script - shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-unit-file-name SplunkForwarder.service -user splunk --accept-license --answer-yes --no-prompt" + shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-managed 1 -user splunk --accept-license --answer-yes --no-prompt" args: creates: /etc/systemd/system/SplunkForwarder.service register: create_boot_script From 5d36bd2a2e046169e71339b8e8dda597cc181d4a Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Fri, 22 Oct 2021 16:34:50 +0500 Subject: [PATCH 196/664] Revert "chore: enable systemd boot-start splunkforwarder (PSRE-1104)" --- playbooks/roles/splunkforwarder/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/splunkforwarder/tasks/main.yml b/playbooks/roles/splunkforwarder/tasks/main.yml index 668e72faf78..8c646292d3f 100644 --- a/playbooks/roles/splunkforwarder/tasks/main.yml +++ b/playbooks/roles/splunkforwarder/tasks/main.yml @@ -69,7 +69,7 @@ when: download_rpm.changed or download_deb.changed - name: Create boot script - shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-managed 1 -user splunk --accept-license --answer-yes --no-prompt" + shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-unit-file-name SplunkForwarder.service -user splunk --accept-license --answer-yes --no-prompt" args: creates: /etc/systemd/system/SplunkForwarder.service register: create_boot_script From a08ff5cbcd88c41e5b51ff91b06648c2a6012c50 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Mon, 25 Oct 2021 11:01:07 +0500 Subject: [PATCH 197/664] Revert "Revert "chore: enable systemd boot-start splunkforwarder (PSRE-1104)"" --- playbooks/roles/splunkforwarder/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/splunkforwarder/tasks/main.yml b/playbooks/roles/splunkforwarder/tasks/main.yml index 8c646292d3f..668e72faf78 100644 --- a/playbooks/roles/splunkforwarder/tasks/main.yml +++ b/playbooks/roles/splunkforwarder/tasks/main.yml @@ -69,7 +69,7 @@ when: download_rpm.changed or download_deb.changed - name: Create boot script - shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-unit-file-name SplunkForwarder.service -user splunk --accept-license --answer-yes --no-prompt" + shell: "{{ splunkforwarder_output_dir }}/bin/splunk enable boot-start -systemd-managed 1 -user splunk --accept-license --answer-yes --no-prompt" args: creates: /etc/systemd/system/SplunkForwarder.service register: create_boot_script From 4c2787b6baab1d736d97baed04e0428bec3ed48b Mon Sep 17 00:00:00 2001 From: syed-awais-ali Date: Mon, 25 Oct 2021 12:35:41 +0500 Subject: [PATCH 198/664] fix: print statements, brackets --- .../kustomize_update_checker/kustomize_update_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/jenkins/kustomize_update_checker/kustomize_update_checker.py b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py index 88910255049..7b8e6ee5b90 100644 --- a/util/jenkins/kustomize_update_checker/kustomize_update_checker.py +++ b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py @@ -43,7 +43,7 @@ def parse_yaml(file_name): LOGGER.error("error in configuration file: %s" % str(exc)) sys.exit(1) except KeyError as e: - print('I got a KeyError - reason "%s"' % str(e)) + print(f"I got a KeyError - reason {str(e)}") def check_version(app_name, app_version, latest_version): @@ -89,7 +89,7 @@ def send_an_email(to_addr, from_addr, app_list, region): ) message += """""" - print(("Sending the following as email to {}".format(to_addr))) + print(f"Sending the following as email to {to_addr}") print(message) ses_client.send_email( Source=from_addr, From e608703d94759e941704902a3229c93515062329 Mon Sep 17 00:00:00 2001 From: Syed Awais Ali Date: Tue, 26 Oct 2021 15:52:32 +0500 Subject: [PATCH 199/664] chore: tweaking helm script --- .../helm_update_checker/helm_update_checker.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/util/jenkins/helm_update_checker/helm_update_checker.py b/util/jenkins/helm_update_checker/helm_update_checker.py index b83593de8db..b55b9b1a4f3 100644 --- a/util/jenkins/helm_update_checker/helm_update_checker.py +++ b/util/jenkins/helm_update_checker/helm_update_checker.py @@ -44,7 +44,7 @@ def parse_yaml(file_name): LOGGER.error("error in configuration file: %s" % str(exc)) sys.exit(1) except KeyError as e: - print('I got a KeyError - reason "%s"' % str(e)) + print(f"I got a KeyError - reason {str(e)}") def add_helm(repo_url, repo_name): @@ -62,12 +62,12 @@ def update_helm(): def get_repo_name(repo_url): try: - cmd_list = 'helm repo list -o json' - output = subprocess.check_output(cmd_list, shell=True).strip() - output_list = json.loads(output.decode()) - for dic in output_list: - if dic["url"] == repo_url: - return dic['name'] + get_repo_cmd = 'helm repo list -o json' + repositories = subprocess.check_output(get_repo_cmd, shell=True).strip() + repo_list = json.loads(repositories.decode()) + for repo in repo_list: + if repo["url"] == repo_url: + repo['name'] except subprocess.CalledProcessError as e: print(e.output) @@ -87,9 +87,7 @@ def check_version(chart_name, app_name, repo_name, app_version): def compare_version(current_version, latest_version): - if current_version == latest_version: - return True - return False + return True if current_version == latest_version else False def find(name, path): From 91c258d6178240064741b9c8fc2555afc8ffc4ef Mon Sep 17 00:00:00 2001 From: Maari <56252537+mrtmm@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:44:52 +0300 Subject: [PATCH 200/664] feat: Add IPv6 support to MFE's (#6589) This is a follow-up change to https://github.com/edx/configuration/pull/6557 which added an option to make nginx listen on both, IPv4 and IPv6 wildcard addresses. This change adds that option for MFE's as well. --- .../mfe/templates/edx/app/nginx/sites-available/app.j2 | 6 ++++++ .../templates/edx/app/nginx/sites-available/app.j2 | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/playbooks/roles/mfe/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/mfe/templates/edx/app/nginx/sites-available/app.j2 index e4003a149c0..6b11cd67693 100644 --- a/playbooks/roles/mfe/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/mfe/templates/edx/app/nginx/sites-available/app.j2 @@ -5,6 +5,9 @@ server { server_name {{ MFE_HOSTNAME }}; listen {{ MFE_NGINX_PORT }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ MFE_NGINX_PORT }}; + {% endif %} # Increase accepted header size to account for overenthusiastic usage of cookies large_client_header_buffers 8 16k; @@ -29,6 +32,9 @@ server { server { server_name {{ MFE_HOSTNAME }}; listen {{ MFE_SSL_NGINX_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ MFE_SSL_NGINX_PORT }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; diff --git a/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/app.j2 index d748d86ff21..f38939d92d5 100644 --- a/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/mfe_deployer/templates/edx/app/nginx/sites-available/app.j2 @@ -5,6 +5,9 @@ server { server_name {{ MFE_DEPLOY_COMMON_HOSTNAME }}; listen {{ MFE_DEPLOY_NGINX_PORT }}; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ MFE_DEPLOY_NGINX_PORT }}; + {% endif %} {% if NGINX_ENABLE_SSL %} {% include "concerns/handle-ip-disclosure.j2" %} @@ -24,6 +27,9 @@ server { server { server_name {{ MFE_DEPLOY_COMMON_HOSTNAME }}; listen {{ MFE_DEPLOY_SSL_NGINX_PORT }} ssl; + {% if NGINX_ENABLE_IPV6 %} + listen [::]:{{ MFE_DEPLOY_SSL_NGINX_PORT }} ssl; + {% endif %} ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; From e38c0dba2fc9f0f5d8fdefe9f6f438621c586004 Mon Sep 17 00:00:00 2001 From: irfanuddinahmad Date: Mon, 1 Nov 2021 13:41:16 +0500 Subject: [PATCH 201/664] fix: Updated tableau version --- playbooks/roles/tableau/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/tableau/defaults/main.yml b/playbooks/roles/tableau/defaults/main.yml index f38d29a7b44..f58df8ee5e0 100644 --- a/playbooks/roles/tableau/defaults/main.yml +++ b/playbooks/roles/tableau/defaults/main.yml @@ -25,7 +25,7 @@ tableau_user_shell: '/bin/false' tableau_repo: "https://github.com/tableau/server-install-script-samples.git" tableau_version: 'master' -tableau_server_deb_pkg: 'https://downloads.tableau.com/esdalt/2020.4.5/tableau-server-2020-4-5_amd64.deb' +tableau_server_deb_pkg: 'https://downloads.tableau.com/esdalt/2021.3.2/tableau-server-2021-3-2_amd64.deb' tableau_server_deb_pkg_name: 'tableau-server-amd64.deb' TABLEAU_SERVER_ADMIN_USER: "tableau001" From 12616dcb32466b8fb59e6cae3a9da293995106c6 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 1 Nov 2021 20:09:03 -0400 Subject: [PATCH 202/664] refactor(docker): remove CELERY_QUEUES overrides for lms, studio (#6604) Instead, just use the default value of `CELERY_QUEUES` as set in edx-platform's settings files. Functionally, this means that in addition to the existing queues that LMS and Studio define, there is now a "low priority" queue for Studio, suitable for tasks like a CourseGraph dump. --- CHANGELOG.md | 9 +++++++++ docker/build/edxapp/lms.yml | 4 ---- docker/build/edxapp/studio.yml | 3 --- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0731d5930ea..840c8739158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-11-01 + - Docker: edxapp + - Removed unnecessary `CELERY_QUEUES` overrides for LMS and Studio. + Instead, just use the default value of `CELERY_QUEUES` as set in + edx-platform's settings files. + Functionally, this means that in addition to the existing queues + that LMS and Studio defined, there is now a "low priority" queue + for Studio, suitable for tasks like a CourseGraph dump. + - 2021-10-20 - Role neo4j - Upgrade Neo4j from 3.2.2 to 3.5.28. diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 3c48a6cdefc..6572821b61b 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -107,10 +107,6 @@ CELERY_BROKER_USER: celery CELERY_BROKER_USE_SSL: false CELERY_BROKER_VHOST: '' CELERY_EVENT_QUEUE_TTL: null -CELERY_QUEUES: -- edx.lms.core.default -- edx.lms.core.high -- edx.lms.core.high_mem CELERY_TIMEZONE: UTC CERTIFICATE_TEMPLATE_LANGUAGES: en: English diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 26cf0483a1b..6ca98c522bc 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -88,9 +88,6 @@ CELERY_BROKER_USER: celery CELERY_BROKER_USE_SSL: false CELERY_BROKER_VHOST: '' CELERY_EVENT_QUEUE_TTL: null -CELERY_QUEUES: -- edx.cms.core.default -- edx.cms.core.high CELERY_TIMEZONE: UTC CERTIFICATE_TEMPLATE_LANGUAGES: en: English From 5a4480751ac6c5f498732ffefeb89e98170edc0d Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 3 Nov 2021 17:00:44 +0500 Subject: [PATCH 203/664] docs: add mongo 4.2 upgrade ADR --- documentation/decisions/0000-placeholder.rst | 0 .../decisions/0001-mongo-4.2-upgrade.rst | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+) delete mode 100644 documentation/decisions/0000-placeholder.rst create mode 100644 documentation/decisions/0001-mongo-4.2-upgrade.rst diff --git a/documentation/decisions/0000-placeholder.rst b/documentation/decisions/0000-placeholder.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/documentation/decisions/0001-mongo-4.2-upgrade.rst b/documentation/decisions/0001-mongo-4.2-upgrade.rst new file mode 100644 index 00000000000..5975c5d8db8 --- /dev/null +++ b/documentation/decisions/0001-mongo-4.2-upgrade.rst @@ -0,0 +1,23 @@ +Mongo 4.2 Upgrade +-------------------------------- +Status +===== +Accepted + +Context +======= + +The Open edX community relese Maple will be released on December 9th 2021. Mongo version 4.0 security support will be ended on 30 Apr 2022 ( https://endoflife.date/mongodb ) and Mongo 4.0 End-of-Life (EOL) is before we move for a new Open edX release after Maple. +Maple should be running with supported version of Mongo during its lifecycle. + +Decision +======== + +Initial plan was to upgrade Mongo to 4.4.As, we are running mongo community edition in all our environments and mongo cloud manager is breaking the support of incremental backup starting from Mongo 4.2 with FCV (4.2). To run backup and restore for MongoDB 4.2 or later with FCV 4.2 or later Mongo should be running with MongoDB enterprise edition and must run with MongoDB agent on every node of cluster. Because of these requirements and limitations we have decided to upgrade MongoDB to 4.2 with FCV 4.0. + +Requirements +=========== + +* To upgrade an existing MongoDB deployment to 4.2, earlier version should run Mongo 4.0-series. +* MongoDB driver should compatible for MongoDB 4.2. + Check python driver (pymongo) compatibility for edx-platform (https://docs.mongodb.com/drivers/pymongo/#compatibility) and ruby driver (mongoid) for cs_comment_service here (https://docs.mongodb.com/mongoid/master/reference/compatibility/#mongodb-server-compatibility) From 8d0a7befaa563ce0c27fd6bb38fcac26094969a4 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 3 Nov 2021 11:22:20 -0400 Subject: [PATCH 204/664] fix: ensure that neo4j auto-starts when coursegraph boots (#6607) A freshly built and run CourseGraph box will start the Neo4j application. However, after rebooting the box, Neo4j needed to be explicitly started again. This 'enables' the Neo4j service, which means that it'll now auto-start whenever CourseGraph is booted up. --- playbooks/roles/neo4j/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/neo4j/tasks/main.yml b/playbooks/roles/neo4j/tasks/main.yml index f5f1d213d01..c6e26e7f8a8 100644 --- a/playbooks/roles/neo4j/tasks/main.yml +++ b/playbooks/roles/neo4j/tasks/main.yml @@ -199,6 +199,7 @@ service: name: neo4j state: restarted + enabled: yes tags: - manage - manage:start From e82f2700780d4cf0624c74f69072eaa4ec2562c4 Mon Sep 17 00:00:00 2001 From: pkulkark Date: Mon, 8 Nov 2021 16:59:12 +0530 Subject: [PATCH 205/664] feat: Add completion aggregator URL Needed for displaying progress bar --- playbooks/roles/edxapp/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 2863b481bdd..d856f6bda16 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1068,6 +1068,9 @@ EDXAPP_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" # Dashboard courses default limit EDXAPP_DASHBOARD_COURSE_LIMIT: null +# Completion Aggregator URL needed for progress bar +EDXAPP_COMPLETION_AGGREGATOR_URL: null + #-------- Everything below this line is internal to the role ------------ #Use YAML references (& and *) and hash merge <<: to factor out shared settings From 6293ff1260c4088dfaa912cf9d71c8fab93d8ec0 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Wed, 10 Nov 2021 12:37:16 +0500 Subject: [PATCH 206/664] chore: Update jenkins plugins --- playbooks/roles/jenkins_build/defaults/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 93b7d8469c6..06d337f5b5a 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -160,7 +160,7 @@ build_jenkins_plugins_list: version: '1.21' group: 'org.jenkins-ci.plugins' - name: 'jackson2-api' - version: '2.13.1' + version: '2.12.1' group: 'org.jenkins-ci.plugins' - name: 'javadoc' version: '1.3' @@ -268,7 +268,7 @@ build_jenkins_plugins_list: version: '1.17' group: 'org.jenkins-ci.plugins' - name: 'ssh-credentials' - version: '1.17.3' + version: '1.18.1' group: 'org.jenkins-ci.plugins' - name: 'ssh-slaves' version: '1.28.1' @@ -277,7 +277,7 @@ build_jenkins_plugins_list: version: '1.23' group: 'org.jenkins-ci.plugins' - name: 'subversion' - version: '2.13.2' + version: '2.15.1' group: 'org.jenkins-ci.plugins' - name: 'text-finder' version: '1.10' @@ -294,6 +294,9 @@ build_jenkins_plugins_list: - name: 'translation' version: '1.16' group: 'org.jenkins-ci.plugins' + - name: 'trilead-api' + version: '1.0.13' + group: 'org.jenkins-ci.plugins' - name: 'variant' version: '1.4' group: 'org.jenkins-ci.plugins' From cc101f4360a5f22d78a6a5f11451754e43d3153c Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Wed, 17 Nov 2021 12:35:50 -0500 Subject: [PATCH 207/664] chore: Bump AMI creation timeout from 60 to 90 minutes. --- playbooks/continuous_delivery/create_ami.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index d7c9be1e309..e1f45874f68 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -37,7 +37,7 @@ vars: ec2_region: us-east-1 ami_wait: yes - ami_creation_timeout: 3600 + ami_creation_timeout: 5400 no_reboot: no artifact_path: /tmp/ansible-runtime extra_name_identifier: 0 From 67953d0ac4cd70e455316328e81024dbf2e75510 Mon Sep 17 00:00:00 2001 From: Jazib Humayun Date: Thu, 11 Nov 2021 18:00:47 +0500 Subject: [PATCH 208/664] feat: tableau DE ansible role Added tableau DE playbook that runs uses default tableau role and then runs tableau DE role to add/install features specific to DE. This helps us spinning up instance quickly and make Tableau instance ready to use --- .../aws_cloudwatch_agent/defaults/main.yml | 13 ++++ .../roles/aws_cloudwatch_agent/tasks/main.yml | 60 +++++++++++++++++++ .../templates/amazon-cloudwatch-agent.json | 36 +++++++++++ playbooks/roles/tableau_de/defaults/main.yml | 22 +++++++ playbooks/roles/tableau_de/meta/main.yml | 3 + playbooks/roles/tableau_de/tasks/main.yml | 59 ++++++++++++++++++ playbooks/tableau_de.yml | 11 ++++ 7 files changed, 204 insertions(+) create mode 100644 playbooks/roles/aws_cloudwatch_agent/defaults/main.yml create mode 100644 playbooks/roles/aws_cloudwatch_agent/tasks/main.yml create mode 100644 playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json create mode 100644 playbooks/roles/tableau_de/defaults/main.yml create mode 100644 playbooks/roles/tableau_de/meta/main.yml create mode 100644 playbooks/roles/tableau_de/tasks/main.yml create mode 100644 playbooks/tableau_de.yml diff --git a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml new file mode 100644 index 00000000000..4fa14e6cdf9 --- /dev/null +++ b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml @@ -0,0 +1,13 @@ +--- + +# +# Defaults specified here should not contain +# any secrets or host identifying information. +# + +# The Amazon CloudWatch Agent URL +cloudwatch_url: https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb + +# Collectd installation parameters +collectd_version: "5.7.2-2ubuntu1" +collectd_install_recommends: yes diff --git a/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml new file mode 100644 index 00000000000..72f136a3aef --- /dev/null +++ b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml @@ -0,0 +1,60 @@ +--- + +- name: Install Collectd to use with AWS CloudWatch Agent + apt: + name: collectd={{ collectd_version }} + install_recommends: "{{ collectd_install_recommends }}" + state: present + tags: + - install + - install:base + +- name: Install xz-utils (required when using deb parameter of apt module) + apt: + name: xz-utils + state: present + tags: + - install + - install:base + +- name: Download the AWS CloudWatch Agent Debian package + get_url: + url: "{{ cloudwatch_url }}" + dest: /tmp/amazon-cloudwatch-agent.deb + tags: + - install + - install:base + +- name: Install AWS CloudWatch Agent Debian package + apt: + deb: /tmp/amazon-cloudwatch-agent.deb + state: present + tags: + - install + - install:base + +- name: Copy amazon-cloudwatch-agent template + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode }}" + with_items: + - { src: 'amazon-cloudwatch-agent.json', dest: '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', mode: '0644' } + tags: + - install + - install:base + +- name: Enable AWS CloudWatch Agent + service: + name: amazon-cloudwatch-agent + enabled: yes + tags: + - install + - install:base + +- name: Run AWS CloudWatch Agent + shell: " /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json" + become: yes + tags: + - install + - install:code diff --git a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json new file mode 100644 index 00000000000..1ae641247dc --- /dev/null +++ b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json @@ -0,0 +1,36 @@ +{ + "agent": { + "metrics_collection_interval": 60, + "run_as_user": "root" + }, + "metrics": { + "append_dimensions": { + "InstanceId": "${aws:InstanceId}" + }, + "metrics_collected": { + "collectd": { + "metrics_aggregation_interval": 60 + }, + "disk": { + "measurement": [ + "used_percent" + ], + "metrics_collection_interval": 60, + "resources": [ + "*" + ] + }, + "mem": { + "measurement": [ + "mem_used_percent" + ], + "metrics_collection_interval": 60 + }, + "statsd": { + "metrics_aggregation_interval": 60, + "metrics_collection_interval": 60, + "service_address": ":8125" + } + } + } +} diff --git a/playbooks/roles/tableau_de/defaults/main.yml b/playbooks/roles/tableau_de/defaults/main.yml new file mode 100644 index 00000000000..48c088ab8fb --- /dev/null +++ b/playbooks/roles/tableau_de/defaults/main.yml @@ -0,0 +1,22 @@ +# variables common to the tableau_de role, automatically loaded +# when the role is included +--- + +# +# Defaults specified here should not contain +# any secrets or host identifying information. +# + +tableau_user_group: "tableau" +tableau_user: "{{ TABLEAU_ADMIN_USER }}" + +os_default_user: "ubuntu" + +# Snowflake driver parameter +snowflake_odbc_driver_endpoint: "https://sfc-repo.snowflakecomputing.com/odbc/linux" +snowflake_driver_version: "2.24.2" + +# Postgre driver parameter +postgre_driver_endpoint: "https://downloads.tableau.com/drivers/linux/postgresql" +postgre_driver_version: "42.2.22" + diff --git a/playbooks/roles/tableau_de/meta/main.yml b/playbooks/roles/tableau_de/meta/main.yml new file mode 100644 index 00000000000..202420c7af1 --- /dev/null +++ b/playbooks/roles/tableau_de/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - aws_cloudwatch_agent \ No newline at end of file diff --git a/playbooks/roles/tableau_de/tasks/main.yml b/playbooks/roles/tableau_de/tasks/main.yml new file mode 100644 index 00000000000..08fce68c5f1 --- /dev/null +++ b/playbooks/roles/tableau_de/tasks/main.yml @@ -0,0 +1,59 @@ +--- + +- name: Install AWS CLI + apt: + name: awscli + state: present + tags: + - install + - install:base + +- name: Add OS default user in tableau group + shell: "usermod -a -G {{ tableau_user_group }} {{ os_default_user }}" + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Enable tableau user shell + shell: "usermod -s /bin/bash {{ tableau_user }}" + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Create directory for Postgre driver if it does not exist + file: + path: /opt/tableau/tableau_driver/jdbc + state: directory + mode: '0777' + tags: + - install + - install:base + +- name: Download Postgre driver jar file + get_url: + url: "{{postgre_driver_endpoint}}/postgresql-{{postgre_driver_version}}.jar" + dest: /opt/tableau/tableau_driver/jdbc/postgresql-{{postgre_driver_version}}.jar + become_user: "{{ tableau_user }}" + tags: + - install + - install:base + +- name: Download Snowflake odbc driver + get_url: + url: "{{snowflake_odbc_driver_endpoint}}/{{snowflake_driver_version}}/snowflake-odbc-{{snowflake_driver_version}}.x86_64.deb" + dest: /tmp/snowflake-odbc-{{snowflake_driver_version}}.x86_64.deb + tags: + - install + - install:base + +- name: Install Snowflake odbc driver + dpkg_selections: + name: snowflake-odbc-{{snowflake_driver_version}}.x86_64.deb + selection: install + tags: + - install + - install:base diff --git a/playbooks/tableau_de.yml b/playbooks/tableau_de.yml new file mode 100644 index 00000000000..1b293a816f5 --- /dev/null +++ b/playbooks/tableau_de.yml @@ -0,0 +1,11 @@ +- name: Deploy tableau + hosts: all + become: True + gather_facts: True + vars: + serial_count: 1 + CLUSTER_NAME: 'tableau' + serial: "{{ serial_count }}" + roles: + - tableau + - tableau_de From d1f7b40c4be3aa674b1e0dc4da3653e72a9b07f6 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 18 Nov 2021 18:38:57 -0500 Subject: [PATCH 209/664] chore: Upgrade Build Jenkins plugins ISRE-1115 --- .../roles/jenkins_build/defaults/main.yml | 105 +++++++++--------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 06d337f5b5a..59c971ae9f0 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -34,10 +34,10 @@ build_jenkins_plugins_list: version: '1.96' group: 'org.jvnet.hudson.plugins' - name: 'ansicolor' - version: '0.5.2' + version: '1.0.0' group: 'org.jenkins-ci.plugins' - name: 'ant' - version: '1.8' + version: '1.12' group: 'org.jenkins-ci.plugins' - name: 'antisamy-markup-formatter' version: '2.1' @@ -46,22 +46,22 @@ build_jenkins_plugins_list: version: '1.28.1' group: 'org.jenkins-ci.plugins' - name: 'aws-java-sdk' - version: '1.11.700' + version: '1.12.101-300.vc09c7be9cb57' group: 'org.jenkins-ci.plugins' - name: 'badge' - version: '1.5' + version: '1.9' group: 'org.jenkins-ci.plugins' - name: 'bootstrap5-api' version: '5.1.1-1' group: 'io.jenkins.plugins' - name: 'bouncycastle-api' - version: '2.18' + version: '2.25' group: 'org.jenkins-ci.plugins' - name: 'build-name-setter' - version: '1.3' + version: '2.1.0' group: 'org.jenkins-ci.plugins' - name: 'build-timeout' - version: '1.19' + version: '1.20' group: 'org.jenkins-ci.plugins' - name: 'build-user-vars-plugin' version: '1.5' @@ -73,37 +73,37 @@ build_jenkins_plugins_list: version: '6.16' group: 'org.jenkins-ci.plugins' - name: 'cobertura' - version: '1.16' + version: '1.17' group: 'org.jenkins-ci.plugins' - name: 'code-coverage-api' version: '2.0.2' group: 'io.jenkins.plugins' - name: 'copyartifact' - version: '1.44' + version: '1.46.2' group: 'org.jenkins-ci.plugins' - name: 'credentials' version: '2.6.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' - version: '1.23' + version: '1.27' group: 'org.jenkins-ci.plugins' - name: 'cvs' - version: '2.17' + version: '2.19' group: 'org.jenkins-ci.plugins' - name: 'data-tables-api' - version: '1.11.2-1' + version: '1.11.3-1' group: 'io.jenkins.plugins' - name: 'display-url-api' version: '2.3.5' group: 'org.jenkins-ci.plugins' - name: 'docker-commons' - version: '1.8' + version: '1.14' group: 'org.jenkins-ci.plugins' - name: 'ec2' - version: '1.50.3' + version: '1.58' group: 'org.jenkins-ci.plugins' - name: 'email-ext' - version: '2.66' + version: '2.62' group: 'org.jenkins-ci.plugins' - name: 'envinject' version: '2.3.0' @@ -115,19 +115,19 @@ build_jenkins_plugins_list: version: '0.8' group: 'org.jenkins-ci.plugins' - name: 'external-monitor-job' - version: '1.4' + version: '1.7' group: 'org.jenkins-ci.plugins' - name: 'flexible-publish' - version: '0.15.2' + version: '0.16.1' group: 'org.jenkins-ci.plugins' - name: 'font-awesome-api' version: '5.15.4-1' group: 'io.jenkins.plugins' - name: 'forensics-api' - version: '1.4.0' + version: '1.5.0' group: 'io.jenkins.plugins' - name: 'ghprb' - version: '1.42.1' + version: '1.42.2' group: 'org.jenkins-ci.plugins' - name: 'git' version: '4.2.2' @@ -139,10 +139,10 @@ build_jenkins_plugins_list: version: '1.29.2' group: 'com.coravy.hudson.plugins.github' - name: 'github-api' - version: '1.111' + version: '1.133' group: 'org.jenkins-ci.plugins' - name: 'github-branch-source' - version: '2.3.6' + version: '2.9.9' group: 'org.jenkins-ci.plugins' - name: 'github-oauth' version: '0.33' @@ -151,25 +151,25 @@ build_jenkins_plugins_list: version: '1.29' group: 'org.jenkins-ci.plugins' - name: 'groovy' - version: '2.2' + version: '2.4' group: 'org.jenkins-ci.plugins' - name: 'groovy-postbuild' - version: '2.4' + version: '2.5' group: 'org.jvnet.hudson.plugins' - name: 'htmlpublisher' - version: '1.21' + version: '1.25' group: 'org.jenkins-ci.plugins' - name: 'jackson2-api' - version: '2.12.1' + version: '2.13.0' group: 'org.jenkins-ci.plugins' - name: 'javadoc' - version: '1.3' + version: '1.6' group: 'org.jenkins-ci.plugins' - name: 'jobConfigHistory' - version: '2.24' + version: '2.19' group: 'org.jenkins-ci.plugins' - name: 'jdk-tool' - version: '1.2' + version: '1.5' group: 'org.jenkins-ci.plugins' - name: 'job-dsl' version: '1.77' @@ -178,10 +178,10 @@ build_jenkins_plugins_list: version: '3.6.0-2' group: 'io.jenkins.plugins' - name: 'junit' - version: '1.30' + version: '1.53' group: 'org.jenkins-ci.plugins' - name: 'ldap' - version: '1.20' + version: '1.26' group: 'org.jenkins-ci.plugins' - name: 'lockable-resources' version: '2.10' @@ -193,7 +193,7 @@ build_jenkins_plugins_list: version: '2.13' group: 'org.jenkins-ci.plugins' - name: 'matrix-auth' - version: '2.6.6' + version: '2.6.8' group: 'org.jenkins-ci.plugins' - name: 'matrix-project' version: '1.18' @@ -202,49 +202,49 @@ build_jenkins_plugins_list: version: '3.4' group: 'org.jenkins-ci.main' - name: 'monitoring' - version: '1.76.0' + version: '1.88.0' group: 'org.jvnet.hudson.plugins' - name: 'multiple-scms' version: '0.6' group: 'org.jenkins-ci.plugins' - name: 'nodelabelparameter' - version: '1.7.2' + version: '1.9.2' group: 'org.jenkins-ci.plugins' - name: 'pam-auth' - version: '1.5.1' + version: '1.6.1' group: 'org.jenkins-ci.plugins' - name: 'parameterized-trigger' - version: '2.35.2' + version: '2.39' group: 'org.jenkins-ci.plugins' - name: 'pipeline-model-definition' version: '1.3.4.1' group: 'org.jenkinsci.plugins' - name: 'pipeline-utility-steps' - version: '2.0.2' + version: '2.10.0' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' version: '2.5.0' group: 'io.jenkins.plugins' - name: 'popper2-api' - version: '2.10.1-1' + version: '2.10.2-1' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' group: 'org.jenkins-ci.plugins' - name: 'promoted-builds' - version: '3.9.1' + version: '3.10' group: 'org.jenkins-ci.plugins' - name: 'pyenv-pipeline' - version: '2.0.1' + version: '2.1.2' group: 'org.jenkins-ci.plugins' - name: 'rebuild' - version: '1.31' + version: '1.32' group: 'com.sonyericsson.hudson.plugins.rebuild' - name: 'resource-disposer' - version: '0.12' + version: '0.16' group: 'org.jenkins-ci.plugins' - name: 'run-condition' - version: '1.0' + version: '1.5' group: 'org.jenkins-ci.plugins' - name: 'script-security' version: '1.78' @@ -259,13 +259,13 @@ build_jenkins_plugins_list: version: '1.29.1' group: 'io.jenkins.plugins' - name: 'splunk-devops' - version: '1.8.1' + version: '1.9.7' group: 'com.splunk.splunkins' - name: 'splunk-devops-extend' - version: '1.8.1' + version: '1.9.7' group: 'com.splunk.splunkins' - name: 'ssh-agent' - version: '1.17' + version: '1.22' group: 'org.jenkins-ci.plugins' - name: 'ssh-credentials' version: '1.18.1' @@ -280,13 +280,13 @@ build_jenkins_plugins_list: version: '2.15.1' group: 'org.jenkins-ci.plugins' - name: 'text-finder' - version: '1.10' + version: '1.17' group: 'org.jenkins-ci.plugins' - name: 'throttle-concurrents' version: '2.0.1' group: 'org.jenkins-ci.plugins' - name: 'timestamper' - version: '1.11.2' + version: '1.14' group: 'org.jenkins-ci.plugins' - name: 'token-macro' version: '2.13' @@ -303,11 +303,8 @@ build_jenkins_plugins_list: - name: 'violations' version: '0.7.11' group: 'org.jenkins-ci.plugins' - - name: 'warnings' - version: '5.0.2' - group: 'org.jvnet.hudson.plugins' - name: 'warnings-ng' - version: '8.10.0' + version: '9.5.1' group: 'io.jenkins.plugins' - name: 'workflow-aggregator' version: '2.6' @@ -316,10 +313,10 @@ build_jenkins_plugins_list: version: '2.47' group: 'org.jenkins-ci.plugins.workflow' - name: 'windows-slaves' - version: '1.3.1' + version: '1.8' group: 'org.jenkins-ci.plugins' - name: 'workflow-cps' - version: '2.94' + version: '2633.v6baeedc13805' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-cps-global-lib' version: '2.15' @@ -337,7 +334,7 @@ build_jenkins_plugins_list: version: '3.8' group: 'org.jenkins-ci.plugins.workflow' - name: 'ws-cleanup' - version: '0.37' + version: '0.39' group: 'org.jenkins-ci.plugins' - name: 'xunit' version: '1.93' From fd089804536961909131e8b298dba98c2f2dec27 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 22 Nov 2021 14:07:01 -0500 Subject: [PATCH 210/664] chore: Update build jenkins version ISRE-1115 --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 59c971ae9f0..3fb568eb311 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.263.4 +BUILD_JENKINS_VERSION: jenkins_2.303.3 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: From 955c2925e9e61844608acef369f7df468d6ceca8 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 24 Nov 2021 10:39:31 -0500 Subject: [PATCH 211/664] fix: Match version with build jenkins current --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 3fb568eb311..a477cf1277f 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -103,7 +103,7 @@ build_jenkins_plugins_list: version: '1.58' group: 'org.jenkins-ci.plugins' - name: 'email-ext' - version: '2.62' + version: '2.66' group: 'org.jenkins-ci.plugins' - name: 'envinject' version: '2.3.0' From 2ca8973d3ea34528e339c17118cb3ee6e18bc112 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 24 Nov 2021 13:09:59 -0500 Subject: [PATCH 212/664] fix: Remove python bootstrap I don't think this is needed on Ubuntu 20.04 and it breaks the playbook --- playbooks/jenkins_build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/playbooks/jenkins_build.yml b/playbooks/jenkins_build.yml index 52d3a267cb7..dc9c07d80b6 100644 --- a/playbooks/jenkins_build.yml +++ b/playbooks/jenkins_build.yml @@ -1,11 +1,4 @@ --- -- name: Bootstrap instance(s) - hosts: all - gather_facts: no - become: True - roles: - - python - - name: Configure instance(s) hosts: all become: True From 2d3bb0dc90a1e23e4f9d2c720137f532e92a8794 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 24 Nov 2021 14:28:52 -0500 Subject: [PATCH 213/664] fix: rever ec2 plugin upgrade --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index a477cf1277f..613198539ef 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -100,7 +100,7 @@ build_jenkins_plugins_list: version: '1.14' group: 'org.jenkins-ci.plugins' - name: 'ec2' - version: '1.58' + version: '1.50.3' group: 'org.jenkins-ci.plugins' - name: 'email-ext' version: '2.66' From e29b15d32c695891de7884b88a81bf1ee28225fa Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 29 Nov 2021 15:15:03 +0500 Subject: [PATCH 214/664] chore: Fix helm script --- util/jenkins/helm_update_checker/helm_update_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/helm_update_checker/helm_update_checker.py b/util/jenkins/helm_update_checker/helm_update_checker.py index b55b9b1a4f3..8f46f1eb9c9 100644 --- a/util/jenkins/helm_update_checker/helm_update_checker.py +++ b/util/jenkins/helm_update_checker/helm_update_checker.py @@ -67,7 +67,7 @@ def get_repo_name(repo_url): repo_list = json.loads(repositories.decode()) for repo in repo_list: if repo["url"] == repo_url: - repo['name'] + return repo['name'] except subprocess.CalledProcessError as e: print(e.output) From 508236e326981a5e1865e5633e6755769541bcea Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 29 Nov 2021 16:18:53 +0500 Subject: [PATCH 215/664] chore: Fix kustomize script --- .../kustomize_update_checker/kustomize_update_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/kustomize_update_checker/kustomize_update_checker.py b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py index 7b8e6ee5b90..b8b720f219f 100644 --- a/util/jenkins/kustomize_update_checker/kustomize_update_checker.py +++ b/util/jenkins/kustomize_update_checker/kustomize_update_checker.py @@ -120,7 +120,7 @@ def send_an_email(to_addr, from_addr, app_list, region): @click.option('--recipient', multiple=True, help='Recipient Email address') @click.option('--sender', multiple=True, help='Sender email address') def controller(file_name, file_path, region, recipient, sender): - parse_yaml(file_name) + parse_yaml(file_path + "/" + file_name) if len(global_list) > 0: send_an_email(recipient[0], sender[0], global_list, region[0]) From ed8ad2d08c4f32454cd5664042faa6db03d4178f Mon Sep 17 00:00:00 2001 From: Jesse Day Date: Mon, 29 Nov 2021 14:53:55 -0500 Subject: [PATCH 216/664] chore: Update the prospectus node version. Updated to match the version in local dev environments for prospectus. --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 2681efde7d4..2d569919794 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '16.7.0' +PROSPECTUS_NODE_VERSION: '16.13.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From 814efb624ed0e411056d392707e64096ab80294a Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Thu, 2 Dec 2021 12:34:05 +0500 Subject: [PATCH 217/664] chore: Update token macro plugin --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 613198539ef..9f9d7865138 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -289,7 +289,7 @@ build_jenkins_plugins_list: version: '1.14' group: 'org.jenkins-ci.plugins' - name: 'token-macro' - version: '2.13' + version: '267.vcdaea6462991' group: 'org.jenkins-ci.plugins' - name: 'translation' version: '1.16' From b8ab26c654a54d754cd01448f72f8b783c95ef6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Boros?= Date: Tue, 7 Dec 2021 01:31:18 +0100 Subject: [PATCH 218/664] [SE-4860] make SINGLE_BEAT_IDENTIFIER configurable (#6586) * feat: make SINGLE_BEAT_IDENTIFIER configurable * fix: celery 5.0+ does not support --app as a worker flag * refactor: remove duplicated app declaration * fix: --config flag is global since Celery 5.0+ * fix: replace single-beat with an updated version * fix: add missing comma * fix: replace maxtasksperchild worker flag * refactor: address PR review comments Signed-off-by: Gabor Boros * refactor: remove celery<5 adjustments Signed-off-by: Gabor Boros * fix: set single-beat autorestart Signed-off-by: Gabor Boros --- CHANGELOG.md | 4 +++ .../edx/app/ecomworker/ecomworker.sh.j2 | 2 +- .../conf.d.available/app-workers.conf.j2 | 14 ++------- playbooks/roles/edxapp/defaults/main.yml | 6 +++- playbooks/roles/edxapp/tasks/deploy.yml | 3 +- .../edx/app/edxapp/beat_scheduler.sh.j2 | 6 ++-- .../templates/edx/app/edxapp/worker.sh.j2 | 4 +-- .../conf.d.available/workers.conf.j2 | 30 ++++++++++--------- 8 files changed, 34 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840c8739158..90a533b4c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2021-11-30 + - Upgrade celery to 5.2.0 and adjust CLI call parameters too + - Bumped single-beat to use a more supported fork of the project + - 2021-11-01 - Docker: edxapp - Removed unnecessary `CELERY_QUEUES` overrides for LMS and Studio. diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index d2c355d1c97..516e775452c 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -17,4 +17,4 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env # We exec so that celery is the child of supervisor and can be managed properly -exec {{ executable }} -A ecommerce_worker worker --app ecommerce_worker.celery_app:app --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing +exec {{ executable }} --app ecommerce_worker.celery_app:app worker -A ecommerce_worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing diff --git a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 index 6ec910b4254..437c590ac62 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 @@ -10,14 +10,9 @@ directory={{ edx_django_service_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ edx_django_service_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency=1 {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} +command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ edx_django_service_name }}.celery:app worker -A {{ edx_django_service_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency=1 {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }} -; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit -; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers -; are auto restarted even when exiting with code 0. -; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. -autorestart=true {% endfor %} {% endfor %} @@ -39,14 +34,9 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log {% set worker_app_name = 'course_discovery' %} {% endif %} -command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh worker -A {{ edx_django_service_name }} --app {{ worker_app_name }}.celery:app --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} +command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker -A {{ edx_django_service_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }} -; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit -; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers -; are auto restarted even when exiting with code 0. -; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. -autorestart=true {% endfor %} diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 2863b481bdd..874b118f95a 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -185,7 +185,11 @@ EDXAPP_ENABLE_CELERY_BEAT: false EDXAPP_SINGLE_BEAT_LOCK_TIME: 60 # EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2 EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL: 29 -EDXAPP_SINGLE_BEAT_VERSION: "0.4.2" +EDXAPP_SINGLE_BEAT_REPO: "https://github.com/akachanov/single-beat.git" +EDXAPP_SINGLE_BEAT_VERSION: "e500ac4b56756cdf96836666883af8060aaef455" +EDXAPP_SINGLE_BEAT_USER: "{{ EDXAPP_CELERY_USER }}" +EDXAPP_SINGLE_BEAT_PASSWORD: "{{ EDXAPP_CELERY_PASSWORD }}" +EDXAPP_SINGLE_BEAT_IDENTIFIER: "celerybeat" EDXAPP_BRANCH_IO_KEY: "" EDXAPP_AUTH_USE_OPENID_PROVIDER: true diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index c1446b165c6..86c3d2fa9f1 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -367,8 +367,7 @@ - name: install single-beat to run only one celerybeat scheduler pip: - name: single-beat - version: "{{ EDXAPP_SINGLE_BEAT_VERSION|default(omit) }}" + name: "git+{{ EDXAPP_SINGLE_BEAT_REPO }}@{{ EDXAPP_SINGLE_BEAT_VERSION|default(master) }}#egg=single-beat" virtualenv: "{{ edxapp_venv_dir }}" state: present become_user: "{{ edxapp_user }}" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 index acf54f89058..12068c05450 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 @@ -5,7 +5,7 @@ {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} source {{ edxapp_app_dir }}/edxapp_env {% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery beat' %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery' %} export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini" if command -v ec2metadata >/dev/null 2>&1; then @@ -14,9 +14,9 @@ if command -v ec2metadata >/dev/null 2>&1; then export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" fi {% else %} -{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery beat' %} +{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery' %} {% endif %} # We exec so that celery is the child of supervisor and can be managed properly -exec {{ executable }} $@ +exec {{ executable }} --config="${SERVICE_CONFIG}" beat $@ diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 index 75e3c116f98..ccd903b8b5b 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 @@ -5,7 +5,7 @@ {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} source {{ edxapp_app_dir }}/edxapp_env {% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery worker' %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery' %} export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini" if command -v ec2metadata >/dev/null 2>&1; then @@ -14,7 +14,7 @@ if command -v ec2metadata >/dev/null 2>&1; then export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" fi {% else %} -{% set executable = edxapp_venv_bin + '/celery worker' %} +{% set executable = edxapp_venv_bin + '/celery' %} {% endif %} # We exec so that celery is the child of supervisor and can be managed properly diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 index 245eb933c42..3dd9641c9a7 100644 --- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 @@ -7,14 +7,9 @@ directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} +command={{ edxapp_app_dir }}/worker.sh --app={{ w.service_variant }}.celery:APP worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not EDXAPP_CELERY_HEARTBEAT_ENABLED|bool else '' }} {{ '-O ' + w.prefetch_optimization if w.prefetch_optimization is defined else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }} -; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit -; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers -; are auto restarted even when exiting with code 0. -; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. -autorestart=true {% endfor %} @@ -37,21 +32,28 @@ environment= EDX_REST_API_CLIENT_NAME=edx.lms.core.default, SINGLE_BEAT_LOCK_TIME={{ EDXAPP_SINGLE_BEAT_LOCK_TIME }}, SINGLE_BEAT_HEARTBEAT_INTERVAL={{ EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL }}, - SINGLE_BEAT_IDENTIFIER="celerybeat", - SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}", - SINGLE_BEAT_WAIT_MODE="supervised" + SINGLE_BEAT_IDENTIFIER="{{ EDXAPP_SINGLE_BEAT_IDENTIFIER }}", + SINGLE_BEAT_REDIS_SERVER="redis://{{ EDXAPP_SINGLE_BEAT_USER }}:{{ EDXAPP_SINGLE_BEAT_PASSWORD }}@{{ EDXAPP_CELERY_BROKER_HOSTNAME }}/{{ EDXAPP_CELERY_BROKER_VHOST }}", + SINGLE_BEAT_WAIT_MODE="supervised", + SERVICE_CONFIG="lms.envs.{{ worker_django_settings_module }}" user={{ common_web_user }} directory={{ edxapp_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -command={{ edxapp_app_dir }}/beat_scheduler.sh --config=lms.envs.{{ worker_django_settings_module }} --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" +command={{ edxapp_app_dir }}/beat_scheduler.sh --loglevel=info --schedule="{{ supervisor_log_dir }}/celerybeat-schedule" --pidfile="{{ supervisor_log_dir }}/celerybeat.pid" killasgroup=true stopwaitsecs={{ EDXAPP_WORKER_DEFAULT_STOPWAITSECS }} -; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit -; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers -; are auto restarted even when exiting with code 0. -; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0. + +# If multiple app servers are running, celery-beat will exit on the new app +# server as it is already running on the old one. Once the process exited with +# status 0, it wont restart again event the old server is deprovisioned. To +# resolve this conflict, use `autorestart=true` to keep trying to start the +# process. Startretries are scheduled 3 times by default; since there is no way +# to set infinite retries, the recommended way is setting a high number. When +# the previous server is stopped, celery-beat will start normally. autorestart=true +startsecs=30 +startretries=10000 {% endif %} From 0b71b48835e336c851c40fd2f5655e61480b4eda Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 3 Dec 2021 16:55:08 +0500 Subject: [PATCH 219/664] chore: Remove shining panda from build jenkins --- playbooks/roles/jenkins_build/defaults/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 9f9d7865138..32ae9227fa7 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -249,9 +249,6 @@ build_jenkins_plugins_list: - name: 'script-security' version: '1.78' group: 'org.jenkins-ci.plugins' - - name: 'shiningpanda' - version: '0.23' - group: 'org.jenkins-ci.plugins' - name: 'slack' version: '2.21' group: 'org.jenkins-ci.plugins' From 4bc34a431951a3455409534bac7604daef9e1e94 Mon Sep 17 00:00:00 2001 From: pkulkark Date: Wed, 8 Dec 2021 16:39:14 +0530 Subject: [PATCH 220/664] fix: add missing setting in env config --- playbooks/roles/edxapp/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index d856f6bda16..2e7bfd844ac 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1516,6 +1516,7 @@ generic_env_config: &edxapp_generic_env PASSWORD_POLICY_COMPLIANCE_ROLLOUT_CONFIG: "{{ EDXAPP_PASSWORD_POLICY_COMPLIANCE_ROLLOUT_CONFIG }}" # Dashboard courses default limit DASHBOARD_COURSE_LIMIT: "{{ EDXAPP_DASHBOARD_COURSE_LIMIT }}" + COMPLETION_AGGREGATOR_URL: "{{ EDXAPP_COMPLETION_AGGREGATOR_URL }}" lms_auth_config: <<: *edxapp_generic_auth From fc931499cef218b685c3e4bb2f04d25f5511f4e8 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 7 Dec 2021 10:16:37 -0500 Subject: [PATCH 221/664] fix: create ecommerce_worker user in the LMS for sandboxes --- playbooks/roles/edxapp/defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 874b118f95a..fbbd3a2142b 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1816,6 +1816,10 @@ SERVICE_WORKER_USERS: username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ ECOMMERCE_SERVICE_USER_EMAIL }}" + username: "{{ ECOMMERCE_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ EDXAPP_RETIREMENT_SERVICE_USER_EMAIL }}" username: "{{ EDXAPP_RETIREMENT_SERVICE_USER_NAME }}" is_staff: true From 6f12b79a2e28207d0861be29a07f6e288e202d0a Mon Sep 17 00:00:00 2001 From: Marek Wrobel Date: Tue, 14 Dec 2021 19:53:28 +0100 Subject: [PATCH 222/664] fix: Add retry logic to AMI registration --- playbooks/continuous_delivery/create_ami.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index e1f45874f68..42f65f60fa2 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -70,6 +70,9 @@ 'deployment':'{{ deployment }}' }" register: ami_register + retries: 5 + delay: 10 + until: ami_register is succeeded - name: Add any tags that are on the instance to the AMI ec2_tag: From 78d30b624c42769e2e8d55b9f0a9b5a84681ff37 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Thu, 16 Dec 2021 14:05:27 -0500 Subject: [PATCH 223/664] chore: Update the checklist link --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7e8124403bc..60f4331a23d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,5 +16,5 @@ Make sure that the following steps are done before merging: - [ ] Update the appropriate internal repo (be sure to update for all our environments) - [ ] If you are updating a secure value rather than an internal one, file a SRE ticket with details. - [ ] Add an entry to the CHANGELOG. - - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/display/EdxOps/Ops+Ansible+Testing+Checklist)? Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes). + - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/spaces/SRE/pages/28967861/Ops+Ansible+Testing+Checklist)? Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes). - [ ] Think about how this change will affect Open edX operators. Have you updated the wiki page for the next Open edX release? From 9cc8aaab79554ca27492be4cf62f0724a87dbcf1 Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Tue, 21 Dec 2021 15:13:02 +0500 Subject: [PATCH 224/664] fix: update course discovery worker app name --- .../edx/app/supervisor/conf.d.available/app-workers.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 index 437c590ac62..980a77356f5 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 @@ -34,7 +34,7 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log {% set worker_app_name = 'course_discovery' %} {% endif %} -command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker -A {{ edx_django_service_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} +command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker -A {{ worker_app_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }} From 52387e4016dc11c3c6b5e47b25cfb21c2998e5f2 Mon Sep 17 00:00:00 2001 From: Marek Wrobel Date: Wed, 22 Dec 2021 18:31:10 +0100 Subject: [PATCH 225/664] Revert "Add retry logic to AMI registration" --- playbooks/continuous_delivery/create_ami.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index 42f65f60fa2..e1f45874f68 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -70,9 +70,6 @@ 'deployment':'{{ deployment }}' }" register: ami_register - retries: 5 - delay: 10 - until: ami_register is succeeded - name: Add any tags that are on the instance to the AMI ec2_tag: From 71e178fc3393e1cde807db1851c1b92da6ceb186 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 4 Jan 2022 02:53:03 -0500 Subject: [PATCH 226/664] refactor: Updated 'upgrade requirements' workflow to use reusable workflows (#6621) --- .../workflows/upgrade-python-requirements.yml | 81 +++++-------------- 1 file changed, 18 insertions(+), 63 deletions(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 384cbf6f033..c3306db0772 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -2,68 +2,23 @@ name: Upgrade Requirements on: schedule: - # will start the job at 01:15 UTC every Monday - - cron: "15 1 * * 1" + - cron: "15 1 * * 1" workflow_dispatch: - inputs: - branch: - description: "Target branch to create requirements PR against" - required: true - default: 'master' - + inputs: + branch: + description: 'Target branch to create requirements PR against' + required: true + default: 'master' jobs: - upgrade_requirements: - runs-on: ubuntu-20.04 - - strategy: - matrix: - python-version: ["3.8"] - - steps: - - name: setup target branch - run: echo "target_branch=$(if ['${{ github.event.inputs.branch }}' = '']; then echo 'master'; else echo '${{ github.event.inputs.branch }}'; fi)" >> $GITHUB_ENV - - - uses: actions/checkout@v1 - with: - ref: ${{ env.target_branch }} - - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: make upgrade - run: | - cd $GITHUB_WORKSPACE - make upgrade - - - name: setup testeng-ci - run: | - git clone https://github.com/edx/testeng-ci.git - cd $GITHUB_WORKSPACE/testeng-ci - ls - pip install -r requirements/base.txt - - name: create pull request - env: - GITHUB_TOKEN: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} - GITHUB_USER_EMAIL: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} - run: | - cd $GITHUB_WORKSPACE/testeng-ci - python -m jenkins.pull_request_creator --repo-root=$GITHUB_WORKSPACE \ - --target-branch="${{ env.target_branch }}" --base-branch-name="upgrade-python-requirements" \ - --commit-message="chore: Updating Python Requirements" --pr-title="Python Requirements Update" \ - --pr-body="Python requirements update.Please review the [changelogs](https://openedx.atlassian.net/wiki/spaces/TE/pages/1001521320/Python+Package+Changelogs) for the upgraded packages." \ - --user-reviewers="" --team-reviewers="devops" --delete-old-pull-requests - - - name: Send failure notification - if: ${{ failure() }} - uses: dawidd6/action-send-mail@v3 - with: - server_address: email-smtp.us-east-1.amazonaws.com - server_port: 465 - username: ${{secrets.EDX_SMTP_USERNAME}} - password: ${{secrets.EDX_SMTP_PASSWORD}} - subject: Upgrade python requirements workflow failed in ${{github.repository}} - to: devops@edx.org - from: github-actions - body: Upgrade python requirements workflow in ${{github.repository}} failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + call-upgrade-python-requirements-workflow: + with: + branch: ${{ github.event.inputs.branch }} + team_reviewers: "devops" + email_address: devops@edx.org + send_success_notification: false + secrets: + requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} + requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} + edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} + edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} + uses: edx/.github/.github/workflows/upgrade-python-requirements.yml@master From 411414cf665485d0ad5daf104089dd0604258026 Mon Sep 17 00:00:00 2001 From: Justin Lapierre Date: Tue, 4 Jan 2022 12:55:12 -0500 Subject: [PATCH 227/664] chore: Enable ORA Grading MFE for sandboxes (#6641) * chore: Enable ORA Grading MFE for sandboxes * chore: Configure ORA Grading MFE to match naming conventions * chore: correctly named the E.S.G. MFE in the configuration using hyphens * chore: fixed erroneous yaml for E.S.G. role * chore: converted configuration to use underscores * chore: Changed underscore to hyphen in MFE_NAME for E.S.G. --- playbooks/edx_continuous_integration.yml | 3 +++ playbooks/ora_grading.yml | 17 +++++++++++++++++ playbooks/roles/edx_ansible/templates/update.j2 | 1 + playbooks/roles/launch_ec2/tasks/main.yml | 2 +- util/jenkins/ansible-provision.sh | 14 ++++++++++++++ 5 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 playbooks/ora_grading.yml diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index d8276715853..be2bf34d7d6 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -56,6 +56,9 @@ - role: mfe MFE_NAME: learning when: LEARNING_MFE_ENABLED + - role: mfe + MFE_NAME: ora-grading + when: ORA_GRADING_MFE_ENABLED - oauth_client_setup - role: datadog when: COMMON_ENABLE_DATADOG diff --git a/playbooks/ora_grading.yml b/playbooks/ora_grading.yml new file mode 100644 index 00000000000..8d63e0c6e65 --- /dev/null +++ b/playbooks/ora_grading.yml @@ -0,0 +1,17 @@ +- name: Deploy the ORA Grading MFE Frontend + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'ora-grading' + ORA_GRADING_ENABLED: True + ORA_GRADING_SANDBOX_BUILD: False + roles: + - role: mfe + MFE_NAME: ora-grading + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELICE_INFRASTRUCTURE + diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index 2bbe1320001..51d3cdc764e 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -81,6 +81,7 @@ repos_to_cmd["prospectus"]="$edx_ansible_cmd prospectus.yml -e 'PROSPECTUS_VERSI repos_to_cmd["authn"]="$edx_ansible_cmd authn_frontend.yml -e 'AUTHN_MFE_VERSION=$2'" repos_to_cmd["payment"]="$edx_ansible_cmd payment.yml -e 'PAYMENT_MFE_VERSION=$2'" repos_to_cmd["learning"]="$edx_ansible_cmd learning.yml -e 'LEARNING_MFE_VERSION=$2'" +repos_to_cmd["ora_grading"]="$edx_ansible_cmd ora_grading.yml -e 'ORA_GRADING_MFE_VERSION=$2'" if [[ -z $1 || -z $2 ]]; then echo diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index c3edb114af1..06c79840053 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,7 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading'] - name: Add new instance to host group local_action: diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index ec81ba75aa3..310cafc0774 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -238,6 +238,14 @@ if [[ -z $learning_version ]]; then LEARNING_MFE_VERSION="master" fi +if [[ -z $ora_grading ]]; then + ora_grading="false" +fi + +if [[ -z $ora_grading_version ]]; then + ORA_GRADING_MFE_VERSION="master" +fi + # Lowercase the dns name to deal with an ansible bug dns_name="${dns_name,,}" @@ -356,6 +364,12 @@ LEARNING_MFE_VERSION: $learning_version LEARNING_MFE_ENABLED: $learning LEARNING_SANDBOX_BUILD: True +ORA_GRADING_NGINX_PORT: 80 +ORA_GRADING_SSL_NGINX_PORT: 443 +ORA_GRADING_MFE_VERSION: $ora_grading_version +ORA_GRADING_MFE_ENABLED: $ora_grading +ORA_GRADING_SANDBOX_BUILD: True + mysql_server_version_5_7: True # User provided extra vars From 5a14577dc2feba478e417fbc5c500565fc00cf18 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 5 Jan 2022 12:46:31 +0500 Subject: [PATCH 228/664] chore: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index f61b76d2f11..a3dcd525379 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.2.1.6990-1" + version: "11.9.0.7253-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From 47cb4caaf477d3ef0c846c713bddece1c756bdd6 Mon Sep 17 00:00:00 2001 From: Alex Dusenbery Date: Wed, 5 Jan 2022 10:23:13 -0500 Subject: [PATCH 229/664] fix: Remove an extraneous `-A ecommerce_worker` from the ecomworker startup script. This was preventing the celery worker process from starting, because it overrode the correct `--app ecommmerce_worker.celery_app:app` option. `-A ecommerce_worker` tells celery to look for an app named "celery" inside ecommerce_worker, and no app with that name exists. Also changes the single application flag to `-A`, since `--app` may not be supported as a worker command option in celery 5.0+ ENT-5285 --- CHANGELOG.md | 4 ++++ .../ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90a533b4c58..404d83f4412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-01-05 + - Remove an extraneous `-A ecommerce_worker` from the ecomworker startup script, + which was preventing the celery worker process from starting. + - 2021-11-30 - Upgrade celery to 5.2.0 and adjust CLI call parameters too - Bumped single-beat to use a more supported fork of the project diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index 516e775452c..cc2aec9ad2b 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -17,4 +17,4 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env # We exec so that celery is the child of supervisor and can be managed properly -exec {{ executable }} --app ecommerce_worker.celery_app:app worker -A ecommerce_worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing +exec {{ executable }} -A ecommerce_worker.celery_app:app worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing From 34b78273b655f5dfb5ac963238875ebb88905c71 Mon Sep 17 00:00:00 2001 From: Agrendalath Date: Thu, 30 Dec 2021 20:29:51 +0100 Subject: [PATCH 230/664] fix: replace `ELASTICSEARCH_URL` with `ELASTICSEARCH_CLUSTER_URL` for discovery The variable name has been changed in: https://github.com/edx/course-discovery/pull/2870/commits/df5e75fd217740bef6a77d0b9f86bcfb7ddc2a2e Co-authored-by: @0x29a --- CHANGELOG.md | 4 ++++ playbooks/roles/discovery/defaults/main.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 404d83f4412..469a9c2d493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-01-06 + - Role: discovery + - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_CLUSTER_URL` in `discovery_service_config_overrides`. + - 2022-01-05 - Remove an extraneous `-A ecommerce_worker` from the ecomworker startup script, which was preventing the celery worker process from starting. diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 5002c787fc7..99f6b367909 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -151,7 +151,7 @@ DISCOVERY_REPOS: DISCOVERY_EXTRA_REQUIREMENTS: [] discovery_service_config_overrides: - ELASTICSEARCH_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}' + ELASTICSEARCH_CLUSTER_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}' ELASTICSEARCH_INDEX_NAME: '{{ DISCOVERY_ELASTICSEARCH_INDEX_NAME }}' PLATFORM_NAME: '{{ DISCOVERY_PLATFORM_NAME }}' From eee778fb316ec1cba77d9f97945f270006db1c24 Mon Sep 17 00:00:00 2001 From: Agrendalath Date: Thu, 30 Dec 2021 20:37:35 +0100 Subject: [PATCH 231/664] fix: replace `ELASTICSEARCH_URL` with `ELASTICSEARCH_DSL` for edx_notes_api The variable name has been changed in: https://github.com/edx/edx-notes-api/commit/e24a2678b5a268529b7bb6e610df75eac788cacc Co-authored-by: @0x29a --- CHANGELOG.md | 4 ++++ playbooks/roles/edx_notes_api/defaults/main.yml | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 469a9c2d493..91399e4a4d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-01-06 + - Role: edx_notes_api + - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_DSL` in `edx_notes_api_service_config`. + - 2022-01-06 - Role: discovery - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_CLUSTER_URL` in `discovery_service_config_overrides`. diff --git a/playbooks/roles/edx_notes_api/defaults/main.yml b/playbooks/roles/edx_notes_api/defaults/main.yml index 0f8996a9f2a..a0c045c80fb 100644 --- a/playbooks/roles/edx_notes_api/defaults/main.yml +++ b/playbooks/roles/edx_notes_api/defaults/main.yml @@ -78,7 +78,9 @@ edx_notes_api_service_config: # replace with your oauth id and secret CLIENT_ID: "{{ EDX_NOTES_API_CLIENT_ID }}" CLIENT_SECRET: "{{ EDX_NOTES_API_CLIENT_SECRET }}" - ELASTICSEARCH_URL: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}" + ELASTICSEARCH_DSL: + default: + hosts: "{{ EDX_NOTES_API_ELASTICSEARCH_URL }}" ELASTICSEARCH_INDEX: "edx_notes" # Number of rows to return by default in result. RESULTS_DEFAULT_SIZE: 25 From b19410635ec96e3eb32825cc512fbb7ec8055006 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 6 Jan 2022 10:55:03 -0500 Subject: [PATCH 232/664] fix: remove edx-zoom dependency Remove the edx-zoom installation from edx-platform because the x-block is being deprecated --- playbooks/roles/edxapp/defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index fbbd3a2142b..66feb38cb9b 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -567,7 +567,6 @@ EDXAPP_PRIVATE_REQUIREMENTS: extra_args: -e - name: git+https://github.com/open-craft/xblock-activetable.git@d3fb772435c382b59293e4e688a6a3096c4f6fd7#egg=activetable-xblock extra_args: -e - - name: edx-zoom==2.0.1 # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - name: git+https://github.com/edx/xblock-qualtrics-survey.git@02d87f567ec2af4579642cf795bb851507d6edf9#egg=xblock_qualtrics_survey extra_args: -e From 64b2cfef629c2f81cba297e6a414664ce98abfea Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 6 Jan 2022 17:26:22 -0500 Subject: [PATCH 233/664] fix: Remove test for deprecated bokchoy ISRE-690 --- .../files/test-development-environment.sh | 7 ------- playbooks/roles/test_build_server/tasks/main.yml | 1 - 2 files changed, 8 deletions(-) diff --git a/playbooks/roles/test_build_server/files/test-development-environment.sh b/playbooks/roles/test_build_server/files/test-development-environment.sh index 76bcd9f9f6c..03b409e725c 100755 --- a/playbooks/roles/test_build_server/files/test-development-environment.sh +++ b/playbooks/roles/test_build_server/files/test-development-environment.sh @@ -45,13 +45,6 @@ case "$1" in paver test_js_run -s lms-coffee ;; - "bokchoy") - - # Run some of the bok-choy tests - paver test_bokchoy -t discussion/test_discussion.py::DiscussionTabMultipleThreadTest - paver test_bokchoy -t studio/test_studio_settings.py::StudioSettingsA11yTest - ;; - "quality") # Generate quality reports paver run_quality diff --git a/playbooks/roles/test_build_server/tasks/main.yml b/playbooks/roles/test_build_server/tasks/main.yml index 08f5895a340..0ecc3912c12 100644 --- a/playbooks/roles/test_build_server/tasks/main.yml +++ b/playbooks/roles/test_build_server/tasks/main.yml @@ -46,4 +46,3 @@ with_items: - "unit" - "js" - - "bokchoy" From 68bebfcf55bc5211f91359c0afd990836ee30b57 Mon Sep 17 00:00:00 2001 From: Aarif Date: Mon, 10 Jan 2022 21:17:50 +0500 Subject: [PATCH 234/664] fix: updated Docker file base to use focal (#6656) --- docker/build/devpi/Dockerfile | 2 +- docker/build/forum/Dockerfile | 2 +- docker/build/insights/Dockerfile | 2 +- docker/build/jenkins_worker/Dockerfile | 2 +- docker/build/notes/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/build/devpi/Dockerfile b/docker/build/devpi/Dockerfile index a460ea2b4ec..b40195de02b 100644 --- a/docker/build/devpi/Dockerfile +++ b/docker/build/devpi/Dockerfile @@ -8,7 +8,7 @@ # with the currently checked-out configuration repo. ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} +FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" ARG ARG_DEVPI_SERVER_VERSION=4.4.0 diff --git a/docker/build/forum/Dockerfile b/docker/build/forum/Dockerfile index a9e4f59136e..900eeb7948f 100644 --- a/docker/build/forum/Dockerfile +++ b/docker/build/forum/Dockerfile @@ -8,7 +8,7 @@ # with the currently checked-out configuration repo. ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} +FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays diff --git a/docker/build/insights/Dockerfile b/docker/build/insights/Dockerfile index 290712d4c89..eb0805698da 100644 --- a/docker/build/insights/Dockerfile +++ b/docker/build/insights/Dockerfile @@ -8,7 +8,7 @@ # with the currently checked-out configuration repo. ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} +FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" ADD . /edx/app/edx_ansible/edx_ansible diff --git a/docker/build/jenkins_worker/Dockerfile b/docker/build/jenkins_worker/Dockerfile index b4c33244de5..daa22ff5424 100644 --- a/docker/build/jenkins_worker/Dockerfile +++ b/docker/build/jenkins_worker/Dockerfile @@ -9,7 +9,7 @@ # Run the edxapp play with custom ansible overrides ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} +FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" USER root diff --git a/docker/build/notes/Dockerfile b/docker/build/notes/Dockerfile index 31c0f7426d9..66619362211 100644 --- a/docker/build/notes/Dockerfile +++ b/docker/build/notes/Dockerfile @@ -8,7 +8,7 @@ # with the currently checked-out configuration repo. ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} +FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" ARG OPENEDX_RELEASE=master From 707b6776fbd0cc13871df182f82c64ad9e0b2f22 Mon Sep 17 00:00:00 2001 From: Jazib Humayun Date: Sun, 9 Jan 2022 19:55:09 +0500 Subject: [PATCH 235/664] chore: Install ws-cleanup plugin Install ws-cleanup plugin in DE new jenkins --- playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index c49bf628434..00989c38356 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -267,6 +267,9 @@ de_jenkins_plugins_list: - name: 'workflow-step-api' version: '2.21' group: 'org.jenkins-ci.plugins.workflow' + - name: 'ws-cleanup' + version: '0.39' + group: 'org.jenkins-ci.plugins' - name: 'xunit' version: '1.93' group: 'org.jenkins-ci.plugins' From d94de9db958ebe41058c40b492db887fb6d3bd15 Mon Sep 17 00:00:00 2001 From: Julia Eskew Date: Thu, 13 Jan 2022 18:33:30 -0500 Subject: [PATCH 236/664] chore: Upgrade the versions of the devpi components to the current latest version, including devpi-server, devpi-web, and devpi-client. Also, change deprecated DEVPI_SERVERDIR var to new DEVPISERVER_SERVERDIR var. --- docker/build/devpi/Dockerfile | 6 +++--- docker/build/devpi/docker-entrypoint.sh | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/build/devpi/Dockerfile b/docker/build/devpi/Dockerfile index b40195de02b..fd26af44d9e 100644 --- a/docker/build/devpi/Dockerfile +++ b/docker/build/devpi/Dockerfile @@ -11,9 +11,9 @@ ARG BASE_IMAGE_TAG=latest FROM edxops/focal-common:${BASE_IMAGE_TAG} LABEL maintainer="edxops" -ARG ARG_DEVPI_SERVER_VERSION=4.4.0 -ARG ARG_DEVPI_WEB_VERSION=3.2.2 -ARG ARG_DEVPI_CLIENT_VERSION=4.0.0 +ARG ARG_DEVPI_SERVER_VERSION=6.3.1 +ARG ARG_DEVPI_WEB_VERSION=4.0.8 +ARG ARG_DEVPI_CLIENT_VERSION=5.2.3 ADD . /edx/app/edx_ansible/edx_ansible diff --git a/docker/build/devpi/docker-entrypoint.sh b/docker/build/devpi/docker-entrypoint.sh index 0dd92722071..54ff157bc7b 100644 --- a/docker/build/devpi/docker-entrypoint.sh +++ b/docker/build/devpi/docker-entrypoint.sh @@ -1,18 +1,18 @@ #!/bin/bash function defaults { - : ${DEVPI_SERVERDIR="/data/server"} + : ${DEVPISERVER_SERVERDIR="/data/server"} : ${DEVPI_CLIENTDIR="/data/client"} - echo "DEVPI_SERVERDIR is ${DEVPI_SERVERDIR}" + echo "DEVPISERVER_SERVERDIR is ${DEVPISERVER_SERVERDIR}" echo "DEVPI_CLIENTDIR is ${DEVPI_CLIENTDIR}" - export DEVPI_SERVERDIR DEVPI_CLIENTDIR + export DEVPISERVER_SERVERDIR DEVPI_CLIENTDIR } function initialize_devpi { echo "[RUN]: Initializing devpi-server..." - if [ ! -d $DEVPI_SERVERDIR ]; then + if [ ! -d $DEVPISERVER_SERVERDIR ]; then devpi-server --restrict-modify root --init --start --host 127.0.0.1 --port 3141 else devpi-server --restrict-modify root --start --host 127.0.0.1 --port 3141 @@ -22,7 +22,7 @@ function initialize_devpi { devpi login root --password='' DEVPI_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` devpi user -m root password="${DEVPI_PASSWORD}" - echo "[RUN]: devpi-server password set to '${DEVPI_PASSWORD}'" > $DEVPI_SERVERDIR/.serverpassword + echo "[RUN]: devpi-server password set to '${DEVPI_PASSWORD}'" > $DEVPISERVER_SERVERDIR/.serverpassword devpi index -y -c public pypi_whitelist='*' devpi-server --stop devpi-server --status @@ -33,7 +33,7 @@ defaults if [ "$1" = 'devpi' ]; then source /home/devpi/venvs/devpi_venv/bin/activate - if [ ! -f $DEVPI_SERVERDIR/.serverversion ]; then + if [ ! -f $DEVPISERVER_SERVERDIR/.serverversion ]; then initialize_devpi fi From 5c1c1a305bf1dca25d48f89b8e0afc50ffc88cd7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 18 Jan 2022 16:03:24 +0500 Subject: [PATCH 237/664] fix: MySQL apt key --- playbooks/roles/mysql/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 46466395a2d..5a0680593d6 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -25,5 +25,5 @@ DEFAULT_MYSQL_CHARACTER_SET: utf8 DEFAULT_MYSQL_COLLATION: utf8_general_ci MYSQL_APT_KEYSERVER: "keyserver.ubuntu.com" -MYSQL_APT_KEY: "8C718D3B5072E1F5" +MYSQL_APT_KEY: "467B942D3A79BD29" MYSQL_REPO: "deb http://repo.mysql.com/apt//ubuntu/ bionic mysql-5.7" From ed0dfdeeef38df44c32801c1f77854e134b92c27 Mon Sep 17 00:00:00 2001 From: Waheed Ahmed Date: Mon, 17 Jan 2022 16:23:55 +0500 Subject: [PATCH 238/664] fix: discovery Celery command to work with version > 5.0 --- .../edx/app/supervisor/conf.d.available/app-workers.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 index 980a77356f5..b7b9e3ca326 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/supervisor/conf.d.available/app-workers.conf.j2 @@ -34,7 +34,7 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log {% set worker_app_name = 'course_discovery' %} {% endif %} -command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker -A {{ worker_app_name }} --loglevel=info --queue={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} +command={{ edx_django_service_home }}/{{ edx_django_service_name }}-workers.sh --app {{ worker_app_name }}.celery:app worker --loglevel=info --queues={{ w.queue }} --hostname={{ edx_django_service_name }}.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--max-tasks-per-child ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }} {{ '--without-heartbeat' if not edx_django_service_celery_heartbeat_enabled|bool else '' }} killasgroup=true stopwaitsecs={{ w.stopwaitsecs | default(edx_django_service_default_stopwaitsecs) }} From 3f8ee76206be2a41e8802cdc6bc4f77662915e95 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 19 Jan 2022 16:56:35 +0500 Subject: [PATCH 239/664] feat: update celery flower --- docker/build/flower/Dockerfile | 6 +++--- docker/build/flower/README.txt | 2 +- playbooks/roles/flower/defaults/main.yml | 9 ++++----- .../edx/app/supervisor/conf.d.available/flower.conf.j2 | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docker/build/flower/Dockerfile b/docker/build/flower/Dockerfile index 764405cba9d..c5f7ed6d55f 100644 --- a/docker/build/flower/Dockerfile +++ b/docker/build/flower/Dockerfile @@ -1,10 +1,10 @@ -FROM ubuntu:xenial +FROM ubuntu:focal # Update and get pip. RUN apt-get update && apt-get install -y python3-pip # Install the required packages -RUN pip3 install --no-cache-dir redis==3.2.0 celery==4.4.7 flower==0.9.5 +RUN pip3 install --no-cache-dir celery==5.2.3 https://github.com/mher/flower/zipball/v1.0.0#egg=flower redis==4.1.1 # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) @@ -22,4 +22,4 @@ USER nobody # Mount a config here if you want to enable OAuth etc ADD docker/build/flower/flowerconfig.py /flowerconfig.py -ENTRYPOINT [ "flower" ] +ENTRYPOINT [ "celery" ] diff --git a/docker/build/flower/README.txt b/docker/build/flower/README.txt index 31be8de0a41..0104a734622 100644 --- a/docker/build/flower/README.txt +++ b/docker/build/flower/README.txt @@ -1,6 +1,6 @@ Example: $ docker build . -t edxops/flower:latest -$ docker run -it --rm -p 127.0.0.1:5555:5555 edxops/flower:latest flower --broker=redis://:@some-redis-url.com:6379 --conf=flowerconfig.py +$ docker run -it --rm -p 127.0.0.1:5555:5555 edxops/flower:latest --broker=redis://:@some-redis-url.com:6379 flower --conf=flowerconfig.py $ curl localhost:5555 diff --git a/playbooks/roles/flower/defaults/main.yml b/playbooks/roles/flower/defaults/main.yml index d73048a9255..04db72d50d9 100644 --- a/playbooks/roles/flower/defaults/main.yml +++ b/playbooks/roles/flower/defaults/main.yml @@ -22,14 +22,13 @@ flower_conf_dir: "{{ flower_app_dir }}" flower_venv_dir: "{{ flower_app_dir }}/venvs/flower" flower_venv_bin: "{{ flower_venv_dir }}/bin" -flower_python_version: "python3.5" +flower_python_version: "python3.8" flower_python_reqs: # Celery version must match version used by edx-platform - - "celery==4.4.7" - - "flower==0.9.5" - - "redis==3.2.0" - - "tornado==5.1.1" + - "https://github.com/mher/flower/zipball/v1.0.0#egg=flower" + - "celery==5.2.3" + - "redis==4.1.1" flower_deploy_path: "{{ flower_venv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/bin:/sbin:/bin" diff --git a/playbooks/roles/flower/templates/edx/app/supervisor/conf.d.available/flower.conf.j2 b/playbooks/roles/flower/templates/edx/app/supervisor/conf.d.available/flower.conf.j2 index 33bc91f90e0..746ce2c8d47 100644 --- a/playbooks/roles/flower/templates/edx/app/supervisor/conf.d.available/flower.conf.j2 +++ b/playbooks/roles/flower/templates/edx/app/supervisor/conf.d.available/flower.conf.j2 @@ -2,6 +2,6 @@ environment=PATH="{{ flower_deploy_path }}" user={{ common_web_user }} -command={{ flower_venv_bin }}/celery flower --broker {{ flower_broker }} --conf={{ flower_conf_dir }}/flowerconfig.py +command={{ flower_venv_bin }}/celery --broker {{ flower_broker }} flower --conf={{ flower_conf_dir }}/flowerconfig.py stdout_logfile={{ supervisor_log_dir }}/{{ FLOWER_USER }}-stdout.log stderr_logfile={{ supervisor_log_dir }}/{{ FLOWER_USER }}-stderr.log From 3b956bcadbc7899160e895cae8365b3bd4233d00 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 19 Jan 2022 19:48:34 +0500 Subject: [PATCH 240/664] fix: use pip version of flower --- docker/build/flower/Dockerfile | 2 +- playbooks/roles/flower/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build/flower/Dockerfile b/docker/build/flower/Dockerfile index c5f7ed6d55f..93a23d99e9b 100644 --- a/docker/build/flower/Dockerfile +++ b/docker/build/flower/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:focal RUN apt-get update && apt-get install -y python3-pip # Install the required packages -RUN pip3 install --no-cache-dir celery==5.2.3 https://github.com/mher/flower/zipball/v1.0.0#egg=flower redis==4.1.1 +RUN pip3 install --no-cache-dir celery==5.2.3 flower==1.0.0 redis==4.1.1 # PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`) # PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`) diff --git a/playbooks/roles/flower/defaults/main.yml b/playbooks/roles/flower/defaults/main.yml index 04db72d50d9..a080bd445d5 100644 --- a/playbooks/roles/flower/defaults/main.yml +++ b/playbooks/roles/flower/defaults/main.yml @@ -26,7 +26,7 @@ flower_python_version: "python3.8" flower_python_reqs: # Celery version must match version used by edx-platform - - "https://github.com/mher/flower/zipball/v1.0.0#egg=flower" + - "flower==1.0.0" - "celery==5.2.3" - "redis==4.1.1" From 98479c32180548559e6d147e676fa600cde29215 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 20 Jan 2022 16:40:03 +0500 Subject: [PATCH 241/664] chore: add stop instance task to create_ami play --- playbooks/continuous_delivery/create_ami.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index e1f45874f68..c7e97cffcb2 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -37,6 +37,7 @@ vars: ec2_region: us-east-1 ami_wait: yes + stop_wait: yes ami_creation_timeout: 5400 no_reboot: no artifact_path: /tmp/ansible-runtime @@ -52,6 +53,13 @@ state: list register: instance_tags + - name: Stop instance + ec2: + instance_ids: "{{ instance_id }}" + state: stopped + wait: "{{ stop_wait }}" + region: "{{ ec2_region }}" + - name: Create AMI ec2_ami: instance_id: "{{ instance_id }}" From e9488e317d3091eb46190bcfb187f13733a8235d Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 21 Jan 2022 12:18:01 -0500 Subject: [PATCH 242/664] build: use the reusable pr-automerge-open-release workflow --- .../workflows/pr-automerge-open-release.yml | 74 +++++-------------- 1 file changed, 18 insertions(+), 56 deletions(-) diff --git a/.github/workflows/pr-automerge-open-release.yml b/.github/workflows/pr-automerge-open-release.yml index 8bf09a27b8d..afb348baf34 100644 --- a/.github/workflows/pr-automerge-open-release.yml +++ b/.github/workflows/pr-automerge-open-release.yml @@ -1,67 +1,29 @@ -# For non-draft changes to Named Release branches: -# - Check if the user belongs to a maintainers team. +# For non-draft changes to named release branches: +# - Check if the user is a maintainer (listed in the MERGERS env var in the +# .github/workflows/pr-automerge-open-release.yml workflow). # - If so, approve the pull request. -# - Tag community-engineering (for now) and the maintainers team. -# - Merge the PR when the author comments `@edx-community-bot merge`. +# - Merge the PR when the author comments `@openedx-community-bot merge`. +# # Required organization secrets -# - CC_GITHUB_TOKEN=... -# - CC_TEAM_CHAMPIONS=org/team-name -# - CC_TEAM_CONTRIBUTORS_ORG=org -# - CC_TEAM_CONTRIBUTORS_TEAM=team-name ---- -name: automerge BTR open-release PRs +# - CC_GITHUB_TOKEN or GITHUB_TOKEN + +name: Automerge BTR open-release PRs + on: issue_comment: branches: - - open-release/* + - open-release/* types: - - created - - edited + - created + - edited pull_request_target: branches: - - open-release/* + - open-release/* types: - - opened - - edited - - ready_for_review + - opened + - edited + - ready_for_review + jobs: automerge: - if: ${{ (github.event.issue.pull_request && !github.event.issue.pull_request.draft) || (github.event.pull_request && !github.event.pull_request.draft) }} - runs-on: ubuntu-latest - steps: - - name: lookup teams - id: teams - uses: tspascoal/get-user-teams-membership@v1 - with: - username: "${{ github.actor }}" - organization: ${{ secrets.CC_TEAM_CONTRIBUTORS_ORG }} - team: ${{ secrets.CC_TEAM_CONTRIBUTORS_TEAM }} - GITHUB_TOKEN: "${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" - - name: approve PR - if: ${{ steps.teams.outputs.isTeamMember == 'true' && (github.event.action == 'opened' || github.event.action == 'ready_for_review') }} - uses: andrewmusgrave/automatic-pull-request-review@0.0.5 - with: - repo-token: ${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - event: APPROVE - body: | - :+1: - - When you're ready to merge, add a comment that says - > @edx-community-bot merge - - and we'll handle the rest! - CC: @${{ secrets.CC_TEAM_CHAMPIONS }} @${{ secrets.CC_TEAM_CONTRIBUTORS_ORG }}/${{ secrets.CC_TEAM_CONTRIBUTORS_TEAM }} - - name: label PR as auto-mergeable - if: ${{ steps.teams.outputs.isTeamMember == 'true' && contains(github.event.comment.body, '@edx-community-bot merge') }} - uses: andymckay/labeler@978f846c4ca6299fd136f465b42c5e87aca28cac - with: - add-labels: 'automerge' - repo-token: ${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} - - name: automerge - uses: "pascalgn/automerge-action@v0.13.1" - env: - GITHUB_TOKEN: "${{ secrets.CC_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" - MERGE_COMMIT_MESSAGE: | - merge(#{pullRequest.number}): {pullRequest.title} - - {pullRequest.body} + uses: openedx/.github/.github/workflows/pr-automerge-open-release.yml@master From 614f1d791b21ebaa1dafa2f46ae41e847b74521e Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 21 Jan 2022 16:38:43 -0500 Subject: [PATCH 243/664] docs(pr-automerge-open-release): use better description --- .github/workflows/pr-automerge-open-release.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-automerge-open-release.yml b/.github/workflows/pr-automerge-open-release.yml index afb348baf34..25af91e0527 100644 --- a/.github/workflows/pr-automerge-open-release.yml +++ b/.github/workflows/pr-automerge-open-release.yml @@ -1,11 +1,6 @@ -# For non-draft changes to named release branches: -# - Check if the user is a maintainer (listed in the MERGERS env var in the -# .github/workflows/pr-automerge-open-release.yml workflow). -# - If so, approve the pull request. -# - Merge the PR when the author comments `@openedx-community-bot merge`. -# -# Required organization secrets -# - CC_GITHUB_TOKEN or GITHUB_TOKEN +# Enable automerging for named release branches. +# See the reusable workflow for details: +# https://github.com/openedx/.github/.github/workflows/pr-automerge-open-release.yml name: Automerge BTR open-release PRs From 0394fbd8f61988127b281dd2bc89f03d06baba4e Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 21 Jan 2022 12:14:35 +0500 Subject: [PATCH 244/664] chore: Update build jenkins plugins --- .../roles/jenkins_build/defaults/main.yml | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 32ae9227fa7..65b82a8e42d 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -11,7 +11,6 @@ build_jenkins_configuration_scripts: - 1addJarsToClasspath.groovy - 3importCredentials.groovy - 3installGroovy.groovy - - 3installPython.groovy - 3mainConfiguration.groovy - 3setGlobalProperties.groovy - 4configureEc2Plugin.groovy @@ -33,6 +32,9 @@ build_jenkins_plugins_list: - name: 'analysis-core' version: '1.96' group: 'org.jvnet.hudson.plugins' + - name: 'analysis-model-api' + version: '10.9.1' + group: 'io.jenkins.plugins' - name: 'ansicolor' version: '1.0.0' group: 'org.jenkins-ci.plugins' @@ -40,7 +42,7 @@ build_jenkins_plugins_list: version: '1.12' group: 'org.jenkins-ci.plugins' - name: 'antisamy-markup-formatter' - version: '2.1' + version: '2.7' group: 'org.jenkins-ci.plugins' - name: 'aws-credentials' version: '1.28.1' @@ -49,10 +51,10 @@ build_jenkins_plugins_list: version: '1.12.101-300.vc09c7be9cb57' group: 'org.jenkins-ci.plugins' - name: 'badge' - version: '1.9' + version: '1.9.1' group: 'org.jenkins-ci.plugins' - name: 'bootstrap5-api' - version: '5.1.1-1' + version: '5.1.3-4' group: 'io.jenkins.plugins' - name: 'bouncycastle-api' version: '2.25' @@ -66,6 +68,9 @@ build_jenkins_plugins_list: - name: 'build-user-vars-plugin' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'branch-api' + version: '2.7.0' + group: 'org.jenkins-ci.plugins' - name: 'checks-api' version: '1.7.2' group: 'io.jenkins.plugins' @@ -85,7 +90,7 @@ build_jenkins_plugins_list: version: '2.6.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' - version: '1.27' + version: '1.27.1' group: 'org.jenkins-ci.plugins' - name: 'cvs' version: '2.19' @@ -97,7 +102,7 @@ build_jenkins_plugins_list: version: '2.3.5' group: 'org.jenkins-ci.plugins' - name: 'docker-commons' - version: '1.14' + version: '1.18' group: 'org.jenkins-ci.plugins' - name: 'ec2' version: '1.50.3' @@ -109,7 +114,7 @@ build_jenkins_plugins_list: version: '2.3.0' group: 'org.jenkins-ci.plugins' - name: 'echarts-api' - version: '5.2.1-1' + version: '5.2.2-2' group: 'io.jenkins.plugins' - name: 'exclusive-execution' version: '0.8' @@ -124,7 +129,7 @@ build_jenkins_plugins_list: version: '5.15.4-1' group: 'io.jenkins.plugins' - name: 'forensics-api' - version: '1.5.0' + version: '1.7.0' group: 'io.jenkins.plugins' - name: 'ghprb' version: '1.42.2' @@ -196,7 +201,7 @@ build_jenkins_plugins_list: version: '2.6.8' group: 'org.jenkins-ci.plugins' - name: 'matrix-project' - version: '1.18' + version: '1.20' group: 'org.jenkins-ci.plugins' - name: 'maven-plugin' version: '3.4' @@ -219,14 +224,26 @@ build_jenkins_plugins_list: - name: 'pipeline-model-definition' version: '1.3.4.1' group: 'org.jenkinsci.plugins' + - name: 'pipeline-model-api' + version: '1.9.3' + group: 'org.jenkinsci.plugins' + - name: 'pipeline-model-extensions' + version: '1.9.3' + group: 'org.jenkinsci.plugins' + - name: 'pipeline-stage-step' + version: '291.vf0a8a7aeeb50' + group: 'org.jenkins-ci.plugins' - name: 'pipeline-utility-steps' version: '2.10.0' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' - version: '2.5.0' + version: '2.12.0' + group: 'io.jenkins.plugins' + - name: 'prism-api' + version: '1.25.0-2' group: 'io.jenkins.plugins' - name: 'popper2-api' - version: '2.10.2-1' + version: '2.11.2-1' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' @@ -246,6 +263,9 @@ build_jenkins_plugins_list: - name: 'run-condition' version: '1.5' group: 'org.jenkins-ci.plugins' + - name: 'scm-api' + version: '595.vd5a_df5eb_0e39' + group: 'org.jenkins-ci.plugins' - name: 'script-security' version: '1.78' group: 'org.jenkins-ci.plugins' @@ -262,16 +282,16 @@ build_jenkins_plugins_list: version: '1.9.7' group: 'com.splunk.splunkins' - name: 'ssh-agent' - version: '1.22' + version: '1.24.1' group: 'org.jenkins-ci.plugins' - name: 'ssh-credentials' - version: '1.18.1' + version: '1.19' group: 'org.jenkins-ci.plugins' - name: 'ssh-slaves' version: '1.28.1' group: 'org.jenkins-ci.plugins' - name: 'structs' - version: '1.23' + version: '308.v852b473a2b8c' group: 'org.jenkins-ci.plugins' - name: 'subversion' version: '2.15.1' @@ -301,7 +321,7 @@ build_jenkins_plugins_list: version: '0.7.11' group: 'org.jenkins-ci.plugins' - name: 'warnings-ng' - version: '9.5.1' + version: '9.11.0' group: 'io.jenkins.plugins' - name: 'workflow-aggregator' version: '2.6' @@ -322,7 +342,7 @@ build_jenkins_plugins_list: version: '2.42' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-multibranch' - version: '2.24' + version: '706.vd43c65dec013' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-scm-step' version: '2.13' From 6efe4d4233f6f17e75f698b436cc0de932ebf225 Mon Sep 17 00:00:00 2001 From: Phillip Shiu Date: Tue, 25 Jan 2022 10:40:41 -0500 Subject: [PATCH 245/664] feat: [REV-2514] new url for major release of SDN API (#6668) --- docker/build/ecommerce/ecommerce.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build/ecommerce/ecommerce.yml b/docker/build/ecommerce/ecommerce.yml index c8517f46caa..baecec8a922 100644 --- a/docker/build/ecommerce/ecommerce.yml +++ b/docker/build/ecommerce/ecommerce.yml @@ -127,5 +127,5 @@ STATIC_ROOT: /edx/var/ecommerce/staticfiles THEME_SCSS: sass/themes/default.scss TIME_ZONE: UTC USERNAME_REPLACEMENT_WORKER: OVERRIDE THIS WITH A VALID USERNAME -SDN_CHECK_API_URL: https://api.trade.gov/gateway/v1/consolidated_screening_list/search +SDN_CHECK_API_URL: https://data.trade.gov/consolidated_screening_list/v1/search SDN_CHECK_API_KEY: sdn search key here From eecea070eb5e17414938c094edaea82fa2e6f157 Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> Date: Thu, 27 Jan 2022 15:48:08 +0500 Subject: [PATCH 246/664] feat: upgrade xblock-problem-builder to version 5.1.0 --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 66feb38cb9b..e7c2456f5a8 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -541,7 +541,7 @@ EDXAPP_EXTRA_REQUIREMENTS: [] # - name: git+https://git.myproject.org/MyProject#egg=MyProject EDXAPP_PRIVATE_REQUIREMENTS: # For Harvard courses: - - name: xblock-problem-builder==5.0.0 + - name: xblock-problem-builder==5.1.0 # Oppia XBlock - name: git+https://github.com/oppia/xblock.git@1030adb3590ad2d32c93443cc8690db0985d76b6#egg=oppia-xblock extra_args: -e From dac1b8aeff5b12daf1f5fc2c04c827a2152e0b31 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 25 Jan 2022 17:17:05 +0500 Subject: [PATCH 247/664] chore: Update tools-jenkins docker to focal --- docker/build/tools_jenkins/Dockerfile | 4 ++-- .../build/tools_jenkins/ansible_overrides.yml | 1 + .../roles/jenkins_master/defaults/main.yml | 7 +++++++ playbooks/roles/jenkins_master/tasks/main.yml | 20 ++++++++++++++++++- .../roles/mongo_client/defaults/main.yml | 8 ++++---- playbooks/roles/mongo_client/tasks/main.yml | 18 ++++++++++++++--- .../roles/tools_jenkins/defaults/main.yml | 3 ++- playbooks/roles/tools_jenkins/meta/main.yml | 2 ++ playbooks/roles/tools_jenkins/tasks/main.yml | 4 ++-- 9 files changed, 54 insertions(+), 13 deletions(-) diff --git a/docker/build/tools_jenkins/Dockerfile b/docker/build/tools_jenkins/Dockerfile index a5726c16d01..8b408d99228 100644 --- a/docker/build/tools_jenkins/Dockerfile +++ b/docker/build/tools_jenkins/Dockerfile @@ -1,4 +1,4 @@ -FROM edxops/trusty-common:latest +FROM edxops/focal-common:latest LABEL maintainer="edxops" USER root @@ -7,6 +7,6 @@ RUN apt-get update ADD . /edx/app/edx_ansible/edx_ansible WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays COPY docker/build/tools_jenkins/ansible_overrides.yml / -RUN PYTHONUNBUFFERED=1 /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook -v jenkins_tools.yml -i '127.0.0.1,' -c local -e@/ansible_overrides.yml -vv +RUN PYTHONUNBUFFERED=1 /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook -v jenkins_tools.yml -i '127.0.0.1,' -c local -e@/ansible_overrides.yml -t 'install' -vv CMD /bin/su -l jenkins --shell=/bin/bash -c "/usr/bin/daemon -f --name=jenkins --inherit --env=JENKINS_HOME=/edx/var/jenkins --output=/var/log/jenkins/jenkins.log --pidfile=/var/run/jenkins/jenkins.pid -- /usr/bin/java -jar /usr/share/jenkins/jenkins.war --webroot=/var/cache/jenkins/war --httpPort=8080 --ajp13Port=-1" diff --git a/docker/build/tools_jenkins/ansible_overrides.yml b/docker/build/tools_jenkins/ansible_overrides.yml index 828dbca7dcf..16240a9e3d2 100644 --- a/docker/build/tools_jenkins/ansible_overrides.yml +++ b/docker/build/tools_jenkins/ansible_overrides.yml @@ -1,3 +1,4 @@ --- DOCKER_TLD: "edx" +jenkins_venv_src_dir: "../../util/jenkins" diff --git a/playbooks/roles/jenkins_master/defaults/main.yml b/playbooks/roles/jenkins_master/defaults/main.yml index e92ed248ace..516c7077e17 100644 --- a/playbooks/roles/jenkins_master/defaults/main.yml +++ b/playbooks/roles/jenkins_master/defaults/main.yml @@ -6,6 +6,13 @@ jenkins_port: 8080 jenkins_nginx_port: 80 jenkins_protocol_https: true jenkins_job_venv_dir: "/edx/var/jenkins/jobvenvs/" +jenkins_venv_src_dir: "../util/jenkins" + +# Mysql5.7 pkg for focal (There is no mysql5.7 pkg in focal, so we added mysql bionic repo) +jenkins_install_mysql_5_7: false +jenkins_mysql_apt_keyserver: "keyserver.ubuntu.com" +jenkins_mysql_apt_key: "467B942D3A79BD29" +jenkins_mysql_repo: "deb http://repo.mysql.com/apt//ubuntu/ bionic mysql-5.7" JENKINS_VERSION: '1.651.3' jenkins_deb_url: "https://archives.jenkins-ci.org/debian-stable/jenkins_{{ JENKINS_VERSION }}_all.deb" diff --git a/playbooks/roles/jenkins_master/tasks/main.yml b/playbooks/roles/jenkins_master/tasks/main.yml index a3ea716759c..679b1f0f8e5 100644 --- a/playbooks/roles/jenkins_master/tasks/main.yml +++ b/playbooks/roles/jenkins_master/tasks/main.yml @@ -1,4 +1,22 @@ --- +- name: add the mysql signing key + apt_key: + keyserver: "{{ jenkins_mysql_apt_keyserver }}" + id: "{{ jenkins_mysql_apt_key }}" + when: ansible_distribution_release == 'focal' and jenkins_install_mysql_5_7|bool + tags: + - install + - install:system-requirements + +- name: add the mysql-5.7 repo to the sources list + apt_repository: + repo: "{{ jenkins_mysql_repo }}" + state: present + when: ansible_distribution_release == 'focal' and jenkins_install_mysql_5_7|bool + tags: + - install + - install:system-requirements + - name: Install jenkins specific system packages apt: name: "{{ jenkins_debian_pkgs }}" @@ -247,7 +265,7 @@ - name: Copy virtualenv script tools to jenkins copy: remote_src: no - src: ../util/jenkins/virtualenv_tools.sh + src: "{{ jenkins_venv_src_dir }}/virtualenv_tools.sh" dest: "{{ jenkins_job_venv_dir }}/virtualenv_tools.sh" owner: "{{ jenkins_user }}" group: "{{ jenkins_group }}" diff --git a/playbooks/roles/mongo_client/defaults/main.yml b/playbooks/roles/mongo_client/defaults/main.yml index 772d7fc6369..71300d4624a 100644 --- a/playbooks/roles/mongo_client/defaults/main.yml +++ b/playbooks/roles/mongo_client/defaults/main.yml @@ -1,8 +1,8 @@ --- -MONGO_CLIENT_MONGODB_APT_KEY: "d68fa50fea312927" -MONGO_CLIENT_MONGODB_APT_KEYSERVER: "keyserver.ubuntu.com" -MONGO_CLIENT_MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" -MONGO_CLIENT_VERSION: 3.2.12 +MONGO_VERSION_MAJOR_MINOR: "4.2" +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" +MONGODB_REPO_BIONIC: "deb http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" +MONGO_CLIENT_VERSION: "4.2.14" mongo_client_debian_pkgs: - "mongodb-org-shell={{ MONGO_CLIENT_VERSION }}" diff --git a/playbooks/roles/mongo_client/tasks/main.yml b/playbooks/roles/mongo_client/tasks/main.yml index a107a7b7ed9..7f02755ab8f 100644 --- a/playbooks/roles/mongo_client/tasks/main.yml +++ b/playbooks/roles/mongo_client/tasks/main.yml @@ -1,20 +1,32 @@ --- - name: add the mongodb signing key apt_key: - id: "{{ MONGO_CLIENT_MONGODB_APT_KEY }}" - keyserver: "{{ MONGO_CLIENT_MONGODB_APT_KEYSERVER }}" + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present + retries: 3 tags: - install - install:system-requirements - name: add the mongodb repo to the sources list apt_repository: - repo: "{{ MONGO_CLIENT_MONGODB_REPO }}" + repo: "{{ MONGODB_REPO }}" state: present tags: - install - install:system-requirements + when: ansible_distribution_release != 'focal' + +# mongo 4.2 does not have any source list for Focal +# use Bionci repo source list to install mongo 4.2 +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO_BIONIC }}" + state: present + tags: + - install + - install:system-requirements + when: ansible_distribution_release == 'focal' - name: install mongo shell apt: diff --git a/playbooks/roles/tools_jenkins/defaults/main.yml b/playbooks/roles/tools_jenkins/defaults/main.yml index 31ba8a978a2..99d9df34bcf 100644 --- a/playbooks/roles/tools_jenkins/defaults/main.yml +++ b/playbooks/roles/tools_jenkins/defaults/main.yml @@ -69,6 +69,7 @@ jenkins_tools_bundled_plugins: - "ssh-slaves" jenkins_tools_helm_url: "https://get.helm.sh/helm-v3.5.2-linux-amd64.tar.gz" +jenkins_venv_src_dir: "../util/jenkins" jenkins_tools_debian_pkgs: - nginx @@ -77,5 +78,5 @@ jenkins_tools_debian_pkgs: - daemon - python-pycurl - psmisc - - mysql-client-core-5.7 + - mysql-client=5.7.* - ruby-sass diff --git a/playbooks/roles/tools_jenkins/meta/main.yml b/playbooks/roles/tools_jenkins/meta/main.yml index 7e942e29cac..b46e2c6acb0 100644 --- a/playbooks/roles/tools_jenkins/meta/main.yml +++ b/playbooks/roles/tools_jenkins/meta/main.yml @@ -10,6 +10,8 @@ dependencies: jenkins_custom_plugins: [] jenkins_bundled_plugins: "{{ jenkins_tools_bundled_plugins }}" jenkins_debian_pkgs: "{{ jenkins_tools_debian_pkgs }}" + jenkins_venv_src_dir: "{{ jenkins_venv_src_dir }}" + jenkins_install_mysql_5_7: true # Needed to be able to build docker images. Used by Docker Image Builder Jobs. - role: docker-tools diff --git a/playbooks/roles/tools_jenkins/tasks/main.yml b/playbooks/roles/tools_jenkins/tasks/main.yml index 0962555522d..48aba09c1c2 100644 --- a/playbooks/roles/tools_jenkins/tasks/main.yml +++ b/playbooks/roles/tools_jenkins/tasks/main.yml @@ -4,7 +4,7 @@ - name: add deadsnakes repository apt_repository: repo: "ppa:deadsnakes/ppa" - when: ansible_distribution_release == 'bionic' + when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' tags: - install - install:system-requirements @@ -19,7 +19,7 @@ - python3.6-dev - python3.8 - python3.8-dev - when: ansible_distribution_release == 'bionic' + when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' tags: - install - install:system-requirements From 9fd14adf7b589e9bda4c2140e15f5d7141567818 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 28 Jan 2022 15:08:47 +0500 Subject: [PATCH 248/664] chore: Updated python requirements Github action to create tickets in ISRE backlog --- .github/workflows/upgrade-python-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index c3306db0772..1b7d2ce51d5 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -14,7 +14,7 @@ jobs: with: branch: ${{ github.event.inputs.branch }} team_reviewers: "devops" - email_address: devops@edx.org + email_address: tools-jenkins-isre@edx.opsgenie.net send_success_notification: false secrets: requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} From 5a81c5b1125204f40a1479fa6c1ef1ffa8f4cf51 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 31 Jan 2022 15:33:47 +0500 Subject: [PATCH 249/664] chore: Updated python requirements Github action to create tickets in ISRE backlog --- .github/workflows/upgrade-python-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 1b7d2ce51d5..c325a5e42cb 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -14,7 +14,7 @@ jobs: with: branch: ${{ github.event.inputs.branch }} team_reviewers: "devops" - email_address: tools-jenkins-isre@edx.opsgenie.net + email_address: ${{ secrets.REQUIREMENTS_BOT_OPSGENIE_EMAIL }} send_success_notification: false secrets: requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} From 2530ae828996136d51e05631b691b16506b7df8c Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Tue, 1 Feb 2022 09:26:11 +0200 Subject: [PATCH 250/664] feat: add PREPEND_LOCALE_PATHS configuration setting edx-platform supports COMPREHENSIVE_THEME_LOCALE_PATHS setting, which appends paths to the end of LOCALE_PATHS, but there's currently no way to add additional paths to the start of the list. We want to be able to prepend locale paths so that we can override existing translations in edx-platform. --- CHANGELOG.md | 5 +++++ playbooks/roles/edxapp/defaults/main.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91399e4a4d0..24bfe528b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-02-01 + - Role: edxapp + - Added a new `EDXAPP_PREPEND_LOCALE_PATHS` setting to allow operators to + override the default translations. + - 2022-01-06 - Role: edx_notes_api - Replaced `ELASTICSEARCH_URL` with `ELASTICSEARCH_DSL` in `edx_notes_api_service_config`. diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index e7c2456f5a8..7aaaf5d5815 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -874,6 +874,9 @@ EDXAPP_COMPREHENSIVE_THEME_DIRS: # list of paths to the comprehensive theme locale directories EDXAPP_COMPREHENSIVE_THEME_LOCALE_PATHS: [] +# list of paths to locale directories to load first +EDXAPP_PREPEND_LOCALE_PATHS: [] + # Name of the default site theme EDXAPP_DEFAULT_SITE_THEME: "" EDXAPP_ENABLE_COMPREHENSIVE_THEMING: false @@ -1477,6 +1480,7 @@ generic_env_config: &edxapp_generic_env COMPREHENSIVE_THEME_DIRS: "{{ EDXAPP_COMPREHENSIVE_THEME_DIRS }}" COMPREHENSIVE_THEME_LOCALE_PATHS: "{{ EDXAPP_COMPREHENSIVE_THEME_LOCALE_PATHS }}" + PREPEND_LOCALE_PATHS: "{{ EDXAPP_PREPEND_LOCALE_PATHS }}" ENABLE_COMPREHENSIVE_THEMING: "{{ EDXAPP_ENABLE_COMPREHENSIVE_THEMING }}" DEFAULT_SITE_THEME: "{{ EDXAPP_DEFAULT_SITE_THEME }}" SESSION_SAVE_EVERY_REQUEST: "{{ EDXAPP_SESSION_SAVE_EVERY_REQUEST }}" From c5eb15a0d24e88da649fa5194bec3e1c772b9e59 Mon Sep 17 00:00:00 2001 From: Arbab Khalil Date: Wed, 2 Feb 2022 19:07:09 +0500 Subject: [PATCH 251/664] feat: add delete data czar python script (#6678) --- util/create_data_czar/remove_data_czar.py | 66 +++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 util/create_data_czar/remove_data_czar.py diff --git a/util/create_data_czar/remove_data_czar.py b/util/create_data_czar/remove_data_czar.py new file mode 100644 index 00000000000..7ddc5589a10 --- /dev/null +++ b/util/create_data_czar/remove_data_czar.py @@ -0,0 +1,66 @@ +import logging +import sys +import argparse +import boto3.session +import botocore.exceptions + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +logger.addHandler(handler) +# Assumes that the Data Czars already have your public key +# Asumes that boto3 is configured with edX Prod account + +def delete_iam_user(session: boto3.session.Session, user_name: str) -> None: + """For a given boto3.session.Session, delete the IAM User and all assoc'd resources.""" + iam = session.resource("iam") + iam_client = session.client("iam") + user = iam.User(user_name) + try: + user.load() + except botocore.exceptions.ClientError as client_error: + # If load failed with NoSuchEntity, IAM User doesn't exist. + if client_error.response.get("Error", {}).get("Code", "") == "NoSuchEntity": + logger.error(f"User {user_name} does not exist") + return + raise client_error + logger.debug(f"Deleting IAM User: {user.arn}") + for group in user.groups.all(): + logger.debug(f"Removing {user.arn} from Group {group.arn}") + user.remove_group(GroupName=group.name) + try: + login_profile = iam.LoginProfile(user.name) + login_profile.load() + logger.debug(f"Deleting Login Profile (I.E. Password) from {user.arn}") + login_profile.delete() + except botocore.exceptions.ClientError as client_error: + # If load failed with NoSuchEntity, No Login Profile + if client_error.response.get("Error", {}).get("Code", "") != "NoSuchEntity": + raise client_error + for access_key in user.access_keys.all(): + logger.debug(f"Deleting Access Key from {user.arn}: {access_key.access_key_id}") + access_key.delete() + for policy in user.policies.all(): + logger.debug(f"Deleting Inline Policy from {user.arn}: {policy.name}") + policy.delete() + for policy in user.attached_policies.all(): + logger.debug(f"Detaching Managed Policy from {user.arn}: {policy.arn}") + user.detach_policy(PolicyArn=policy.arn) + # Deleting IAM User + user.delete() + logger.info(f"Deleted IAM user: {user.name}") + + +if __name__ == "__main__": + # Parser + parser = argparse.ArgumentParser(description="Username of Data Czar.") + parser.add_argument('-u', '--user', help='Email of Data Czar', required=True) + args = parser.parse_args() + + # Create boto3 session and delete user + user_name = args.user + session = boto3.session.Session() + delete_iam_user(session, user_name) From 19f2ddff452f3967d9fbc88bf3ef5cc6c827f513 Mon Sep 17 00:00:00 2001 From: Marek Wrobel Date: Wed, 2 Feb 2022 10:40:36 -0500 Subject: [PATCH 252/664] chore: WS-2491 greenhouse token and drupal cleanup --- playbooks/roles/prospectus/defaults/main.yml | 2 -- playbooks/roles/prospectus/templates/.env.environment.j2 | 1 - 2 files changed, 3 deletions(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 2d569919794..25e484118da 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -19,7 +19,6 @@ PROSPECTUS_OAUTH_SECRET: !!null PROSPECTUS_NEW_RELIC_LICENSE_KEY: 'fake-key' PROSPECTUS_NEW_RELIC_APP_ID: 'fake-id' -PROSPECTUS_GREENHOUSE_API_TOKEN: 'fake-key' PROSPECTUS_CONTENTFUL_SPACE_ID: 'fake-key' PROSPECTUS_CONTENTFUL_ACCESS_TOKEN: 'fake-key' PROSPECTUS_SEGMENT_WRITE_KEY: 'fake-key' @@ -27,7 +26,6 @@ PROSPECTUS_ALGOLIA_BROWSE_KEY: 'fake-key' # nginx vars PROSPECTUS_DATA_DIR: '/edx/var/prospectus' -PROSPECTUS_PROXY_PASS: 'http://localhost:8080' NGINX_PROSPECTUS_PROXY_INTERCEPT_ERRORS: true PROSPECTUS_STATIC_SITES: [] PROSPECTUS_TEMPORARY_REDIRECTS: [] diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index c2ce86a0a99..ccb2ead6cab 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -6,7 +6,6 @@ OAUTH_SECRET={{ PROSPECTUS_OAUTH_SECRET }} NEW_RELIC_LICENSE_KEY={{ PROSPECTUS_NEW_RELIC_LICENSE_KEY }} NEW_RELIC_APP_ID={{ PROSPECTUS_NEW_RELIC_APP_ID }} -GATSBY_GREENHOUSE_API_TOKEN={{ PROSPECTUS_GREENHOUSE_API_TOKEN }} GATSBY_CONTENTFUL_SPACE_ID={{ PROSPECTUS_CONTENTFUL_SPACE_ID }} GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} From f0e4fca0f2b0825a51101da778bb070e995463b3 Mon Sep 17 00:00:00 2001 From: Leonard Camacho Date: Mon, 31 Jan 2022 11:44:09 -0300 Subject: [PATCH 253/664] fix: Update docker-entrypoint.sh with the right commands to run devpi server --- docker/build/devpi/docker-entrypoint.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docker/build/devpi/docker-entrypoint.sh b/docker/build/devpi/docker-entrypoint.sh index 54ff157bc7b..83a360c4223 100644 --- a/docker/build/devpi/docker-entrypoint.sh +++ b/docker/build/devpi/docker-entrypoint.sh @@ -12,20 +12,9 @@ function defaults { function initialize_devpi { echo "[RUN]: Initializing devpi-server..." - if [ ! -d $DEVPISERVER_SERVERDIR ]; then - devpi-server --restrict-modify root --init --start --host 127.0.0.1 --port 3141 - else - devpi-server --restrict-modify root --start --host 127.0.0.1 --port 3141 - fi - devpi-server --status - devpi use http://localhost:3141 - devpi login root --password='' DEVPI_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` - devpi user -m root password="${DEVPI_PASSWORD}" + devpi-init --root-passwd ${DEVPI_PASSWORD} echo "[RUN]: devpi-server password set to '${DEVPI_PASSWORD}'" > $DEVPISERVER_SERVERDIR/.serverpassword - devpi index -y -c public pypi_whitelist='*' - devpi-server --stop - devpi-server --status } defaults From f98208806315db58f7fb3fba757885b72d2b4ee6 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 10 Feb 2022 11:06:25 -0500 Subject: [PATCH 254/664] feat: Add automated users to analytics api This allows analytics api to be setup to allow management command jenkins jobs. --- playbooks/roles/analytics_api/meta/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/analytics_api/meta/main.yml b/playbooks/roles/analytics_api/meta/main.yml index 6284bced836..2b098c31c83 100644 --- a/playbooks/roles/analytics_api/meta/main.yml +++ b/playbooks/roles/analytics_api/meta/main.yml @@ -43,6 +43,7 @@ dependencies: edx_django_service_wsgi_name: '{{ analytics_api_wsgi_name }}' edx_django_service_hostname: '~^((stage|prod)-)?{{ analytics_api_hostname }}.*' edx_django_service_newrelic_appname: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ analytics_api_newrelic_appname }}' + edx_django_service_automated_users: '{{ ANALYTICS_API_AUTOMATED_USERS }}' edx_django_service_cors_whitelist: '{{ ANALYTICS_API_CORS_ORIGIN_WHITELIST }}' edx_django_service_allow_cors_headers: '{{ ANALYTICS_API_ALLOW_CORS_HEADERS }}' edx_django_service_allow_cors_credentials: '{{ ANALYTICS_API_ALLOW_CORS_CREDENTIALS }}' From 59cbc50f1aa29b867b83bf8d8e4de6e57a68ad6b Mon Sep 17 00:00:00 2001 From: Kaustav Banerjee Date: Sun, 13 Feb 2022 11:14:35 +0530 Subject: [PATCH 255/664] feat: add nginx server names hash override --- playbooks/roles/nginx/defaults/main.yml | 4 ++++ playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index e91862c8d2c..c43f56031c8 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -21,6 +21,10 @@ NGINX_OVERRIDE_DEFAULT_MAP_HASH_SIZE: False NGINX_MAP_HASH_MAX_SIZE: 2048 NGINX_MAP_HASH_BUCKET_SIZE: 64 +# Override these vars to alter the memory allocated to server_names_hash +NGINX_OVERRIDE_DEFAULT_SERVER_NAMES_HASH_SIZE: False +NGINX_SERVER_NAMES_HASH_BUCKET_SIZE: 64 + # Override these vars for adding user to nginx.htpasswd NGINX_USERS: - name: "{{ COMMON_HTPASSWD_USER }}" diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index c54faf12ae0..ccec62f53a7 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -27,7 +27,9 @@ http { large_client_header_buffers 8 16k; server_tokens off; - # server_names_hash_bucket_size 64; + {% if NGINX_OVERRIDE_DEFAULT_SERVER_NAMES_HASH_SIZE %} + server_names_hash_bucket_size {{ NGINX_SERVER_NAMES_HASH_BUCKET_SIZE }}; + {% endif %} # server_name_in_redirect off; include /etc/nginx/mime.types; From 7ca54f6849e1dbfa94c562867a215dd3fac02b39 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 22 Feb 2022 11:52:29 +0500 Subject: [PATCH 256/664] feat: update mysql role to install mysql 8.0 --- playbooks/roles/mysql/defaults/main.yml | 9 +++++++++ playbooks/roles/mysql/tasks/mysql.yml | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 5a0680593d6..92cdf4f00ca 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -21,6 +21,15 @@ mysql_dir: /etc/mysql mysql_socket: /var/run/mysqld/mysqld.sock +mysql_8_0_install: false + +mysql_server_8_0_pkgs: + - mysql-client-8.0 + - mysql-client-core-8.0 + - mysql-common + - mysql-server-8.0 + - mysql-server-core-8.0 + DEFAULT_MYSQL_CHARACTER_SET: utf8 DEFAULT_MYSQL_COLLATION: utf8_general_ci diff --git a/playbooks/roles/mysql/tasks/mysql.yml b/playbooks/roles/mysql/tasks/mysql.yml index 94511fc67ee..13b5b4732aa 100644 --- a/playbooks/roles/mysql/tasks/mysql.yml +++ b/playbooks/roles/mysql/tasks/mysql.yml @@ -65,7 +65,7 @@ apt_repository: repo: "{{ MYSQL_REPO }}" state: present - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install # Installing mysql-5.7-client, mysql-5.7-community-server # and mysql-5.7-server in separate tasks to resolve dependencies @@ -74,21 +74,29 @@ name: "{{ mysql_client_5_7_pkg }}" state: present update_cache: yes - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install - name: install mysql-5.7-community-server apt: name: "{{ mysql_community_server_5_7_pkg }}" state: present update_cache: yes - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install - name: install mysql-5.7-server apt: name: "{{ mysql_server_5_7_pkg }}" state: present update_cache: yes - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install + +- name: install mysql-8.0 + apt: + name: "{{ mysql_server_8_0_pkgs }}" + state: present + update_cache: yes + when: ansible_distribution_release == 'focal' and mysql_8_0_install + - name: restart mysql command: service mysql restart @@ -107,4 +115,10 @@ shell: mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password;set password=password('')" become: true - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install + +- name: Alter user root to use mysql_native_password + shell: + mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password by ''" + become: true + when: ansible_distribution_release == 'focal' and mysql_8_0_install From 229be43bfbc7e077c06755264c9c39c090a0c660 Mon Sep 17 00:00:00 2001 From: Abdoul Aw Date: Wed, 2 Mar 2022 13:42:15 -0500 Subject: [PATCH 257/664] feat: upgrade jenkins 12055 --- playbooks/roles/jenkins_build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index 65b82a8e42d..e4dac32418f 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -1,6 +1,6 @@ build_jenkins_user_uid: 1002 build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.303.3 +BUILD_JENKINS_VERSION: jenkins_2.319.3 build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' build_jenkins_python_versions: From 74f47e12519888e73205bf2ba08ed3ff67436186 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Mon, 28 Feb 2022 22:02:38 -0500 Subject: [PATCH 258/664] chore: Edit aws_cloudwatch_agent role to update collectd and make cloudwatch namespaces dynamic --- playbooks/roles/aws_cloudwatch_agent/defaults/main.yml | 5 ++++- .../templates/amazon-cloudwatch-agent.json | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml index 4fa14e6cdf9..341a074d3dc 100644 --- a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml +++ b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml @@ -8,6 +8,9 @@ # The Amazon CloudWatch Agent URL cloudwatch_url: https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb +# Default cloudwatch namespace +cloudwatch_namespace: Analytics/Monitor + # Collectd installation parameters -collectd_version: "5.7.2-2ubuntu1" +collectd_version: "5.9.2.g-1ubuntu5" collectd_install_recommends: yes diff --git a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json index 1ae641247dc..742d0767e6a 100644 --- a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json +++ b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json @@ -4,6 +4,7 @@ "run_as_user": "root" }, "metrics": { + "namespace": "{{ cloudwatch_namespace }}", "append_dimensions": { "InstanceId": "${aws:InstanceId}" }, From f198df6b83e50bef5ad1b0d3118576087d8e08c6 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Mon, 28 Feb 2022 22:04:43 -0500 Subject: [PATCH 259/664] feat: Add role and playbook to install nginx for a dbt docs server --- playbooks/deploy_nginx_for_dbt_docs.yml | 6 ++ .../roles/dbt_docs_nginx/defaults/main.yml | 0 playbooks/roles/dbt_docs_nginx/meta/main.yml | 3 + playbooks/roles/dbt_docs_nginx/tasks/main.yml | 87 +++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 playbooks/deploy_nginx_for_dbt_docs.yml create mode 100644 playbooks/roles/dbt_docs_nginx/defaults/main.yml create mode 100644 playbooks/roles/dbt_docs_nginx/meta/main.yml create mode 100644 playbooks/roles/dbt_docs_nginx/tasks/main.yml diff --git a/playbooks/deploy_nginx_for_dbt_docs.yml b/playbooks/deploy_nginx_for_dbt_docs.yml new file mode 100644 index 00000000000..463d163ed43 --- /dev/null +++ b/playbooks/deploy_nginx_for_dbt_docs.yml @@ -0,0 +1,6 @@ +- name: Deploy dbt docs server + hosts: all + become: True + gather_facts: True + roles: + - dbt_docs_nginx \ No newline at end of file diff --git a/playbooks/roles/dbt_docs_nginx/defaults/main.yml b/playbooks/roles/dbt_docs_nginx/defaults/main.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/playbooks/roles/dbt_docs_nginx/meta/main.yml b/playbooks/roles/dbt_docs_nginx/meta/main.yml new file mode 100644 index 00000000000..202420c7af1 --- /dev/null +++ b/playbooks/roles/dbt_docs_nginx/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - aws_cloudwatch_agent \ No newline at end of file diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml new file mode 100644 index 00000000000..cc831944c25 --- /dev/null +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -0,0 +1,87 @@ +--- + +- name: Update the cache and update all packages + apt: + name: "*" + state: latest + update_cache: yes + tags: + - install + - install:base + +- name: Install AWS CLI + apt: + name: awscli + state: present + tags: + - install + - install:base + +- name: Install and start nginx + apt: + name: nginx + state: present + tags: + - install + - install:base + +- name: Copy dbt docs html files from S3 to the local machine + shell: 'aws s3 sync s3://edx-dbt-docs/ /usr/share/nginx/html/ --delete --include "*"' + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Make a Server Configuration Directory + file: + path: /home/server-config/ + state: directory + mode: 0755 + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Copy dbt docs nginx server configurations from S3 + shell: 'aws s3 cp s3://edx-dbt-docs/server-config/ /home/server-config/ --recursive' + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Copy the nginx configuration file from S3 to the local nginx config directory + copy: + src: /home/server-config/nginx.conf + dest: /etc/nginx/nginx.conf + remote_src: yes + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Make the dbt-files-sync.sh script executable + file: + path: /home/server-config/dbt-files-sync.sh + mode: "a+x" + become: yes + become_method: sudo + tags: + - install + - install:base + +- name: Set up cron job to update the html files from S3 + ansible.builtin.cron: + name: Run the /home/server-config/dbt-files-sync.sh script + minute: 0 + hour: 23 + job: /home/server-config/dbt-files-sync.sh + +- name: Restart nginx and enable it on reboot + ansible.builtin.service: + name: nginx + state: restarted + enabled: yes From 163c8dca720970bae25348274803b09cecdbc429 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Thu, 3 Mar 2022 09:01:29 -0500 Subject: [PATCH 260/664] chore: Add tags to all tasks of dbt_docs_nginx and update cron time --- playbooks/deploy_nginx_for_dbt_docs.yml | 2 +- playbooks/roles/dbt_docs_nginx/tasks/main.yml | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/playbooks/deploy_nginx_for_dbt_docs.yml b/playbooks/deploy_nginx_for_dbt_docs.yml index 463d163ed43..961108c100a 100644 --- a/playbooks/deploy_nginx_for_dbt_docs.yml +++ b/playbooks/deploy_nginx_for_dbt_docs.yml @@ -1,4 +1,4 @@ -- name: Deploy dbt docs server +- name: Deploy dbt-docs server hosts: all become: True gather_facts: True diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml index cc831944c25..2eaf67b1c41 100644 --- a/playbooks/roles/dbt_docs_nginx/tasks/main.yml +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -25,7 +25,7 @@ - install - install:base -- name: Copy dbt docs html files from S3 to the local machine +- name: Copy dbt-docs html files from S3 to the local machine shell: 'aws s3 sync s3://edx-dbt-docs/ /usr/share/nginx/html/ --delete --include "*"' become: yes become_method: sudo @@ -44,7 +44,7 @@ - install - install:base -- name: Copy dbt docs nginx server configurations from S3 +- name: Copy dbt-docs nginx server configurations from S3 shell: 'aws s3 cp s3://edx-dbt-docs/server-config/ /home/server-config/ --recursive' become: yes become_method: sudo @@ -76,12 +76,18 @@ - name: Set up cron job to update the html files from S3 ansible.builtin.cron: name: Run the /home/server-config/dbt-files-sync.sh script - minute: 0 - hour: 23 + minute: 30 + hour: "*/6" job: /home/server-config/dbt-files-sync.sh + tags: + - install + - install:base - name: Restart nginx and enable it on reboot ansible.builtin.service: name: nginx state: restarted enabled: yes + tags: + - install + - install:base From 2be3c19ce23863008aafcd31c0f519214727b7f0 Mon Sep 17 00:00:00 2001 From: Hamza442 Date: Tue, 8 Mar 2022 11:02:46 +0500 Subject: [PATCH 261/664] chore: cloudwatch alert added to jenkins old and new server --- playbooks/jenkins_data_engineering.yml | 3 +++ playbooks/jenkins_data_engineering_new.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/playbooks/jenkins_data_engineering.yml b/playbooks/jenkins_data_engineering.yml index 2f416563e5d..f78e1f1ee31 100644 --- a/playbooks/jenkins_data_engineering.yml +++ b/playbooks/jenkins_data_engineering.yml @@ -32,3 +32,6 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE tags: - newreliconly + - role: aws_cloudwatch_agent + tags: + - cloudwatch diff --git a/playbooks/jenkins_data_engineering_new.yml b/playbooks/jenkins_data_engineering_new.yml index 0b3cdf0d5c1..5d7176f8198 100644 --- a/playbooks/jenkins_data_engineering_new.yml +++ b/playbooks/jenkins_data_engineering_new.yml @@ -34,3 +34,6 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE tags: - newreliconly + - role: aws_cloudwatch_agent + tags: + - cloudwatch From f24f140a06b89aabb5fc9f45fafcec84d371665f Mon Sep 17 00:00:00 2001 From: Syed Imran Hassan <45480841+syedimranhassan@users.noreply.github.com> Date: Tue, 8 Mar 2022 13:54:36 +0500 Subject: [PATCH 262/664] Revert "chore: Update build jenkins plugins" --- .../roles/jenkins_build/defaults/main.yml | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml index e4dac32418f..8eb05c54799 100644 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ b/playbooks/roles/jenkins_build/defaults/main.yml @@ -32,9 +32,6 @@ build_jenkins_plugins_list: - name: 'analysis-core' version: '1.96' group: 'org.jvnet.hudson.plugins' - - name: 'analysis-model-api' - version: '10.9.1' - group: 'io.jenkins.plugins' - name: 'ansicolor' version: '1.0.0' group: 'org.jenkins-ci.plugins' @@ -42,7 +39,7 @@ build_jenkins_plugins_list: version: '1.12' group: 'org.jenkins-ci.plugins' - name: 'antisamy-markup-formatter' - version: '2.7' + version: '2.1' group: 'org.jenkins-ci.plugins' - name: 'aws-credentials' version: '1.28.1' @@ -51,10 +48,10 @@ build_jenkins_plugins_list: version: '1.12.101-300.vc09c7be9cb57' group: 'org.jenkins-ci.plugins' - name: 'badge' - version: '1.9.1' + version: '1.9' group: 'org.jenkins-ci.plugins' - name: 'bootstrap5-api' - version: '5.1.3-4' + version: '5.1.1-1' group: 'io.jenkins.plugins' - name: 'bouncycastle-api' version: '2.25' @@ -68,9 +65,6 @@ build_jenkins_plugins_list: - name: 'build-user-vars-plugin' version: '1.5' group: 'org.jenkins-ci.plugins' - - name: 'branch-api' - version: '2.7.0' - group: 'org.jenkins-ci.plugins' - name: 'checks-api' version: '1.7.2' group: 'io.jenkins.plugins' @@ -90,7 +84,7 @@ build_jenkins_plugins_list: version: '2.6.1' group: 'org.jenkins-ci.plugins' - name: 'credentials-binding' - version: '1.27.1' + version: '1.27' group: 'org.jenkins-ci.plugins' - name: 'cvs' version: '2.19' @@ -102,7 +96,7 @@ build_jenkins_plugins_list: version: '2.3.5' group: 'org.jenkins-ci.plugins' - name: 'docker-commons' - version: '1.18' + version: '1.14' group: 'org.jenkins-ci.plugins' - name: 'ec2' version: '1.50.3' @@ -114,7 +108,7 @@ build_jenkins_plugins_list: version: '2.3.0' group: 'org.jenkins-ci.plugins' - name: 'echarts-api' - version: '5.2.2-2' + version: '5.2.1-1' group: 'io.jenkins.plugins' - name: 'exclusive-execution' version: '0.8' @@ -129,7 +123,7 @@ build_jenkins_plugins_list: version: '5.15.4-1' group: 'io.jenkins.plugins' - name: 'forensics-api' - version: '1.7.0' + version: '1.5.0' group: 'io.jenkins.plugins' - name: 'ghprb' version: '1.42.2' @@ -201,7 +195,7 @@ build_jenkins_plugins_list: version: '2.6.8' group: 'org.jenkins-ci.plugins' - name: 'matrix-project' - version: '1.20' + version: '1.18' group: 'org.jenkins-ci.plugins' - name: 'maven-plugin' version: '3.4' @@ -224,26 +218,14 @@ build_jenkins_plugins_list: - name: 'pipeline-model-definition' version: '1.3.4.1' group: 'org.jenkinsci.plugins' - - name: 'pipeline-model-api' - version: '1.9.3' - group: 'org.jenkinsci.plugins' - - name: 'pipeline-model-extensions' - version: '1.9.3' - group: 'org.jenkinsci.plugins' - - name: 'pipeline-stage-step' - version: '291.vf0a8a7aeeb50' - group: 'org.jenkins-ci.plugins' - name: 'pipeline-utility-steps' version: '2.10.0' group: 'org.jenkins-ci.plugins' - name: 'plugin-util-api' - version: '2.12.0' - group: 'io.jenkins.plugins' - - name: 'prism-api' - version: '1.25.0-2' + version: '2.5.0' group: 'io.jenkins.plugins' - name: 'popper2-api' - version: '2.11.2-1' + version: '2.10.2-1' group: 'io.jenkins.plugins' - name: 'PrioritySorter' version: '2.9' @@ -263,9 +245,6 @@ build_jenkins_plugins_list: - name: 'run-condition' version: '1.5' group: 'org.jenkins-ci.plugins' - - name: 'scm-api' - version: '595.vd5a_df5eb_0e39' - group: 'org.jenkins-ci.plugins' - name: 'script-security' version: '1.78' group: 'org.jenkins-ci.plugins' @@ -282,16 +261,16 @@ build_jenkins_plugins_list: version: '1.9.7' group: 'com.splunk.splunkins' - name: 'ssh-agent' - version: '1.24.1' + version: '1.22' group: 'org.jenkins-ci.plugins' - name: 'ssh-credentials' - version: '1.19' + version: '1.18.1' group: 'org.jenkins-ci.plugins' - name: 'ssh-slaves' version: '1.28.1' group: 'org.jenkins-ci.plugins' - name: 'structs' - version: '308.v852b473a2b8c' + version: '1.23' group: 'org.jenkins-ci.plugins' - name: 'subversion' version: '2.15.1' @@ -321,7 +300,7 @@ build_jenkins_plugins_list: version: '0.7.11' group: 'org.jenkins-ci.plugins' - name: 'warnings-ng' - version: '9.11.0' + version: '9.5.1' group: 'io.jenkins.plugins' - name: 'workflow-aggregator' version: '2.6' @@ -342,7 +321,7 @@ build_jenkins_plugins_list: version: '2.42' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-multibranch' - version: '706.vd43c65dec013' + version: '2.24' group: 'org.jenkins-ci.plugins.workflow' - name: 'workflow-scm-step' version: '2.13' From 2c40faa8fb2d825606abb4263c4d87980147f1c6 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 8 Mar 2022 19:03:06 +0500 Subject: [PATCH 263/664] feat: update flowerconfig to add basic_auth support --- playbooks/roles/flower/defaults/main.yml | 3 +++ .../roles/flower/templates/edx/app/flower/flowerconfig.py.j2 | 1 + 2 files changed, 4 insertions(+) diff --git a/playbooks/roles/flower/defaults/main.yml b/playbooks/roles/flower/defaults/main.yml index a080bd445d5..a83525c144c 100644 --- a/playbooks/roles/flower/defaults/main.yml +++ b/playbooks/roles/flower/defaults/main.yml @@ -13,6 +13,9 @@ FLOWER_OAUTH2_KEY: "A Client ID from Google's OAUTH2 provider" FLOWER_OAUTH2_SECRET: "A Client Secret from Google's OAUTH2 provider" FLOWER_OAUTH2_REDIRECT: "A URL registered with Google's OAUTH2 provider" FLOWER_AUTH_REGEX: ".*@example.com" # Can be blank to disable auth +# A list of user:password pairs seperated by a comma to restrict flower access +# using usernames and passwords +FLOWER_BASIC_AUTH: [] FLOWER_USER: "flower" flower_app_dir: "{{ COMMON_APP_DIR }}/{{ FLOWER_USER }}" diff --git a/playbooks/roles/flower/templates/edx/app/flower/flowerconfig.py.j2 b/playbooks/roles/flower/templates/edx/app/flower/flowerconfig.py.j2 index 1199499aab1..f7a7d5fd49f 100644 --- a/playbooks/roles/flower/templates/edx/app/flower/flowerconfig.py.j2 +++ b/playbooks/roles/flower/templates/edx/app/flower/flowerconfig.py.j2 @@ -5,3 +5,4 @@ oauth2_key = "{{ FLOWER_OAUTH2_KEY }}" oauth2_secret = "{{ FLOWER_OAUTH2_SECRET }}" oauth2_redirect_uri = "{{ FLOWER_OAUTH2_REDIRECT }}" auth = "{{ FLOWER_AUTH_REGEX }}" +basic_auth = {{ FLOWER_BASIC_AUTH }} From 0cdf7411b116476168e85432fda442bc1cdc71f6 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 8 Mar 2022 13:12:46 -0500 Subject: [PATCH 264/664] feat: npm installation doesn't perform known host checking JIRA:AU-549 --- playbooks/roles/edxapp/defaults/main.yml | 3 +++ playbooks/roles/edxapp/tasks/deploy.yml | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 7aaaf5d5815..b46a8c92169 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1159,6 +1159,9 @@ edxapp_environment_extra: {} edxapp_environment: "{{ edxapp_environment_default | combine(edxapp_environment_extra) }}" +git_ssh_environment_mixin: + GIT_SSH: "{{ edxapp_git_ssh }}" + edxapp_generic_contentstore_config: &edxapp_generic_default_contentstore ENGINE: 'xmodule.contentstore.mongo.MongoContentStore' # diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 86c3d2fa9f1..20bc2629757 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -237,12 +237,11 @@ - install - install:app-requirements -#install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: install node dependencies - shell: "easy_install --version && npm install" + shell: "easy_install --version && npm ci" args: chdir: "{{ edxapp_code_dir }}" - environment: "{{ edxapp_environment }}" + environment: "{{ edxapp_environment | combine(git_ssh_environment_mixin) }}" become_user: "{{ edxapp_user }}" tags: - install From 8a4bcde7eb3e123202909b106b9b7974c1708ae8 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 17 Mar 2022 10:57:04 +0500 Subject: [PATCH 265/664] fix: add default for automated user --- playbooks/roles/analytics_api/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index 1b034e28bc7..16595bd7e13 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -185,3 +185,6 @@ ANALYTICS_API_HERMES_ENABLED: "{{ COMMON_HERMES_ENABLED }}" ANALYTICS_API_DECRYPT_CONFIG_ENABLED: "{{ COMMON_DECRYPT_CONFIG_ENABLED }}" ANALYTICS_API_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" + +# See edx_django_service_automated_users for an example of what this should be +ANALYTICS_API_AUTOMATED_USERS: {} From bbfcfdb14cad6a130729af2eef4e098402d9ef22 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 17 Mar 2022 13:56:08 -0400 Subject: [PATCH 266/664] feat!: Drop settings related to importing the demo-test-course. That course is being removed per https://github.com/openedx/public-engineering/issues/73 It looks like the mentions in here were vestigial as there were settings set to import the course but no use of the settings once they were set. BREAKING_CHANGE: We're removing the ENABLE_DEMO_TEST_COURSE from ansible_provision.sh meaning if there is something elsewhere in the system making use of that, it may no longer import this course. --- playbooks/sample_vars/server_vars.yml | 7 ------- util/jenkins/ansible-provision.sh | 5 ----- 2 files changed, 12 deletions(-) diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index ff14f08a23f..291af0ecb77 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -14,7 +14,6 @@ #COMMON_ENABLE_DATADOG: $enable_datadog #FORUM_NEW_RELIC_ENABLE: $enable_newrelic #ENABLE_PERFORMANCE_COURSE: $performance_course -#ENABLE_DEMO_TEST_COURSE: $demo_test_course #ENABLE_EDX_DEMO_COURSE: $edx_demo_course #EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms #EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms @@ -159,12 +158,6 @@ #ANALYTICS_API_GIT_IDENTITY: "{{ _local_git_identity }}" # #TESTCOURSES_EXPORTS: -# - github_url: "https://github.com/edx/demo-performance-course.git" -# install: "{{ ENABLE_PERFORMANCE_COURSE }}" -# course_id: "course-v1:DemoX+PERF101+course" -# - github_url: "https://github.com/edx/demo-test-course.git" -# install: "{{ ENABLE_DEMO_TEST_COURSE }}" -# course_id: "course-v1:edX+Test101+course" # - github_url: "https://github.com/edx/edx-demo-course.git" # install: "{{ ENABLE_EDX_DEMO_COURSE }}" # course_id: "course-v1:edX+DemoX+Demo_Course" diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 310cafc0774..2419b3ffd95 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -158,10 +158,6 @@ if [[ -z $performance_course ]]; then performance_course="false" fi -if [[ -z $demo_test_course ]]; then - demo_test_course="false" -fi - if [[ -z $edx_demo_course ]]; then edx_demo_course="false" fi @@ -423,7 +419,6 @@ COMMON_ENABLE_DATADOG: $enable_datadog COMMON_OAUTH_BASE_URL: "https://${deploy_host}" FORUM_NEW_RELIC_ENABLE: $enable_newrelic ENABLE_PERFORMANCE_COURSE: $performance_course -ENABLE_DEMO_TEST_COURSE: $demo_test_course ENABLE_EDX_DEMO_COURSE: $edx_demo_course EDXAPP_ENABLE_AUTO_AUTH: $enable_automatic_auth_for_testing EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms From b703cf9662732e5dbbe484749ea594bfc2c685cc Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Tue, 22 Mar 2022 09:04:14 -0400 Subject: [PATCH 267/664] chore: Use the nginx and aws roles from SRE --- playbooks/roles/dbt_docs_nginx/meta/main.yml | 2 ++ playbooks/roles/dbt_docs_nginx/tasks/main.yml | 12 ++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/dbt_docs_nginx/meta/main.yml b/playbooks/roles/dbt_docs_nginx/meta/main.yml index 202420c7af1..feab3e69901 100644 --- a/playbooks/roles/dbt_docs_nginx/meta/main.yml +++ b/playbooks/roles/dbt_docs_nginx/meta/main.yml @@ -1,3 +1,5 @@ --- dependencies: + - aws + - nginx - aws_cloudwatch_agent \ No newline at end of file diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml index 2eaf67b1c41..ed3d7ad31be 100644 --- a/playbooks/roles/dbt_docs_nginx/tasks/main.yml +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -9,18 +9,10 @@ - install - install:base -- name: Install AWS CLI +- name: Uninstall AWS CLI from the package manager apt: name: awscli - state: present - tags: - - install - - install:base - -- name: Install and start nginx - apt: - name: nginx - state: present + state: absent tags: - install - install:base From f6aecda050b7df28b35bed16a5f374bb00330af8 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Tue, 22 Mar 2022 14:30:59 -0400 Subject: [PATCH 268/664] feat: Put the nginx config into a template and use the template module --- playbooks/roles/dbt_docs_nginx/README.md | 10 +++ playbooks/roles/dbt_docs_nginx/tasks/main.yml | 33 ++++------ .../templates/nginx/nginx.conf.j2 | 65 +++++++++++++++++++ 3 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 playbooks/roles/dbt_docs_nginx/README.md create mode 100644 playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 diff --git a/playbooks/roles/dbt_docs_nginx/README.md b/playbooks/roles/dbt_docs_nginx/README.md new file mode 100644 index 00000000000..07071626c15 --- /dev/null +++ b/playbooks/roles/dbt_docs_nginx/README.md @@ -0,0 +1,10 @@ +### Configurations for provisioning the dbt docs server + +### Main role: +This is the main role that defines all the tasks necessary for provisioning the dbtdocs server. It is used by the `deploy_nginx_for_dbt_docs.yml` playbook. + +### Dependencies: +This role uses the `aws`, `aws_cloudwatch_agent` and `nginx` as dependencies. Therefore, it uses all the default values of those roles. + +### Variables: +The only variable that this role requires is `hostname_variable`, which is the string hostname value that comes before `.edx.org`. For instance, to provision the server for the full hostname `hello.world.edx.org`, then set `hostname_variable` to `hello.world`. diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml index ed3d7ad31be..9567fb22042 100644 --- a/playbooks/roles/dbt_docs_nginx/tasks/main.yml +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -15,7 +15,7 @@ state: absent tags: - install - - install:base + - install:aws - name: Copy dbt-docs html files from S3 to the local machine shell: 'aws s3 sync s3://edx-dbt-docs/ /usr/share/nginx/html/ --delete --include "*"' @@ -23,9 +23,9 @@ become_method: sudo tags: - install - - install:base + - install:docs -- name: Make a Server Configuration Directory +- name: Make a directory to store cron scripts file: path: /home/server-config/ state: directory @@ -36,24 +36,15 @@ - install - install:base -- name: Copy dbt-docs nginx server configurations from S3 - shell: 'aws s3 cp s3://edx-dbt-docs/server-config/ /home/server-config/ --recursive' - become: yes - become_method: sudo - tags: - - install - - install:base - -- name: Copy the nginx configuration file from S3 to the local nginx config directory - copy: - src: /home/server-config/nginx.conf +- name: Copy the nginx configuration file + template: + src: "nginx/nginx.conf.j2" dest: /etc/nginx/nginx.conf - remote_src: yes become: yes become_method: sudo tags: - install - - install:base + - install:nginx - name: Make the dbt-files-sync.sh script executable file: @@ -63,23 +54,23 @@ become_method: sudo tags: - install - - install:base + - install:cronjob - name: Set up cron job to update the html files from S3 - ansible.builtin.cron: + cron: name: Run the /home/server-config/dbt-files-sync.sh script minute: 30 hour: "*/6" job: /home/server-config/dbt-files-sync.sh tags: - install - - install:base + - install:cronjob - name: Restart nginx and enable it on reboot - ansible.builtin.service: + service: name: nginx state: restarted enabled: yes tags: - install - - install:base + - install:nginx diff --git a/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 b/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 new file mode 100644 index 00000000000..f5ac702b314 --- /dev/null +++ b/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 @@ -0,0 +1,65 @@ +user www-data; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # SSL Settings + ## + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Gzip Settings + ## + + gzip on; + + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; + server{ + + server_name {{ hostname_variable }}.edx.org; + root /usr/share/nginx/html; + + } + + +} From 08bf612abd6c026dfcd3af91ce1770960e6abef0 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Tue, 22 Mar 2022 17:08:23 -0400 Subject: [PATCH 269/664] feat: Specify aws, nginx, and aws_cloudwatch_agent as roles --- playbooks/deploy_nginx_for_dbt_docs.yml | 5 ++++- playbooks/roles/dbt_docs_nginx/README.md | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/playbooks/deploy_nginx_for_dbt_docs.yml b/playbooks/deploy_nginx_for_dbt_docs.yml index 961108c100a..cefc7258a43 100644 --- a/playbooks/deploy_nginx_for_dbt_docs.yml +++ b/playbooks/deploy_nginx_for_dbt_docs.yml @@ -3,4 +3,7 @@ become: True gather_facts: True roles: - - dbt_docs_nginx \ No newline at end of file + - aws + - nginx + - aws_cloudwatch_agent + - dbt_docs_nginx \ No newline at end of file diff --git a/playbooks/roles/dbt_docs_nginx/README.md b/playbooks/roles/dbt_docs_nginx/README.md index 07071626c15..38d2be6d57e 100644 --- a/playbooks/roles/dbt_docs_nginx/README.md +++ b/playbooks/roles/dbt_docs_nginx/README.md @@ -7,4 +7,7 @@ This is the main role that defines all the tasks necessary for provisioning the This role uses the `aws`, `aws_cloudwatch_agent` and `nginx` as dependencies. Therefore, it uses all the default values of those roles. ### Variables: -The only variable that this role requires is `hostname_variable`, which is the string hostname value that comes before `.edx.org`. For instance, to provision the server for the full hostname `hello.world.edx.org`, then set `hostname_variable` to `hello.world`. +The following variables are required by this role: +- `hostname_variable`: This is the string hostname value that comes before `.edx.org`. For instance, to provision the server for the full hostname `hello.world.edx.org`, then set `hostname_variable` to `hello.world`. +- `s3_bucket`: This is the name of the S3 bucket where the compiled html files are stored. + From f8eb9492f11d30bc01cce64b2445f937ed582a02 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Tue, 22 Mar 2022 17:09:29 -0400 Subject: [PATCH 270/664] feat: Template the cron bash script rather than pull it from S3 --- playbooks/roles/dbt_docs_nginx/meta/main.yml | 5 ----- playbooks/roles/dbt_docs_nginx/tasks/main.yml | 13 +++++++------ .../templates/nginx/dbt_files_sync.sh.j2 | 2 ++ 3 files changed, 9 insertions(+), 11 deletions(-) create mode 100644 playbooks/roles/dbt_docs_nginx/templates/nginx/dbt_files_sync.sh.j2 diff --git a/playbooks/roles/dbt_docs_nginx/meta/main.yml b/playbooks/roles/dbt_docs_nginx/meta/main.yml index feab3e69901..e69de29bb2d 100644 --- a/playbooks/roles/dbt_docs_nginx/meta/main.yml +++ b/playbooks/roles/dbt_docs_nginx/meta/main.yml @@ -1,5 +0,0 @@ ---- -dependencies: - - aws - - nginx - - aws_cloudwatch_agent \ No newline at end of file diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml index 9567fb22042..d9be3a9ad2b 100644 --- a/playbooks/roles/dbt_docs_nginx/tasks/main.yml +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -9,7 +9,7 @@ - install - install:base -- name: Uninstall AWS CLI from the package manager +- name: Uninstall AWS CLI from the package manager to avoid conflicts with the aws role apt: name: awscli state: absent @@ -46,9 +46,10 @@ - install - install:nginx -- name: Make the dbt-files-sync.sh script executable - file: - path: /home/server-config/dbt-files-sync.sh +- name: Make the dbt-files-sync.sh cron script + template: + src: "nginx/dbt_files_sync.sh.j2" + dest: /home/server-config/dbt_files_sync.sh mode: "a+x" become: yes become_method: sudo @@ -58,10 +59,10 @@ - name: Set up cron job to update the html files from S3 cron: - name: Run the /home/server-config/dbt-files-sync.sh script + name: Run the /home/server-config/dbt_files_sync.sh script minute: 30 hour: "*/6" - job: /home/server-config/dbt-files-sync.sh + job: /home/server-config/dbt_files_sync.sh tags: - install - install:cronjob diff --git a/playbooks/roles/dbt_docs_nginx/templates/nginx/dbt_files_sync.sh.j2 b/playbooks/roles/dbt_docs_nginx/templates/nginx/dbt_files_sync.sh.j2 new file mode 100644 index 00000000000..93374c4e13f --- /dev/null +++ b/playbooks/roles/dbt_docs_nginx/templates/nginx/dbt_files_sync.sh.j2 @@ -0,0 +1,2 @@ +#!/bin/bash +sudo aws s3 sync s3://{{ s3_bucket }}/ /usr/share/nginx/html/ --delete --include "*" \ No newline at end of file From fdb2640d9e2dc31269140d6afe47d8dac9d51ece Mon Sep 17 00:00:00 2001 From: Hamza442 Date: Thu, 24 Mar 2022 13:22:30 +0500 Subject: [PATCH 271/664] chore: updated vault cli for jenkins old and new --- playbooks/roles/jenkins_data_engineering/defaults/main.yml | 2 +- playbooks/roles/jenkins_data_engineering/tasks/main.yml | 2 +- playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 2 +- playbooks/roles/jenkins_data_engineering_new/tasks/main.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index 1e2225a0284..ac6ca73d54d 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -41,7 +41,7 @@ JENKINS_DATA_ENGINEERING_EXTRA_PKGS: - zip # See https://www.vaultproject.io/downloads for latest version. -DE_JENKINS_VAULT_CLI_VERSION: 1.5.4 +DE_JENKINS_VAULT_CLI_VERSION: 1.9.4 # plugins de_jenkins_plugins_list: diff --git a/playbooks/roles/jenkins_data_engineering/tasks/main.yml b/playbooks/roles/jenkins_data_engineering/tasks/main.yml index 22da014bf1a..4a5e66e735b 100644 --- a/playbooks/roles/jenkins_data_engineering/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering/tasks/main.yml @@ -21,7 +21,7 @@ get_url: url: "https://releases.hashicorp.com/vault/{{ DE_JENKINS_VAULT_CLI_VERSION }}/vault_{{ DE_JENKINS_VAULT_CLI_VERSION }}_linux_amd64.zip" dest: "/tmp/vault_{{ DE_JENKINS_VAULT_CLI_VERSION }}_linux_amd64.zip" - checksum: sha256:50156e687b25b253a63c83b649184c79a1311f862c36f4ba16fd020ece4ed3b3 + checksum: sha256:9be49dc07a1b73cc78dd5e5cca88588758bb1994fd954ae2c983eb5986887db5 tags: - jenkins-vault - name: install vault CLI globally diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 00989c38356..49848704e00 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -74,7 +74,7 @@ JENKINS_DATA_ENGINEERING_EXTRA_PKGS: - zip # See https://www.vaultproject.io/downloads for latest version. -DE_JENKINS_VAULT_CLI_VERSION: 1.5.4 +DE_JENKINS_VAULT_CLI_VERSION: 1.9.4 # plugins de_jenkins_plugins_list: diff --git a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml index 1940750c0d3..8b873d29f6f 100644 --- a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml @@ -21,7 +21,7 @@ get_url: url: "https://releases.hashicorp.com/vault/{{ DE_JENKINS_VAULT_CLI_VERSION }}/vault_{{ DE_JENKINS_VAULT_CLI_VERSION }}_linux_amd64.zip" dest: "/tmp/vault_{{ DE_JENKINS_VAULT_CLI_VERSION }}_linux_amd64.zip" - checksum: sha256:50156e687b25b253a63c83b649184c79a1311f862c36f4ba16fd020ece4ed3b3 + checksum: sha256:9be49dc07a1b73cc78dd5e5cca88588758bb1994fd954ae2c983eb5986887db5 tags: - jenkins-vault - name: install vault CLI globally From 7cedf28d8c73fd63450c0b221c797cd1a6d1923a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 24 Mar 2022 17:29:19 +0500 Subject: [PATCH 272/664] feat: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 4 ++-- playbooks/roles/mongo_mms/tasks/main.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index a3dcd525379..19b2e077149 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -2,12 +2,12 @@ base_url: "https://cloud.mongodb.com/download/agent" pkg_arch: "amd64" pkg_format: "deb" -os_version: "ubuntu1604" +os_version: "ubuntu1604" mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.9.0.7253-1" + version: "11.13.1.7441-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" diff --git a/playbooks/roles/mongo_mms/tasks/main.yml b/playbooks/roles/mongo_mms/tasks/main.yml index 71253e4e39a..721308d88f5 100644 --- a/playbooks/roles/mongo_mms/tasks/main.yml +++ b/playbooks/roles/mongo_mms/tasks/main.yml @@ -20,7 +20,7 @@ - name: install mongodb agent apt: - deb: "/tmp/{{ item.agent }}-{{ item.version }}.deb" + deb: "/tmp/{{ item.agent }}_{{ item.version }}.deb" when: download_mongodb_deb.changed with_items: "{{ mongodb_agent }}" From 01f552f7667097e6be2295e48b992d569574deea Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Fri, 25 Mar 2022 10:16:51 +0200 Subject: [PATCH 273/664] feat: added new setting CUSTOM_RESOURCE_TEMPLATES_DIRECTORY This setting allows loading of Resource Templates from outside the edx-platform codebase. Operators will be able to add their own custom templates without needing to fork edx-platform. --- CHANGELOG.md | 5 +++++ playbooks/roles/edxapp/defaults/main.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24bfe528b36..662eb078412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-03-25 + - Role: edxapp + - Added a new `CUSTOM_RESOURCE_TEMPLATES_DIRECTORY` setting to allow operators to + override the default resource templates. + - 2022-02-01 - Role: edxapp - Added a new `EDXAPP_PREPEND_LOCALE_PATHS` setting to allow operators to diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 1a8f711fbe3..07bac68a280 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -881,6 +881,9 @@ EDXAPP_PREPEND_LOCALE_PATHS: [] EDXAPP_DEFAULT_SITE_THEME: "" EDXAPP_ENABLE_COMPREHENSIVE_THEMING: false +# Path to directory to load custom resource templates for the studio/lms. +EDXAPP_CUSTOM_RESOURCE_TEMPLATES_DIRECTORY: null + # Git repo for the comprehensive theme (if using a comprehensive theme # other than the ones bundled with edx/platform) EDXAPP_COMPREHENSIVE_THEME_SOURCE_REPO: "" @@ -1488,6 +1491,7 @@ generic_env_config: &edxapp_generic_env COMPREHENSIVE_THEME_LOCALE_PATHS: "{{ EDXAPP_COMPREHENSIVE_THEME_LOCALE_PATHS }}" PREPEND_LOCALE_PATHS: "{{ EDXAPP_PREPEND_LOCALE_PATHS }}" ENABLE_COMPREHENSIVE_THEMING: "{{ EDXAPP_ENABLE_COMPREHENSIVE_THEMING }}" + CUSTOM_RESOURCE_TEMPLATES_DIRECTORY: "{{ EDXAPP_CUSTOM_RESOURCE_TEMPLATES_DIRECTORY }}" DEFAULT_SITE_THEME: "{{ EDXAPP_DEFAULT_SITE_THEME }}" SESSION_SAVE_EVERY_REQUEST: "{{ EDXAPP_SESSION_SAVE_EVERY_REQUEST }}" SOCIAL_SHARING_SETTINGS: "{{ EDXAPP_SOCIAL_SHARING_SETTINGS }}" From 4b7e4f434c02236a92570d27943fefa52ef4b2be Mon Sep 17 00:00:00 2001 From: Waheed Ahmed Date: Wed, 30 Mar 2022 12:43:57 +0500 Subject: [PATCH 274/664] fix: CORS issue for CSRF token API If basic auth is enabled on sandbox, CSRF token endpoint is resolving to location / and setting auth_basic to `Restricted` for this endpoint. Fixed by defining its own rule. --- .../nginx/templates/edx/app/nginx/sites-available/lms.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 index e40dedd8bdc..b55931a7f99 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 @@ -199,6 +199,11 @@ error_page {{ k }} {{ v }}; try_files $uri @proxy_to_lms_app_api; } + # CSRF token API + location /csrf/api { + try_files $uri @proxy_to_lms_app_api; + } + #enterprise API location /enterprise/api { try_files $uri @proxy_to_lms_app_api; From 9641fb76da1757f68256fd61115df1ca05e3006c Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 31 Mar 2022 11:09:50 +0500 Subject: [PATCH 275/664] feat: update mongo agent --- playbooks/roles/mongo_mms/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index 19b2e077149..17908e9a449 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -2,12 +2,12 @@ base_url: "https://cloud.mongodb.com/download/agent" pkg_arch: "amd64" pkg_format: "deb" - os_version: "ubuntu1604" + mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.13.1.7441-1" + version: "11.13.2.7444-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From 12fcc7c4c27ac519ac561a539fcc70aff568d70d Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 4 Apr 2022 13:55:53 +0500 Subject: [PATCH 276/664] feat: update mongo agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index 17908e9a449..22d3e77272e 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.13.2.7444-1" + version: "11.14.0.7475-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From e3eac5e92b577fe6876ca48a808eca21193768ad Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Mon, 4 Apr 2022 18:25:07 -0400 Subject: [PATCH 277/664] docs: Add description to temporary .github/ISSUE_TEMPLATE files --- .github/ISSUE_TEMPLATE | 91 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE diff --git a/.github/ISSUE_TEMPLATE b/.github/ISSUE_TEMPLATE new file mode 100644 index 00000000000..93251af4205 --- /dev/null +++ b/.github/ISSUE_TEMPLATE @@ -0,0 +1,91 @@ +## This configuration file overrides the inherited configuration file defined +## in openedx/.github/.github/ISSUE_TEMPLATE because this repo currently does +## not have Issues turned on, so we create this override to *only* show DEPR +## issues to users creating Issues. Once Issues are turned on and the repo is +## ready to accept Issues of all types, this file must be deleted so inheritance +## of standard openedx configuration works properly. + +name: Deprecation (DEPR) Ticket +description: Per OEP-21, use this template to begin the technology deprecation process. +title: "[DEPR]: " +labels: ["DEPR"] +body: + - type: markdown + attributes: + value: | + Refer to [OEP-21](https://open-edx-proposals.readthedocs.io/en/latest/processes/oep-0021-proc-deprecation.html) for more detail on the deprecation and removal process. This ticket should only be used for proposing the removal of an Open edX technology. + Please leave [DEPR] in the title of your ticket! + - type: input + id: todays-date + attributes: + label: Proposal Date + description: The start date of this proposal (likely today) + placeholder: 29 February 2020 + validations: + required: true + - type: input + id: accept-date + attributes: + label: Ticket Acceptance Date + description: When is the target date for getting this proposal accepted? + placeholder: 29 February 2020 + validations: + required: true + - type: input + id: remove-date + attributes: + label: Technology Removal Date + description: When is the target date for getting this technology removed? + placeholder: 29 February 2020 + validations: + required: true + - type: input + id: named-release-without + attributes: + label: First Open edX Named Release Without This Functionality + description: Named releases are generally CUT in early April and early October. Based on the above removal date, what named release would be the first without this code? Please reach out to the Build Test Release working group (#wg-build-test-release in Slack) if you're not sure. + placeholder: Dogwood + validations: + required: true + - type: textarea + id: rationale + attributes: + label: Rationale + description: Explain, in a few sentences, why this technology should be removed - what's the usage pattern? What's wrong with keeping it around? + validations: + required: true + - type: textarea + id: removal + attributes: + label: Removal + description: Include a description with links to what exactly is being removed. + validations: + required: true + - type: textarea + id: replacement + attributes: + label: Replacement + description: Include a description with links to what this is being replaced by. + validations: + required: true + - type: textarea + id: deprecation + attributes: + label: Deprecation + description: If you plan to mark the code for deprecation, explain how. + validations: + required: false + - type: textarea + id: migration + attributes: + label: Migration + description: If automated migration will be needed, explain your migration plan. + validations: + required: false + - type: textarea + id: addl-info + attributes: + label: Additional Info + description: If there is any additional publicly sharable information or data from your earlier analysis, include that. + validations: + required: false \ No newline at end of file From 1e55e8a6dedfacb1c470c69495185a61c5b7a968 Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Tue, 11 Jan 2022 08:56:24 +0100 Subject: [PATCH 278/664] feat: add SIMPLETHEME_I18N_DJANGO variable This makes it possible to add or override django translations via the simple-theme role. --- CHANGELOG.md | 5 +++ .../roles/simple_theme/defaults/main.yml | 34 +++++++++++++++ playbooks/roles/simple_theme/tasks/deploy.yml | 42 +++++++++++++++++++ .../simple_theme/templates/i18n/domain.po.j2 | 12 ++++++ 4 files changed, 93 insertions(+) create mode 100644 playbooks/roles/simple_theme/templates/i18n/domain.po.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 662eb078412..25ab4efb338 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-04-06 + - Role: simple_theme + - Added a new `SIMPLETHEME_I18N_DJANGO` setting to allow operators to provide + additional translations, or override existing django translations. + - 2022-03-25 - Role: edxapp - Added a new `CUSTOM_RESOURCE_TEMPLATES_DIRECTORY` setting to allow operators to diff --git a/playbooks/roles/simple_theme/defaults/main.yml b/playbooks/roles/simple_theme/defaults/main.yml index a5651da1944..d04de48f691 100644 --- a/playbooks/roles/simple_theme/defaults/main.yml +++ b/playbooks/roles/simple_theme/defaults/main.yml @@ -106,3 +106,37 @@ SIMPLETHEME_STATIC_FILES_URLS: [] # border-top: 3px solid $main-color; # } SIMPLETHEME_EXTRA_SASS: "" + + +# Use this variable to configure django translations. +# Note that edx-platform does not pick up translations from themes by default. +# You will have to manually configure either `COMPREHENSIVE_THEME_LOCALE_PATHS` or +# `PREPEND_LOCALE_PATHS` to include the path to the theme's i18n/locale folder for +# these translations to get picked up. +# +# The SIMPLETHEME_I18n_DJANGO variable should contain a list of dictionaries with these keys: +# - `lang`: the language code +# - `domain`: the i18n domain (typically one of "django" or "djangojs") +# - `headers`: (optional) Additional PO file headers. +# - `messages`: Translation messages. It should be a raw string of PO formatted messages. +# +# Samle: +# SIMPLETHEME_I18N_DJANGO: +# - lang: en +# domain: django +# headers: | +# "Plural-Forms: nplurals=2; plural=(n > 1);\n" +# messages: | +# msgid "my id" +# msgstr "my translation" +# +# msgid "one" +# msgid_plural "many" +# msgstr[0] "just one" +# msgstr[1] "a lot" +# - lang: en +# domain: djangojs +# messages: | +# msgid "my id" +# msgstr "my JS translation" +SIMPLETHEME_I18N_DJANGO: [] diff --git a/playbooks/roles/simple_theme/tasks/deploy.yml b/playbooks/roles/simple_theme/tasks/deploy.yml index ac601a0bdab..a541fc0ccaf 100644 --- a/playbooks/roles/simple_theme/tasks/deploy.yml +++ b/playbooks/roles/simple_theme/tasks/deploy.yml @@ -137,3 +137,45 @@ owner: "{{ edxapp_user }}" group: "{{ common_web_group }}" with_items: "{{ SIMPLETHEME_STATIC_FILES_URLS }}" + +# Handle translations. +- block: + - name: Install needed packages + apt: + name: gettext + state: present + update_cache: true + cache_valid_time: 3600 + - name: Create directories for django translations + file: + path: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES" + state: directory + owner: "{{ edxapp_user }}" + group: "{{ common_web_group }}" + with_items: "{{ SIMPLETHEME_I18N_DJANGO }}" + - name: Make sure .po files exist + file: + path: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po" + state: touch + owner: "{{ edxapp_user }}" + group: "{{ common_web_group }}" + with_items: "{{ SIMPLETHEME_I18N_DJANGO }}" + - name: Create temporary .po files with custom translations + template: + src: "i18n/domain.po.j2" + dest: "{{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po_" + owner: "{{ edxapp_user }}" + group: "{{ common_web_group }}" + with_items: "{{ SIMPLETHEME_I18N_DJANGO }}" + - name: Merge temporary .po files into default translations + shell: > + msgcat --use-first {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po_ + {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po > + {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po + with_items: "{{ SIMPLETHEME_I18N_DJANGO }}" + - name: Compile .po files into .mo + shell: > + msgfmt {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.po + -o {{ simpletheme_folder }}/i18n/conf/locale/{{ item.lang }}/LC_MESSAGES/{{ item.domain }}.mo + with_items: "{{ SIMPLETHEME_I18N_DJANGO }}" + when: SIMPLETHEME_I18N_DJANGO | length > 0 diff --git a/playbooks/roles/simple_theme/templates/i18n/domain.po.j2 b/playbooks/roles/simple_theme/templates/i18n/domain.po.j2 new file mode 100644 index 00000000000..3e08854bee0 --- /dev/null +++ b/playbooks/roles/simple_theme/templates/i18n/domain.po.j2 @@ -0,0 +1,12 @@ +msgid "" +msgstr "" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: {{ item.lang }}\n" +{% if 'headers' in item -%} +{{ item.headers }} +{%- endif %} + +{% if 'messages' in item -%} +{{ item.messages }} +{%- endif %} From 1fbdc9af60b77e27dbed51861d937e6bcdb3d09f Mon Sep 17 00:00:00 2001 From: Matjaz Gregoric Date: Mon, 28 Mar 2022 17:39:27 +0200 Subject: [PATCH 279/664] feat: optionally collect JS translations on deploy If the EDXAPP_COMPILE_JSI18N is enabled, this will invoke the compilejsi18n management command when deploying edxapp. This is useful when you use custom JS translations for example via a theme. --- CHANGELOG.md | 6 ++++++ playbooks/roles/edxapp/defaults/main.yml | 3 +++ playbooks/roles/edxapp/tasks/deploy.yml | 9 +++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ab4efb338..9fa4dd21a40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-04-06 + - Role: edxapp + - Added a new `EDXAPP_COMPILE_JSI18N` variable to control whether + to run the `compilejsi18n` management command on edxapp deploy. + Defaults to false. + - 2022-04-06 - Role: simple_theme - Added a new `SIMPLETHEME_I18N_DJANGO` setting to allow operators to provide diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 07bac68a280..82e4a44677e 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -629,6 +629,9 @@ EDXAPP_DCS_SESSION_COOKIE_SAMESITE_FORCE_ALL: True # Whether to run reindex_course on deploy EDXAPP_REINDEX_ALL_COURSES: false +# Whether to run compilejsi18n on deploy +EDXAPP_COMPILE_JSI18N: false + # XML Course related flags EDXAPP_XML_FROM_GIT: false EDXAPP_XML_S3_BUCKET: !!null diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 20bc2629757..2270c052c1d 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -381,6 +381,15 @@ when: - celery_worker is not defined +- name: compile JS translations + shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} compilejsi18n" + args: + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + when: EDXAPP_COMPILE_JSI18N and celery_worker is not defined + tags: + - assets + # creates the supervisor jobs for the # service variants configured, runs # gather_assets and db migrations From ac0dd337e01f94cbbc0cc2d76a16fced43bc0c96 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 7 Apr 2022 14:34:35 -0400 Subject: [PATCH 280/664] feat: add conductor support for newer versions of gatsby Newer versions of gatsby appear to rely on files it generates in the page-data directory, which didn't match our previous regular expression. --- .../templates/edx/app/nginx/sites-available/conductor.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 index 816313e21f5..9e5bb04ba08 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 @@ -55,6 +55,11 @@ server { proxy_pass {{ static_site.proxied_path }}/$1$2; } + # Matches: //page-data/[.../] + location ~ ^/{{ static_site.router_path }}/page-data/((?:\w+\/+)*)([\w\-\.]+\.[\w\-\.]+) { + proxy_pass {{ static_site.proxied_path }}/page-data/$1$2; + } + # Matches: ///[.../] location ~ ^/{{ static_site.router_path }}/([a-z0-9-]+)[/]? { proxy_pass {{ static_site.proxied_path }}/$1/index.html; From f7b04c915b327fdcfa3e17e0eafb64261e939e92 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Thu, 7 Apr 2022 17:05:36 -0400 Subject: [PATCH 281/664] feat: let conductor proxy s3 file paths containing hyphens generalization of https://github.com/openedx/configuration/pull/6713 --- .../templates/edx/app/nginx/sites-available/conductor.j2 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 index 9e5bb04ba08..5c193f0e1a8 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/conductor.j2 @@ -51,15 +51,10 @@ server { } # Matches: //[.../] - location ~ ^/{{ static_site.router_path }}/((?:\w+\/+)*)([\w\-\.]+\.[\w\-\.]+) { + location ~ ^/{{ static_site.router_path }}/((?:[\w\-]+\/+)*)([\w\-\.]+\.[\w\-\.]+) { proxy_pass {{ static_site.proxied_path }}/$1$2; } - # Matches: //page-data/[.../] - location ~ ^/{{ static_site.router_path }}/page-data/((?:\w+\/+)*)([\w\-\.]+\.[\w\-\.]+) { - proxy_pass {{ static_site.proxied_path }}/page-data/$1$2; - } - # Matches: ///[.../] location ~ ^/{{ static_site.router_path }}/([a-z0-9-]+)[/]? { proxy_pass {{ static_site.proxied_path }}/$1/index.html; From b7085299d2d6e2150171779a195b69a7d5bca810 Mon Sep 17 00:00:00 2001 From: Jawayria <39649635+Jawayria@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:02:44 +0500 Subject: [PATCH 282/664] chore!: Upgraded to Node 16 and NPM 8 (#6692) * chore!: Upgraded to Node 16 and NPM 8 BREAKING CHANGE: Node and NPM versions upgraded * fix: include dev reqs also during installation of npm overrides As we are setting `NODE_ENV=production` in env vars, so if we install npm packages from `MFE_NPM_OVERRIDES` list without `--include=dev`, it uninstalls dev requirements which is causing build failure in node 16. - Also changing `--dev` with `--include=dev` as earlier one is deprecated Co-authored-by: Soban Javed --- playbooks/roles/mfe/defaults/main.yml | 4 ++-- playbooks/roles/mfe/tasks/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 02c5a9bf0e8..7d99dc5a94e 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -15,8 +15,8 @@ MFE_NODEENV_DIR: '{{ MFE_HOME }}/nodeenvs/{{ MFE_NAME }}' MFE_NODEENV_BIN: '{{ MFE_NODEENV_DIR }}/bin' MFE_NODE_MODULES_DIR: '{{ MFE_CODE_DIR }}/node_modules' MFE_NODE_BIN: '{{ MFE_NODE_MODULES_DIR }}/.bin' -MFE_NODE_VERSION: '12.13.1' -MFE_NPM_VERSION: '6.12.1' +MFE_NODE_VERSION: '16.13.2' +MFE_NPM_VERSION: '8.1.2' MFE_DEBIAN_PKGS_DEFAULT: - gettext diff --git a/playbooks/roles/mfe/tasks/main.yml b/playbooks/roles/mfe/tasks/main.yml index 3dc4d63b407..fbea317ffcc 100644 --- a/playbooks/roles/mfe/tasks/main.yml +++ b/playbooks/roles/mfe/tasks/main.yml @@ -75,7 +75,7 @@ - install:system-requirements - name: install npm dependencies - shell: "npm install --dev --no-save" + shell: "npm install --include=dev --no-save" args: chdir: "{{ MFE_CODE_DIR }}" become_user: "{{ MFE_USER }}" @@ -85,7 +85,7 @@ - install:app-requirements - name: install npm overrides - shell: "npm install {{ item }} --no-save" + shell: "npm install {{ item }} --include=dev --no-save" args: chdir: "{{ MFE_CODE_DIR }}" become_user: "{{ MFE_USER }}" From 9babf2a7bd41b04e2bf1c3cd2aba62596d908bfb Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Tue, 12 Apr 2022 13:46:48 -0400 Subject: [PATCH 283/664] chore: upgrade node version to v16 on learner_portal code base --- playbooks/roles/learner_portal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/learner_portal/defaults/main.yml b/playbooks/roles/learner_portal/defaults/main.yml index db12c14fc5b..98e263f41d9 100644 --- a/playbooks/roles/learner_portal/defaults/main.yml +++ b/playbooks/roles/learner_portal/defaults/main.yml @@ -17,7 +17,7 @@ edx_django_service_use_python3: false learner_portal_repo: 'https://github.com/edx/frontend-app-learner-portal.git' LEARNER_PORTAL_VERSION: 'master' learner_portal_service_name: 'learner_portal' -LEARNER_PORTAL_NODE_VERSION: '12.11.1' +LEARNER_PORTAL_NODE_VERSION: '16.14.2' learner_portal_nodeenv_dir: '{{ learner_portal_home }}/nodeenvs/{{ learner_portal_service_name }}' learner_portal_nodeenv_bin: '{{learner_portal_nodeenv_dir}}/bin' From cedeb5a273b98e217d00af93faed7301ff31acd7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 13 Apr 2022 13:33:30 +0500 Subject: [PATCH 284/664] fix: git clone --- playbooks/roles/git_clone/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/git_clone/tasks/main.yml b/playbooks/roles/git_clone/tasks/main.yml index 09d3bf32dff..bad56a2c0c2 100644 --- a/playbooks/roles/git_clone/tasks/main.yml +++ b/playbooks/roles/git_clone/tasks/main.yml @@ -57,6 +57,7 @@ - name: Check that working tree is clean shell: test ! -e "{{ item }}" || git -C "{{ item }}" status --porcelain --untracked-files=no register: dirty_files + become_user: "{{ repo_owner }}" # Using the map here means that the items will only be the DESTINATION strings, # rather than the full GIT_REPOS structures, which have data we don't want to log, # so we don't have to use no_log on this task. @@ -113,7 +114,7 @@ - name: Run git clean after checking out code shell: cd {{ item.DESTINATION }} && git clean -xdf - become: true + become_user: "{{ repo_owner }}" with_items: "{{ GIT_REPOS }}" no_log: "{{ GIT_CLONE_NO_LOGGING }}" tags: From ec89652043cd1c817d0323b15ff7af5a16a3da3b Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Wed, 13 Apr 2022 10:30:39 -0400 Subject: [PATCH 285/664] chore: Set up tasks to copy the ssh public key of jenkins users to analytics instances --- playbooks/roles/dbt_docs_nginx/tasks/main.yml | 12 ++++++++++++ .../roles/jenkins_data_engineering/tasks/main.yml | 11 +++++++++++ .../jenkins_data_engineering_new/tasks/main.yml | 11 +++++++++++ playbooks/roles/tableau_de/tasks/main.yml | 12 ++++++++++++ 4 files changed, 46 insertions(+) diff --git a/playbooks/roles/dbt_docs_nginx/tasks/main.yml b/playbooks/roles/dbt_docs_nginx/tasks/main.yml index d9be3a9ad2b..82952e46f6b 100644 --- a/playbooks/roles/dbt_docs_nginx/tasks/main.yml +++ b/playbooks/roles/dbt_docs_nginx/tasks/main.yml @@ -75,3 +75,15 @@ tags: - install - install:nginx + +# Add the jenkins user's ssh public key to the running user's autorized keys +# This is needed so that this jenkins instance can be used to update system users +- name: Add the jenkins user's ssh public key to the running user's autorized keys + lineinfile: + path: /home/{{ ansible_ssh_user }}/.ssh/authorized_keys + create: yes + line: "{{ lookup('file', jenkins_ssh_public_keyfile) }}" + when: jenkins_ssh_public_keyfile is defined and jenkins_ssh_public_keyfile + tags: + - ssh + - ssh:keys diff --git a/playbooks/roles/jenkins_data_engineering/tasks/main.yml b/playbooks/roles/jenkins_data_engineering/tasks/main.yml index 4a5e66e735b..8424df84466 100644 --- a/playbooks/roles/jenkins_data_engineering/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering/tasks/main.yml @@ -132,3 +132,14 @@ service: name=jenkins state=restarted tags: - jenkins-auth + +# Add the jenkins user's ssh public key to the running user's autorized keys +# This is needed so that this jenkins instance can be used to update system users +- name: Add the jenkins user's ssh public key to the running user's autorized keys + lineinfile: + path: /home/{{ ansible_ssh_user }}/.ssh/authorized_keys + create: yes + line: "{{ lookup('file', JENKINS_DATA_ENGINEERING_AUTOMATION_PUBLIC_KEY_SOURCE_PATH) }}" + tags: + - ssh + - ssh:keys diff --git a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml index 8b873d29f6f..e0e2a7a47ce 100644 --- a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml @@ -134,3 +134,14 @@ service: name=jenkins state=restarted tags: - jenkins-auth + +# Add the jenkins user's ssh public key to the running user's autorized keys +# This is needed so that this jenkins instance can be used to update system users +- name: Add the jenkins user's ssh public key to the running user's autorized keys + lineinfile: + path: /home/{{ ansible_ssh_user }}/.ssh/authorized_keys + create: yes + line: "{{ lookup('file', JENKINS_DATA_ENGINEERING_AUTOMATION_PUBLIC_KEY_SOURCE_PATH) }}" + tags: + - ssh + - ssh:keys diff --git a/playbooks/roles/tableau_de/tasks/main.yml b/playbooks/roles/tableau_de/tasks/main.yml index 08fce68c5f1..8d4ff7156a2 100644 --- a/playbooks/roles/tableau_de/tasks/main.yml +++ b/playbooks/roles/tableau_de/tasks/main.yml @@ -57,3 +57,15 @@ tags: - install - install:base + +# Add the jenkins user's ssh public key to the running user's autorized keys +# This is needed so that this jenkins instance can be used to update system users +- name: Add the jenkins user's ssh public key to the running user's autorized keys + lineinfile: + path: /home/{{ ansible_ssh_user }}/.ssh/authorized_keys + create: yes + line: "{{ lookup('file', jenkins_ssh_public_keyfile) }}" + when: jenkins_ssh_public_keyfile is defined and jenkins_ssh_public_keyfile + tags: + - ssh + - ssh:keys From a650e82fd1fb2005eb47a1b3cafd8e51bb73d44e Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Thu, 14 Apr 2022 17:15:26 +0500 Subject: [PATCH 286/664] chore: upgrade the node version. --- playbooks/roles/discovery/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 99f6b367909..be77ea1e72f 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -34,7 +34,7 @@ discovery_user: "{{ discovery_service_name }}" discovery_home: "{{ COMMON_APP_DIR }}/{{ discovery_service_name }}" discovery_code_dir: "{{ discovery_home }}/{{ discovery_service_name }}" -DISCOVERY_NODE_VERSION: '12.11.1' +DISCOVERY_NODE_VERSION: '16' DISCOVERY_USE_PYTHON38: True # From b5c1974bd19b908b37a23de2705b3c9f5625e83b Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Fri, 15 Apr 2022 15:09:46 +0500 Subject: [PATCH 287/664] chore: upgrade the node version. --- playbooks/roles/discovery/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index be77ea1e72f..7b4d6d33a15 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -34,7 +34,7 @@ discovery_user: "{{ discovery_service_name }}" discovery_home: "{{ COMMON_APP_DIR }}/{{ discovery_service_name }}" discovery_code_dir: "{{ discovery_home }}/{{ discovery_service_name }}" -DISCOVERY_NODE_VERSION: '16' +DISCOVERY_NODE_VERSION: '16.14.0' DISCOVERY_USE_PYTHON38: True # From 76caa431bf7e629f61e21458931b9517696294ec Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 15 Apr 2022 16:08:17 +0500 Subject: [PATCH 288/664] chore: Update email address to fix upgrade-python-requirements GH action --- .github/workflows/upgrade-python-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index c325a5e42cb..1b7d2ce51d5 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -14,7 +14,7 @@ jobs: with: branch: ${{ github.event.inputs.branch }} team_reviewers: "devops" - email_address: ${{ secrets.REQUIREMENTS_BOT_OPSGENIE_EMAIL }} + email_address: tools-jenkins-isre@edx.opsgenie.net send_success_notification: false secrets: requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} From 1f4ff355d9df684a215617ea7e3d8802864b6f8a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 15 Apr 2022 16:26:44 +0500 Subject: [PATCH 289/664] fix: add safe directory config and use escalated privileges with git clean --- playbooks/roles/git_clone/tasks/main.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/git_clone/tasks/main.yml b/playbooks/roles/git_clone/tasks/main.yml index bad56a2c0c2..646b6536be5 100644 --- a/playbooks/roles/git_clone/tasks/main.yml +++ b/playbooks/roles/git_clone/tasks/main.yml @@ -112,9 +112,18 @@ - install - install:code +- name: Run git safe.directory + shell: git config --global --add safe.directory {{ item.DESTINATION }} + become: true + with_items: "{{ GIT_REPOS }}" + no_log: "{{ GIT_CLONE_NO_LOGGING }}" + tags: + - install + - install:code + - name: Run git clean after checking out code shell: cd {{ item.DESTINATION }} && git clean -xdf - become_user: "{{ repo_owner }}" + become: true with_items: "{{ GIT_REPOS }}" no_log: "{{ GIT_CLONE_NO_LOGGING }}" tags: From e0a81ac739acc0453615c672eb7e9eed816f9b0e Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 18 Apr 2022 14:00:25 +0500 Subject: [PATCH 290/664] feat: Upgrade pip to 21.2.1 --- docker/build/analytics_pipeline/Dockerfile | 2 +- playbooks/roles/common_vars/defaults/main.yml | 2 +- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- pre-requirements.txt | 2 +- util/install/ansible-bootstrap.sh | 2 +- util/install/native.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/build/analytics_pipeline/Dockerfile b/docker/build/analytics_pipeline/Dockerfile index 8dbd4ccb094..3f6e8f1aa89 100644 --- a/docker/build/analytics_pipeline/Dockerfile +++ b/docker/build/analytics_pipeline/Dockerfile @@ -29,7 +29,7 @@ ENV BOTO_CONFIG=/dev/null \ ANALYTICS_PIPELINE_VENV=/edx/app/analytics_pipeline/venvs \ BOOTSTRAP=/etc/bootstrap.sh \ COMMON_BASE_DIR=/edx \ - COMMON_PIP_PACKAGES_PIP='pip==20.3.4' \ + COMMON_PIP_PACKAGES_PIP='pip==21.2.1' \ COMMON_PIP_PACKAGES_SETUPTOOLS='setuptools==44.1.0' \ COMMON_PIP_PACKAGES_VIRTUALENV='virtualenv==20.1.0' \ COMMON_MYSQL_READ_ONLY_USER='read_only' \ diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index e57b9fee9a0..d0757ed491e 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -152,7 +152,7 @@ old_python_debian_pkgs: - "python2.7=2.7.10-0+{{ ansible_distribution_release }}1" -COMMON_PIP_VERSION: '20.3.4' +COMMON_PIP_VERSION: '21.2.1' common_pip_pkgs: - pip=={{ COMMON_PIP_VERSION }} diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 2270c052c1d..385cc0a216a 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -274,7 +274,7 @@ - name: Pin pip to a specific version. # Not pinning to the same version as everything else because sandboxes are still python 2.7 - command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==20.3.4" + command: "{{ edxapp_sandbox_venv_dir }}/bin/pip install pip==21.2.1" args: chdir: "{{ edxapp_code_dir }}" become_user: "{{ edxapp_sandbox_user }}" diff --git a/pre-requirements.txt b/pre-requirements.txt index a9a3cb756e0..ed2bcefab31 100644 --- a/pre-requirements.txt +++ b/pre-requirements.txt @@ -1 +1 @@ -pip==20.3.4 +pip==21.2.1 diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index 3a51e877c75..fafc80b108e 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -33,7 +33,7 @@ fi # Bootstrapping constants # VIRTUAL_ENV_VERSION="16.7.10" -PIP_VERSION="20.3.4" +PIP_VERSION="21.2.1" SETUPTOOLS_VERSION="44.1.0" VIRTUAL_ENV="/tmp/bootstrap" PYTHON_BIN="${VIRTUAL_ENV}/bin" diff --git a/util/install/native.sh b/util/install/native.sh index d5337a499a2..ec37a281618 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -101,7 +101,7 @@ sudo apt-get upgrade -y sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python3-pip libmysqlclient-dev python3-apt python3-dev libxmlsec1-dev libfreetype6-dev swig gcc g++ # ansible-bootstrap installs yaml that pip 19 can't uninstall. sudo apt-get remove -y python-yaml -sudo pip3 install --upgrade pip==20.3.4 +sudo pip3 install --upgrade pip==21.2.1 sudo pip3 install --upgrade setuptools==44.1.0 sudo -H pip3 install --upgrade virtualenv==20.2.0 From 0b6527ba66bef3dff43fa1b8aa5447ed9ee37fc3 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Apr 2022 15:53:42 -0400 Subject: [PATCH 291/664] docs: remove mention of the named release (it was a year out of date anyway) The configuration repo is no longer a supported part of the Open edX software, so we don't need to ping developers every time they open a pull request anymore. --- .github/PULL_REQUEST_TEMPLATE.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 60f4331a23d..b5e6622ff4f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,14 +1,6 @@ Configuration Pull Request --- - - Make sure that the following steps are done before merging: - [ ] A SRE team member has approved the PR if it is code shared across multiple services and you don't own all of the services. From 50978d732a3854d75f380e31811afb54e2df80f6 Mon Sep 17 00:00:00 2001 From: Soban Javed Date: Wed, 20 Apr 2022 15:23:10 +0500 Subject: [PATCH 292/664] fix: use npm v8 for discovery --- playbooks/roles/discovery/defaults/main.yml | 1 + playbooks/roles/discovery/meta/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 7b4d6d33a15..efd7158a2c0 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -35,6 +35,7 @@ discovery_home: "{{ COMMON_APP_DIR }}/{{ discovery_service_name }}" discovery_code_dir: "{{ discovery_home }}/{{ discovery_service_name }}" DISCOVERY_NODE_VERSION: '16.14.0' +DISCOVERY_NPM_VERSION: '8.5.5' DISCOVERY_USE_PYTHON38: True # diff --git a/playbooks/roles/discovery/meta/main.yml b/playbooks/roles/discovery/meta/main.yml index 6a2ef4d7838..7bb3b3d4867 100644 --- a/playbooks/roles/discovery/meta/main.yml +++ b/playbooks/roles/discovery/meta/main.yml @@ -48,6 +48,7 @@ dependencies: edx_django_service_extra_apps: '{{ DISCOVERY_EXTRA_APPS }}' edx_django_service_session_expire_at_browser_close: '{{ DISCOVERY_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' edx_django_service_node_version: '{{ DISCOVERY_NODE_VERSION }}' + edx_django_service_npm_version: '{{ DISCOVERY_NPM_VERSION }}' edx_django_service_automated_users: '{{ DISCOVERY_AUTOMATED_USERS }}' edx_django_service_post_migrate_commands: '{{ DISCOVERY_POST_MIGRATE_COMMANDS }}' edx_django_service_enable_newrelic_distributed_tracing: '{{ DISCOVERY_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}' From 3f2ac85184d6eb9481ddcebc6542fa28e0595cf4 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Tue, 19 Apr 2022 03:25:46 -0400 Subject: [PATCH 293/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 14 +++++----- requirements.txt | 27 +++++++++---------- requirements/pip-tools.txt | 10 +++---- util/elasticsearch/requirements.txt | 4 +-- util/jenkins/requirements-cloudflare.txt | 8 +++--- util/jenkins/requirements.txt | 22 +++++++-------- util/pingdom/requirements.txt | 6 ++--- util/vpc-tools/requirements.txt | 6 ++--- 8 files changed, 48 insertions(+), 49 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 93d02af498c..ab0473a5ae5 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.20.63 +awscli==1.22.97 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.18.63 +boto3==1.21.42 # via -r requirements/aws.in -botocore==1.21.63 +botocore==1.24.42 # via # awscli # boto3 @@ -19,7 +19,7 @@ colorama==0.4.3 # via awscli docutils==0.15.2 # via awscli -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore @@ -29,7 +29,7 @@ python-dateutil==2.8.2 # via # botocore # s3cmd -python-magic==0.4.24 +python-magic==0.4.25 # via s3cmd pyyaml==5.3.1 # via @@ -39,11 +39,11 @@ rsa==4.7.2 # via awscli s3cmd==2.2.0 # via -r requirements/aws.in -s3transfer==0.5.0 +s3transfer==0.5.2 # via # awscli # boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.7 +urllib3==1.26.9 # via botocore diff --git a/requirements.txt b/requirements.txt index 46db7919a7e..05ef66e9c92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.8.20 # via -r requirements/base.in -awscli==1.20.63 +awscli==1.22.97 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,9 +14,9 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.18.63 +boto3==1.21.42 # via -r requirements/base.in -botocore==1.21.63 +botocore==1.24.42 # via # awscli # boto3 @@ -28,17 +28,17 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl -charset-normalizer==2.0.7 +charset-normalizer==2.0.12 # via requests colorama==0.4.3 # via awscli -cryptography==35.0.0 +cryptography==36.0.2 # via # ansible # paramiko datadog==0.8.0 # via -r requirements/base.in -decorator==5.1.0 +decorator==5.1.1 # via # datadog # networkx @@ -54,11 +54,11 @@ jinja2==2.8 # via # -r requirements/base.in # ansible -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore -markupsafe==2.0.1 +markupsafe==2.1.1 # via jinja2 mysqlclient==1.4.6 # via -r requirements/base.in @@ -74,13 +74,13 @@ pyasn1==0.4.8 # via # paramiko # rsa -pycparser==2.20 +pycparser==2.21 # via cffi pycrypto==2.6.1 # via -r requirements/base.in pymongo==3.9.0 # via -r requirements/base.in -pynacl==1.4.0 +pynacl==1.5.0 # via paramiko python-dateutil==2.8.2 # via botocore @@ -89,13 +89,13 @@ pyyaml==5.4.1 # -r requirements/base.in # ansible # awscli -requests==2.26.0 +requests==2.27.1 # via # -r requirements/base.in # datadog rsa==4.7.2 # via awscli -s3transfer==0.5.0 +s3transfer==0.5.2 # via # awscli # boto3 @@ -103,9 +103,8 @@ six==1.16.0 # via # bcrypt # pathlib2 - # pynacl # python-dateutil -urllib3==1.26.7 +urllib3==1.26.9 # via # botocore # requests diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index cbc229e3b6d..a31aa926d35 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,15 +4,15 @@ # # make upgrade # -click==8.0.3 +click==8.1.2 # via pip-tools -pep517==0.11.1 +pep517==0.12.0 # via pip-tools -pip-tools==6.4.0 +pip-tools==6.6.0 # via -r requirements/pip-tools.in -tomli==1.2.1 +tomli==2.0.1 # via pep517 -wheel==0.37.0 +wheel==0.37.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 952c857ee9e..f7e34bd955d 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -8,7 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -jsonpickle==2.0.0 +jsonpickle==2.1.0 # via deepdiff -urllib3==1.26.7 +urllib3==1.26.9 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 1102b13ce93..4a691480f97 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -6,13 +6,13 @@ # certifi==2021.10.8 # via requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.12 # via requests -click==8.0.3 +click==8.1.2 # via -r requirements/cloudflare.in idna==3.3 # via requests -requests==2.26.0 +requests==2.27.1 # via -r requirements/cloudflare.in -urllib3==1.26.7 +urllib3==1.26.9 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index f50a12fcd1d..bedfe3b822d 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.20.63 +awscli==1.22.97 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,9 +16,9 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.18.63 +boto3==1.21.42 # via -r requirements/jenkins.in -botocore==1.21.63 +botocore==1.24.42 # via # awscli # boto3 @@ -29,7 +29,7 @@ certifi==2021.10.8 # via # opsgenie-sdk # requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.12 # via requests click==6.7 # via -r requirements/jenkins.in @@ -39,7 +39,7 @@ docutils==0.15.2 # via awscli idna==3.3 # via requests -jmespath==0.10.0 +jmespath==1.0.0 # via # boto3 # botocore @@ -56,11 +56,11 @@ python-dateutil==2.8.2 # botocore # opsgenie-sdk # s3cmd -python-gnupg==0.4.7 +python-gnupg==0.4.8 # via -r requirements/jenkins.in -python-magic==0.4.24 +python-magic==0.4.25 # via s3cmd -pytz==2021.3 +pytz==2022.1 # via # celery # opsgenie-sdk @@ -70,13 +70,13 @@ pyyaml==5.4.1 # awscli redis==2.10.6 # via -r requirements/jenkins.in -requests==2.26.0 +requests==2.27.1 # via opsgenie-sdk rsa==4.7.2 # via awscli s3cmd==2.2.0 # via -r requirements/jenkins.in -s3transfer==0.5.0 +s3transfer==0.5.2 # via # awscli # boto3 @@ -86,7 +86,7 @@ six==1.16.0 # python-dateutil splunk-sdk==1.6.6 # via -r requirements/jenkins.in -urllib3==1.26.7 +urllib3==1.26.9 # via # botocore # opsgenie-sdk diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index b3dbf7a6f70..bf934d0b7d6 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -6,7 +6,7 @@ # certifi==2021.10.8 # via requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.12 # via requests click==6.7 # via -r requirements/pingdom.in @@ -14,9 +14,9 @@ idna==3.3 # via requests pyyaml==6.0 # via -r requirements/pingdom.in -requests==2.26.0 +requests==2.27.1 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.26.7 +urllib3==1.26.9 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 92b14065b77..78061aa41d8 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -8,13 +8,13 @@ boto==2.49.0 # via -r requirements/vpc-tools.in certifi==2021.10.8 # via requests -charset-normalizer==2.0.7 +charset-normalizer==2.0.12 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in idna==3.3 # via requests -requests==2.26.0 +requests==2.27.1 # via -r requirements/vpc-tools.in -urllib3==1.26.7 +urllib3==1.26.9 # via requests From e8bbe636dbaa1846bdaa8fd24046a933d622023d Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 19 Apr 2022 15:48:37 +0500 Subject: [PATCH 294/664] chore: Pinned markupsafe to 2.0.1 --- requirements.txt | 6 ++++-- requirements/base.in | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 05ef66e9c92..6dcbfe4f1be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,8 +58,10 @@ jmespath==1.0.0 # via # boto3 # botocore -markupsafe==2.1.1 - # via jinja2 +markupsafe==2.0.1 + # via + # -r requirements/base.in + # jinja2 mysqlclient==1.4.6 # via -r requirements/base.in networkx==1.11 diff --git a/requirements/base.in b/requirements/base.in index ab61bceb885..7e12eaefb18 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -9,6 +9,7 @@ datadog==0.8.0 docopt==0.6.2 ecdsa==0.13.3 Jinja2==2.8 +markupsafe==2.0.1 # Pining this until we upgrade jinja2, as in newer version on markupsafe soft_unicode is removed and jinja2==2.8 use this pkg mysqlclient==1.4.6 # Needed for the mysql_db module, 1,4,6 is the newest version that support python 2 which we really need to stop using networkx==1.11 paramiko==2.4.2 From 18370ab55b950a73335c0643625aeac3a1d171f8 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Thu, 21 Apr 2022 12:33:55 +0500 Subject: [PATCH 295/664] feat: override npm and node version for ecom --- playbooks/roles/ecommerce/defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 2680a854653..78a569c31cd 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -318,6 +318,10 @@ ECOMMERCE_ENABLE_NEWRELIC_DISTRIBUTED_TRACING: false # Python 3 conversion ECOMMERCE_USE_PYTHON38: true +# ECOM Node and Npm Version +ECOMMERCE_NODE_VERSION: '16.14.0' +ECOMMERCE_NPM_VERSION: '8.5.5' + # # OS packages # From 14800b0f8543155253a6ad456454d1a612171e38 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Thu, 21 Apr 2022 12:36:38 +0500 Subject: [PATCH 296/664] fix: add versions in meta file --- playbooks/roles/ecommerce/meta/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index f10b421dbd1..c80ca6ca149 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -36,6 +36,8 @@ dependencies: edx_django_service_language_code: '{{ ECOMMERCE_LANGUAGE_CODE }}' edx_django_service_secret_key: '{{ ECOMMERCE_SECRET_KEY }}' edx_django_service_memcache: '{{ ECOMMERCE_MEMCACHE }}' + edx_django_service_node_version: '{{ ECOMMERCE_NODE_VERSION }}' + edx_django_service_npm_version: '{{ ECOMMERCE_NPM_VERSION }}' edx_django_service_databases: default: ENGINE: 'django.db.backends.mysql' From 709f3068318c153ed1145856af589027f467c764 Mon Sep 17 00:00:00 2001 From: Soban Javed Date: Thu, 21 Apr 2022 18:20:52 +0500 Subject: [PATCH 297/664] build: upgrade node and npm version of insights --- playbooks/roles/insights/defaults/main.yml | 3 ++- playbooks/roles/insights/tasks/main.yml | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/insights/defaults/main.yml b/playbooks/roles/insights/defaults/main.yml index b71423e1c0e..05fa0574588 100644 --- a/playbooks/roles/insights/defaults/main.yml +++ b/playbooks/roles/insights/defaults/main.yml @@ -213,7 +213,8 @@ insights_nodeenv_dir: "{{ insights_home }}/nodeenvs/{{ insights_service_name }}" insights_nodeenv_bin: "{{ insights_nodeenv_dir }}/bin" insights_node_modules_dir: "{{ insights_code_dir }}/node_modules" insights_node_bin: "{{ insights_node_modules_dir }}/.bin" -INSIGHTS_NODE_VERSION: "12.11.1" +INSIGHTS_NODE_VERSION: "16.14.0" +INSIGHTS_NPM_VERSION: "8.5.5" insights_gunicorn_host: "127.0.0.1" insights_gunicorn_port: "8110" diff --git a/playbooks/roles/insights/tasks/main.yml b/playbooks/roles/insights/tasks/main.yml index 85a26133ded..24c18627a54 100644 --- a/playbooks/roles/insights/tasks/main.yml +++ b/playbooks/roles/insights/tasks/main.yml @@ -52,6 +52,14 @@ - install - install:system-requirements +- name: upgrade npm + command: "npm install -g npm@{{ INSIGHTS_NPM_VERSION }}" + become_user: "{{ insights_user }}" + environment: "{{ insights_environment }}" + tags: + - install + - install:system-requirements + # install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: install node dependencies shell: "{{ insights_nodeenv_bin }}/npm install" From e10db621fb20bb3bb2b4d435dfb9915711a2654b Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 22 Apr 2022 11:00:45 +0500 Subject: [PATCH 298/664] chore: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index 22d3e77272e..726e49710b5 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.14.0.7475-1" + version: "11.15.0.7497-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From 6cb5c23231541e03653e37e1bfbd27eb0b41c1c6 Mon Sep 17 00:00:00 2001 From: Zulqarnain Date: Fri, 21 May 2021 15:37:48 +0500 Subject: [PATCH 299/664] refactor!: Python Code Clean-up --- .github/workflows/syntax-test.yml | 2 +- docker/build/flower/flowerconfig.py | 1 - playbooks/active_instances_in_asg.py | 2 - playbooks/callback_plugins/sqs.py | 3 -- playbooks/ec2.py | 31 ++++++------ playbooks/lifecycle_inventory.py | 2 - .../filter_plugins/config_encoders.py | 48 +++++++++---------- .../roles/gh_mirror/files/repos_from_orgs.py | 20 ++++---- .../supervisor/files/pre_supervisor_checks.py | 33 ++++++------- tests/test_mongodb_replica_set.py | 1 - util/ansible_msg.py | 7 ++- util/asg_event_notifications_util.py | 12 ++--- util/aws_ip_locator/ipcollector.py | 6 +-- .../check_for_key_collisions.py | 16 +++---- util/check_rds_configs/check_rds_configs.py | 16 +++---- util/cloudflare/by_origin_purger/purger.py | 7 +-- util/cluster_instance_monitoring.py | 10 ++-- util/elasticsearch/verify-index-copy.py | 7 +-- .../elb_tls_policy_management_util.py | 14 +++--- .../helm_values_to_rst_table_util.py | 4 +- .../add_xqueue_to_dashboard.py | 12 ++--- util/jenkins/check-ses-limits.py | 12 ++--- .../check_table_size/check_table_size.py | 8 ++-- util/jenkins/cloudflare-hit-rate.py | 4 +- .../export_dead_locks/export_dead_locks.py | 4 +- .../export_slow_query_logs.py | 6 +-- .../list_mysql_process/list_mysql_process.py | 10 ++-- .../missing_alerts_checker.py | 22 ++++----- .../primary_keys/check_primary_keys.py | 23 ++++----- .../rds_alarms_checker/missing_rds_alarms.py | 4 +- util/old/import_xml_courses.py | 4 +- util/parsefiles.py | 4 +- util/pingdom/create_pingdom_alerts.py | 31 ++++++------ util/publish_rds_logs_to_cloudwatch.py | 10 ++-- util/rabbitmq/shovel.py | 22 ++++----- util/rds_sgs/rds_sgs.py | 8 ++-- util/s3_acl.py | 16 +++---- util/vpc-tools/asg_lifcycle_watcher.py | 19 ++++---- util/vpc-tools/tag-old-ebs.py | 26 +++++----- util/vpc-tools/vpc_dns.py | 42 ++++++++-------- util/vpc-tools/vpcutil.py | 3 +- 41 files changed, 227 insertions(+), 305 deletions(-) diff --git a/.github/workflows/syntax-test.yml b/.github/workflows/syntax-test.yml index d53dbce9ac8..2d56edb212d 100644 --- a/.github/workflows/syntax-test.yml +++ b/.github/workflows/syntax-test.yml @@ -27,4 +27,4 @@ jobs: pip install -r requirements.txt - name: Run Make test.syntax run: | - timeout 90m make --keep-going test.syntax + timeout 90m make --keep-going test.syntax diff --git a/docker/build/flower/flowerconfig.py b/docker/build/flower/flowerconfig.py index 40422de8f02..077667112ea 100644 --- a/docker/build/flower/flowerconfig.py +++ b/docker/build/flower/flowerconfig.py @@ -1,4 +1,3 @@ - import os address = os.getenv('ADDRESS', "0.0.0.0") diff --git a/playbooks/active_instances_in_asg.py b/playbooks/active_instances_in_asg.py index 14ea5661e50..eeadf711bc6 100755 --- a/playbooks/active_instances_in_asg.py +++ b/playbooks/active_instances_in_asg.py @@ -19,8 +19,6 @@ """ -from __future__ import print_function -from __future__ import absolute_import import argparse import botocore.session import botocore.exceptions diff --git a/playbooks/callback_plugins/sqs.py b/playbooks/callback_plugins/sqs.py index 980d5f1b0d3..d9e7d2038d2 100644 --- a/playbooks/callback_plugins/sqs.py +++ b/playbooks/callback_plugins/sqs.py @@ -16,9 +16,6 @@ # along with Ansible. If not, see . # From https://github.com/ansible/ansible/issues/31527#issuecomment-335495855 -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - import os import sys diff --git a/playbooks/ec2.py b/playbooks/ec2.py index f66adc9a21a..01fb64afbf3 100755 --- a/playbooks/ec2.py +++ b/playbooks/ec2.py @@ -108,8 +108,6 @@ ###################################################################### -from __future__ import absolute_import -from __future__ import print_function import sys import os import argparse @@ -122,7 +120,6 @@ import six.moves.configparser import traceback import six -from six.moves import range try: import json @@ -130,7 +127,7 @@ import simplejson as json -class Ec2Inventory(object): +class Ec2Inventory: def _empty_inventory(self): return {"_meta": {"hostvars": {}}} @@ -238,9 +235,9 @@ def read_settings(self): else: aws_profile = "" - self.cache_path_cache = cache_path + "/{}ansible-ec2.cache".format(aws_profile) - self.cache_path_tags = cache_path + "/{}ansible-ec2.tags.cache".format(aws_profile) - self.cache_path_index = cache_path + "/{}ansible-ec2.index".format(aws_profile) + self.cache_path_cache = cache_path + f"/{aws_profile}ansible-ec2.cache" + self.cache_path_tags = cache_path + f"/{aws_profile}ansible-ec2.tags.cache" + self.cache_path_index = cache_path + f"/{aws_profile}ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age') def parse_cli_args(self): @@ -296,7 +293,7 @@ def get_instances_by_region(self, region): # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported if conn is None: - print(("region name: %s likely not supported, or AWS is down. connection to region failed." % region)) + print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) sys.exit(1) reservations = conn.get_all_instances() @@ -336,7 +333,7 @@ def get_instance(self, region, instance_id): # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported if conn is None: - print(("region name: %s likely not supported, or AWS is down. connection to region failed." % region)) + print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) sys.exit(1) reservations = conn.get_all_instances([instance_id]) @@ -393,7 +390,7 @@ def add_instance(self, instance, region): sys.exit(1) # Inventory: Group by tag keys - for k, v in six.iteritems(instance.tags): + for k, v in instance.tags.items(): key = self.to_safe("tag_" + k + "=" + v) self.push(self.inventory, key, dest) self.keep_first(self.inventory, 'first_in_' + key, dest) @@ -526,16 +523,16 @@ def get_host_info(self): value = getattr(instance, key) key = self.to_safe('ec2_' + key) # Handle complex types - if type(value) in [int, bool]: + if isinstance(value, (int, bool)): instance_vars[key] = value - elif type(value) in [str, six.text_type]: + elif isinstance(value, str): instance_vars[key] = value.strip() elif type(value) == type(None): instance_vars[key] = '' elif key == 'ec2_region': instance_vars[key] = value.name elif key == 'ec2_tags': - for k, v in six.iteritems(value): + for k, v in value.items(): key = self.to_safe('ec2_tag_' + k) instance_vars[key] = v elif key == 'ec2_groups': @@ -573,9 +570,9 @@ def get_inventory_from_cache(self): ''' Reads the inventory from the cache file and returns it as a JSON object ''' if self.args.tags_only: - cache = open(self.cache_path_tags, 'r') + cache = open(self.cache_path_tags) else: - cache = open(self.cache_path_cache, 'r') + cache = open(self.cache_path_cache) json_inventory = cache.read() return json_inventory @@ -583,7 +580,7 @@ def get_inventory_from_cache(self): def load_index_from_cache(self): ''' Reads the index from the cache file sets self.index ''' - cache = open(self.cache_path_index, 'r') + cache = open(self.cache_path_index) json_index = cache.read() self.index = json.loads(json_index) @@ -603,7 +600,7 @@ def to_safe(self, word): ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' - return re.sub("[^A-Za-z0-9\-]", "_", word) + return re.sub(r"[^A-Za-z0-9\-]", "_", word) def json_format_dict(self, data, pretty=False): diff --git a/playbooks/lifecycle_inventory.py b/playbooks/lifecycle_inventory.py index 892ab723988..330f5d8b0e5 100755 --- a/playbooks/lifecycle_inventory.py +++ b/playbooks/lifecycle_inventory.py @@ -28,8 +28,6 @@ } """ -from __future__ import absolute_import -from __future__ import print_function import argparse import boto3 import json diff --git a/playbooks/roles/config-encoders/filter_plugins/config_encoders.py b/playbooks/roles/config-encoders/filter_plugins/config_encoders.py index 91a43f8256b..0980488729c 100644 --- a/playbooks/roles/config-encoders/filter_plugins/config_encoders.py +++ b/playbooks/roles/config-encoders/filter_plugins/config_encoders.py @@ -21,12 +21,10 @@ More information: https://github.com/jtyr/ansible-config_encoder_filters """ -from __future__ import (absolute_import, division, print_function) from ansible import errors from copy import copy import re import six -from six.moves import map def _str_is_bool(data): @@ -169,7 +167,7 @@ def encode_apache( elif block_type == 'options': for o in data: - for key, val in sorted(six.iteritems(o)): + for key, val in sorted(o.items()): rv += "%s%s " % (indent * (level-1), key) rv += encode_apache( val, @@ -197,7 +195,7 @@ def encode_apache( else: rv += str(data) - elif isinstance(data, six.string_types): + elif isinstance(data, str): # Value is a string if ( quote_all_strings or @@ -244,7 +242,7 @@ def encode_erlang( rv += "\n" - for key, val in sorted(six.iteritems(data)): + for key, val in sorted(data.items()): rv += "%s{%s," % (indent*level, key) if not isinstance(val, dict): @@ -268,7 +266,7 @@ def encode_erlang( rv += str(data).lower() - elif isinstance(data, six.string_types): + elif isinstance(data, str): # It's a string atom_len = len(atom_value_indicator) @@ -289,7 +287,7 @@ def encode_erlang( for val in data: if ( - isinstance(val, six.string_types) or + isinstance(val, str) or _is_num(val)): rv += "\n%s" % (indent*level) @@ -360,7 +358,7 @@ def encode_ini( rv = "" # First process all standalone properties - for prop, val in sorted(six.iteritems(data)): + for prop, val in sorted(data.items()): if ucase_prop: prop = prop.upper() @@ -377,7 +375,7 @@ def encode_ini( prop, delimiter, quote, _escape(item, quote), quote) # Then process all sections - for section, props in sorted(six.iteritems(data)): + for section, props in sorted(data.items()): if isinstance(props, dict): if rv != "": rv += "\n" @@ -413,7 +411,7 @@ def encode_json( if len(data) > 0: rv += "\n" - items = sorted(six.iteritems(data)) + items = sorted(data.items()) for key, val in items: rv += '%s"%s": ' % (indent * (level+1), key) @@ -447,7 +445,7 @@ def encode_json( rv += str(data).lower() - elif isinstance(data, six.string_types): + elif isinstance(data, str): # It's a string rv += '"%s"' % _escape(_escape(data), format='control') @@ -497,7 +495,7 @@ def encode_logstash( if prevtype in ('value', 'value_hash', 'array'): rv += "{\n" - items = sorted(six.iteritems(data)) + items = sorted(data.items()) for key, val in items: if key[0] == section_prefix: @@ -513,7 +511,7 @@ def encode_logstash( # Last item of the loop if items[-1] == (key, val): if ( - isinstance(val, six.string_types) or + isinstance(val, str) or _is_num(val) or isinstance(val, bool) or ( isinstance(val, dict) and @@ -540,7 +538,7 @@ def encode_logstash( if ( items[-1] != (key, val) and ( - isinstance(val, six.string_types) or + isinstance(val, str) or _is_num(val) or isinstance(val, bool))): rv += "\n" @@ -560,7 +558,7 @@ def encode_logstash( rv += str(data).lower() - elif isinstance(data, six.string_types): + elif isinstance(data, str): # It's a string rv += '"%s"' % _escape(data) @@ -625,7 +623,7 @@ def encode_nginx(data, indent=" ", level=0, block_semicolon=False): item_type = 'section' - elif isinstance(item, six.string_types): + elif isinstance(item, str): # Normal line if item_type == 'section': rv += "\n" @@ -656,7 +654,7 @@ def encode_pam( # Remember previous type to make newline between type blocks prev_type = None - for label, rule in sorted(six.iteritems(data)): + for label, rule in sorted(data.items()): if separate_types: # Add extra newline to separate blocks of the same type if prev_type is not None and prev_type != rule['type']: @@ -714,9 +712,9 @@ def encode_toml( # It's a dict # First process all standalone strings, numbers, booleans and lists - for key, val in sorted(six.iteritems(data)): + for key, val in sorted(data.items()): if ( - isinstance(val, six.string_types) or + isinstance(val, str) or _is_num(val) or isinstance(val, bool) or ( isinstance(val, list) and @@ -737,7 +735,7 @@ def encode_toml( first = False # Then process all data structures - for key, val in sorted(six.iteritems(data)): + for key, val in sorted(data.items()): if ( isinstance(val, dict) or isinstance(val, list) and isinstance(val[0], dict)): @@ -798,7 +796,7 @@ def encode_toml( if prevtype != 'list': rv += "\n" - elif isinstance(data, six.string_types): + elif isinstance(data, str): # It's a string rv += "%s%s%s" % ( @@ -950,7 +948,7 @@ def encode_yaml( if len(list(data.keys())) == 0: rv += "{}\n" else: - for i, (key, val) in enumerate(sorted(six.iteritems(data))): + for i, (key, val) in enumerate(sorted(data.items())): # Skip indentation only for the first pair rv += "%s%s:" % ("" if i == 0 and skip_indent else level*indent, key) @@ -1044,9 +1042,9 @@ def template_replace(data, replacement): if isinstance(local_data, list): local_data = [template_replace(x, replacement) for x in local_data] elif isinstance(local_data, dict): - for key, val in six.iteritems(local_data): + for key, val in local_data.items(): local_data[key] = template_replace(val, replacement) - elif isinstance(local_data, six.string_types): + elif isinstance(local_data, str): # Replace the special string by it's evaluated value p = re.compile(r'\{\[\{\s*(\w+)([^}\s]+|)\s*\}\]\}') local_data = p.sub(__eval_replace, local_data) @@ -1054,7 +1052,7 @@ def template_replace(data, replacement): return local_data -class FilterModule(object): +class FilterModule: """Ansible encoder Jinja2 filters.""" def filters(self): diff --git a/playbooks/roles/gh_mirror/files/repos_from_orgs.py b/playbooks/roles/gh_mirror/files/repos_from_orgs.py index 35a41cf5dae..1c6a28d6e7b 100644 --- a/playbooks/roles/gh_mirror/files/repos_from_orgs.py +++ b/playbooks/roles/gh_mirror/files/repos_from_orgs.py @@ -6,8 +6,6 @@ # a yaml file containing a list of # github organizations -from __future__ import absolute_import -from __future__ import print_function import yaml import sys import requests @@ -26,12 +24,12 @@ def check_running(run_type=''): fp = open(pid_file, 'w') try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: + except OSError: # another instance is running sys.exit(0) def run_cmd(cmd): - logging.debug('running: {}\n'.format(cmd)) + logging.debug(f'running: {cmd}\n') process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) @@ -52,8 +50,8 @@ def refresh_cache(): try: with open(join(path, 'orgs.yml')) as f: orgs = yaml.safe_load(f) - except IOError: - print("Unable to read {}/orgs.yml, does it exist?".format(path)) + except OSError: + print(f"Unable to read {path}/orgs.yml, does it exist?") sys.exit(1) repos = [] @@ -61,7 +59,7 @@ def refresh_cache(): for org in orgs: page = 1 while True: - r = requests.get('https://api.github.com/users/{}/repos?page={}'.format(org, page)) + r = requests.get(f'https://api.github.com/users/{org}/repos?page={page}') org_data = r.json() # request pages until we get zero results if not isinstance(org_data, list) or len(org_data) == 0: @@ -82,12 +80,12 @@ def update_repos(): for repo in repos: repo_path = os.path.join(args.datadir, repo['org'], repo['name'] + '.git') if not os.path.exists(repo_path): - run_cmd('mkdir -p {}'.format(repo_path)) + run_cmd(f'mkdir -p {repo_path}') run_cmd('git clone --mirror {} {}'.format(repo['html_url'], repo_path)) - run_cmd('cd {} && git update-server-info'.format(repo_path)) + run_cmd(f'cd {repo_path} && git update-server-info') else: - run_cmd('cd {} && git fetch --all --tags'.format(repo_path)) - run_cmd('cd {} && git update-server-info'.format(repo_path)) + run_cmd(f'cd {repo_path} && git fetch --all --tags') + run_cmd(f'cd {repo_path} && git update-server-info') if __name__ == '__main__': args = parse_args() diff --git a/playbooks/roles/supervisor/files/pre_supervisor_checks.py b/playbooks/roles/supervisor/files/pre_supervisor_checks.py index 07bde58b889..52b26215d38 100755 --- a/playbooks/roles/supervisor/files/pre_supervisor_checks.py +++ b/playbooks/roles/supervisor/files/pre_supervisor_checks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import argparse import backoff import boto.ec2 @@ -44,11 +42,10 @@ def services_for_instance(instance_id): try: services = instance.tags['services'].split(',') except KeyError as ke: - msg = "Tag named 'services' not found on this instance({})".format(instance_id) + msg = f"Tag named 'services' not found on this instance({instance_id})" raise Exception(msg) - for service in services: - yield service + yield from services def edp_for_instance(instance_id): ec2 = boto.ec2.connect_to_region(REGION) @@ -61,7 +58,7 @@ def edp_for_instance(instance_id): deployment = instance.tags['deployment'] play = instance.tags['play'] except KeyError as ke: - msg = "{} tag not found on this instance({})".format(ke.message, instance_id) + msg = f"{ke.message} tag not found on this instance({instance_id})" raise Exception(msg) return (environment, deployment, play) @@ -133,7 +130,7 @@ def check_command_output_with_backoff(cmd): instance_id=instance_id) break except Exception as e: - print(("Failed to get EDP for {}: {}".format(instance_id, str(e)))) + print(f"Failed to get EDP for {instance_id}: {str(e)}") # With the time limit being 2 minutes we will # try 5 times before giving up. time.sleep(backoff) @@ -163,14 +160,14 @@ def check_command_output_with_backoff(cmd): "ami-id": ami_id, "created": volume.create_time }) except Exception as e: - msg = "Failed to tag volumes associated with {}: {}".format(instance_id, str(e)) + msg = f"Failed to tag volumes associated with {instance_id}: {str(e)}" print(msg) try: for service in services_for_instance(instance_id): if service in NGINX_ENABLE: subprocess.call(NGINX_ENABLE[service], shell=True) - report.append("Enabling nginx: {}".format(service)) + report.append(f"Enabling nginx: {service}") # We have to reload the new config files subprocess.call("/bin/systemctl reload nginx", shell=True) @@ -203,24 +200,24 @@ def check_command_output_with_backoff(cmd): # Run migration check command. output = check_command_output_with_backoff(cmd) if b'[ ]' in output: - raise Exception("Migrations have not been run for {}".format(service)) + raise Exception(f"Migrations have not been run for {service}") else: - report.append("Checked migrations: {}".format(service)) + report.append(f"Checked migrations: {service}") # Link to available service. - available_file = os.path.join(args.available, "{}.conf".format(service)) - link_location = os.path.join(args.enabled, "{}.conf".format(service)) + available_file = os.path.join(args.available, f"{service}.conf") + link_location = os.path.join(args.enabled, f"{service}.conf") if os.path.exists(available_file): - subprocess.call("sudo -u supervisor ln -sf {} {}".format(available_file, link_location), shell=True) - report.append("Enabling service: {}".format(service)) + subprocess.call(f"sudo -u supervisor ln -sf {available_file} {link_location}", shell=True) + report.append(f"Enabling service: {service}") else: - raise Exception("No conf available for service: {}".format(link_location)) + raise Exception(f"No conf available for service: {link_location}") except AWSConnectionError as ae: - msg = "{}: ERROR : {}".format(prefix, ae) + msg = f"{prefix}: ERROR : {ae}" raise ae except Exception as e: - msg = "{}: ERROR : {}".format(prefix, e) + msg = f"{prefix}: ERROR : {e}" print(msg) traceback.print_exc() raise e diff --git a/tests/test_mongodb_replica_set.py b/tests/test_mongodb_replica_set.py index 9bb940531d6..f7bf3e53ecf 100644 --- a/tests/test_mongodb_replica_set.py +++ b/tests/test_mongodb_replica_set.py @@ -5,7 +5,6 @@ # 2. rename mongodb_replica_set to mongodb_replica_set.py # 3. python test_mongodb_replica_set.py -from __future__ import absolute_import import mongodb_replica_set as mrs import unittest, mock from six.moves.urllib.parse import quote_plus diff --git a/util/ansible_msg.py b/util/ansible_msg.py index 9c3aa38642a..84b4c08d434 100755 --- a/util/ansible_msg.py +++ b/util/ansible_msg.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """Simple utility for deciphering Ansible jsonized task output.""" -from __future__ import print_function import json import sys @@ -36,9 +35,9 @@ if unknown_keys: print("== Unknown keys ======================") for key in unknown_keys: - print("{key}: {val!r:80}".format(key=key, val=data[key])) + print(f"{key}: {data[key]!r:80}") for key in GOOD_KEYS: if data.get(key): - print("== {key} ===========================".format(key=key)) - print((data[key])) + print(f"== {key} ===========================") + print(data[key]) diff --git a/util/asg_event_notifications_util.py b/util/asg_event_notifications_util.py index 86f0aee74b9..9abc7902c72 100644 --- a/util/asg_event_notifications_util.py +++ b/util/asg_event_notifications_util.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 import click @@ -43,10 +41,10 @@ def show_asg_event_notifications(): event_notifications = get_asg_event_notifications(asg) if event_notifications: - print(("Event notifications: {0} are set for ASG: {1}".format(event_notifications, - asg))) + print("Event notifications: {} are set for ASG: {}".format(event_notifications, + asg)) else: - print(("No Event Notifications found for ASG {}".format(asg))) + print(f"No Event Notifications found for ASG {asg}") except Exception as e: print(e) @@ -78,7 +76,7 @@ def create_asg_event_notifications( asg_to_create_event_notifications.append(asg_name) if confirm is False: - print(("Would have created the event notification for asgs {}".format(asg_to_create_event_notifications))) + print(f"Would have created the event notification for asgs {asg_to_create_event_notifications}") else: try: for asg in asg_to_create_event_notifications: @@ -87,7 +85,7 @@ def create_asg_event_notifications( client.put_notification_configuration(AutoScalingGroupName=asg, TopicARN=topic_arn, NotificationTypes=[event]) - print(("Created {0} event notifications for auto scaling group {1}").format(event, + print(("Created {} event notifications for auto scaling group {}").format(event, asg)) except Exception as e: print(e) diff --git a/util/aws_ip_locator/ipcollector.py b/util/aws_ip_locator/ipcollector.py index 31d1f082e8c..a4350aa39a0 100644 --- a/util/aws_ip_locator/ipcollector.py +++ b/util/aws_ip_locator/ipcollector.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 import click import socket @@ -87,11 +85,11 @@ def print_header(name): ============================ {0} ============================""" - print((header.format(name))) + print(header.format(name)) def print_line_item(target, ip): line = "[ * ] {0} - {1}" - print((line.format(target, ip))) + print(line.format(target, ip)) def get_instance_ip_by_name_tag(value): client = boto3.client('ec2') diff --git a/util/check_for_key_collisions/check_for_key_collisions.py b/util/check_for_key_collisions/check_for_key_collisions.py index b06d6705351..16a4f73afc0 100644 --- a/util/check_for_key_collisions/check_for_key_collisions.py +++ b/util/check_for_key_collisions/check_for_key_collisions.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import click import yaml import json @@ -13,25 +11,25 @@ def check_for_yaml_key_collisions(files): for file_path in files: content = None if file_path.endswith(".yml") or file_path.endswith(".yaml"): - stream = open(file_path, 'r') + stream = open(file_path) content = yaml.safe_load(stream) elif file_path.endswith(".json"): - with open(file_path, "r") as read_file: + with open(file_path) as read_file: content = json.load(read_file) - for key, value in six.iteritems(content): + for key, value in content.items(): values_for_keys[key].append(value) collisions = {} - for key,value in six.iteritems(values_for_keys): + for key,value in values_for_keys.items(): if len(value) > 1: collisions[key] = value if len(list(collisions.keys())) > 0: - print((str.format("Found key collisions: {}", len(collisions)))) - for key,value in six.iteritems(collisions): - print((str.format("{} {}", key, value))) + print(str.format("Found key collisions: {}", len(collisions))) + for key,value in collisions.items(): + print(str.format("{} {}", key, value)) exit(1) else: print("No collisions found") diff --git a/util/check_rds_configs/check_rds_configs.py b/util/check_rds_configs/check_rds_configs.py index 4b87cbb7dd5..1f360d96468 100644 --- a/util/check_rds_configs/check_rds_configs.py +++ b/util/check_rds_configs/check_rds_configs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.config import Config import click @@ -151,19 +149,19 @@ def cli(db_engine, ignore): if cluster_with_disabled_snapshot_tags: exit_status = 1 - print(("Slow query logs are disabled for RDS Instances\n{0}".format(slow_query_logs_disabled_rds))) + print(f"Slow query logs are disabled for RDS Instances\n{slow_query_logs_disabled_rds}") print() - print(("Instance parameter groups out of sync/pending reboot for RDS Instances\n{0}".format(instances_out_of_sync_with_instance_parameters))) + print(f"Instance parameter groups out of sync/pending reboot for RDS Instances\n{instances_out_of_sync_with_instance_parameters}") print() - print(("Cluster parameter groups out of sync/pending reboot for RDS Instances\n{0}".format(instances_out_of_sync_with_cluster_parameters))) + print(f"Cluster parameter groups out of sync/pending reboot for RDS Instances\n{instances_out_of_sync_with_cluster_parameters}") print() - print("Sanpshot tags are disabled for Clusters\n{0}".format(cluster_with_disabled_snapshot_tags)) + print(f"Sanpshot tags are disabled for Clusters\n{cluster_with_disabled_snapshot_tags}") print() - print("Performance Insights is disabled for RDS Instances\n{0}".format(instances_with_disabled_performance_insights)) + print(f"Performance Insights is disabled for RDS Instances\n{instances_with_disabled_performance_insights}") print() - print("Tags are missing for the RDS Instances\n{0}".format(instances_without_tags)) + print(f"Tags are missing for the RDS Instances\n{instances_without_tags}") print() - print("Tags are missing for the RDS Clusters\n{0}".format(clusters_without_tags)) + print(f"Tags are missing for the RDS Clusters\n{clusters_without_tags}") print() exit(exit_status) diff --git a/util/cloudflare/by_origin_purger/purger.py b/util/cloudflare/by_origin_purger/purger.py index 916aedf4297..6b5d3a126e5 100644 --- a/util/cloudflare/by_origin_purger/purger.py +++ b/util/cloudflare/by_origin_purger/purger.py @@ -1,8 +1,5 @@ -from __future__ import absolute_import -from __future__ import print_function import requests import click -from six.moves import range @@ -39,7 +36,7 @@ def purge(cloudflare_email, cloudflare_api_key, cloudflare_zone_id, origin, clou chunks = divide_chunks(lines, chunk_size) for chunk in chunks: if not confirm: - print((str.format("Will purge: {} at origin {} and {} others like it. Add --confirm to execute.", chunk[0]['url'], chunk[0]['headers']['Origin'], len(chunk)))) + print(str.format("Will purge: {} at origin {} and {} others like it. Add --confirm to execute.", chunk[0]['url'], chunk[0]['headers']['Origin'], len(chunk))) else: headers = {'X-Auth-Email': cloudflare_email, 'X-Auth-Key': cloudflare_api_key, @@ -49,7 +46,7 @@ def purge(cloudflare_email, cloudflare_api_key, cloudflare_zone_id, origin, clou } url = str.format("https://api.cloudflare.com/client/v4/zones/{cloudflare_zone_id}/purge_cache", cloudflare_zone_id=cloudflare_zone_id) response = requests.delete(url, headers=headers, json=payload) - print((response.json())) + print(response.json()) if __name__ == '__main__': purge() diff --git a/util/cluster_instance_monitoring.py b/util/cluster_instance_monitoring.py index 7eb43bdf305..af38b479ca7 100644 --- a/util/cluster_instance_monitoring.py +++ b/util/cluster_instance_monitoring.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 import argparse import sys @@ -16,7 +14,7 @@ def find_active_instances(cluster_file, region): and cluster to find ASG's and active instances for. """ - with open(cluster_file, 'r') as f: + with open(cluster_file) as f: cluster_map = yaml.safe_load(f) asg = boto3.client('autoscaling', region) @@ -75,9 +73,9 @@ def find_active_instances(cluster_file, region): if no_active_instances_triples: print("Fail. There are no active instances for the following cluster(s)") for triple in no_active_instances_triples: - print(('environment: ' + triple.split('-')[0])) - print(('deployment: ' + triple.split('-')[1])) - print(('cluster: ' + triple.split('-')[2])) + print('environment: ' + triple.split('-')[0]) + print('deployment: ' + triple.split('-')[1]) + print('cluster: ' + triple.split('-')[2]) print('----') sys.exit(1) diff --git a/util/elasticsearch/verify-index-copy.py b/util/elasticsearch/verify-index-copy.py index 5f1cfecc1ca..74aeffe5684 100755 --- a/util/elasticsearch/verify-index-copy.py +++ b/util/elasticsearch/verify-index-copy.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- """ Verifies that an index was correctly copied from one ES host to another. """ -from __future__ import absolute_import -from __future__ import print_function import itertools import pprint import random @@ -200,7 +197,7 @@ def scan_documents(old_es, new_es, old_index, new_index): matching += find_matching_ids(new_es, new_index, old_elt_ids, old_elt_docs) total += len(old_elt_ids) if total % 100 == 0: - print('processed {} items'.format(total)) + print(f'processed {total} items') ratio = float(matching)/total print("{}: scanned documents matching ({} out of {}, {:.6}%)".format( @@ -251,7 +248,7 @@ def random_checks(old_es, new_es, old_index, new_index, total_document_count, ch current_offset += num_elts if total % 100 == 0: - print('processed {} items'.format(total)) + print(f'processed {total} items') ratio = float(matching) / total print("{}: random documents matching ({} out of {}, {}%)".format( diff --git a/util/elb_tls_policy_management_util/elb_tls_policy_management_util.py b/util/elb_tls_policy_management_util/elb_tls_policy_management_util.py index 1c6b7d105f6..0abd69d7b26 100644 --- a/util/elb_tls_policy_management_util/elb_tls_policy_management_util.py +++ b/util/elb_tls_policy_management_util/elb_tls_policy_management_util.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 import click import datetime @@ -41,7 +39,7 @@ def check_valid_policy(ctx, param, value): list_of_valid_policy_names = get_tls_security_policy_template_names() if value not in list_of_valid_policy_names: raise click.BadParameter("""Could not find the specified policy version, - found versions: {0}""" + found versions: {}""" .format(list_of_valid_policy_names)) return value @@ -64,12 +62,12 @@ def get_elb_names(): def print_header(header): print("\n\n----------------------------------------------") - print(("[ ] {0}".format(header))) + print(f"[ ] {header}") print("----------------------------------------------") def print_line_item(line_item): - print(("[ * ] {0}".format(line_item))) + print(f"[ * ] {line_item}") def print_list(name, items_list): @@ -88,7 +86,7 @@ def create_tls_policy(elb_name, policy_version_to_copy): }) milli_datetime = str(int(datetime.datetime.now().strftime("%s")) * 1000) print('Creating new policy for elb....') - new_policy_name = "SSLUpdateScript-SSLNegotiationPolicy-{0}-{1}".format( + new_policy_name = "SSLUpdateScript-SSLNegotiationPolicy-{}-{}".format( elb_name, milli_datetime) response = client.create_load_balancer_policy( LoadBalancerName=elb_name, @@ -200,7 +198,7 @@ def update_elb_policies(confirm, policy_version, names, port_override): else: for elb_name in elb_names_to_update: tls_policy_name = create_tls_policy(elb_name, policy_version) - print(("Trying to update...{0}".format(elb_name))) + print(f"Trying to update...{elb_name}") client = get_client() # Determine which policies are actually active @@ -268,7 +266,7 @@ def update_elb_policies(confirm, policy_version, names, port_override): LoadBalancerPort=tls_port, PolicyNames=policy_names ) - print(("Updated {0}\n".format(elb_name))) + print(f"Updated {elb_name}\n") cli.add_command(show_available_policy_versions) cli.add_command(show_elb_policy_versions) diff --git a/util/helm_values_to_rst_table_util/helm_values_to_rst_table_util.py b/util/helm_values_to_rst_table_util/helm_values_to_rst_table_util.py index a61d094b6c5..5e9f167c7b9 100644 --- a/util/helm_values_to_rst_table_util/helm_values_to_rst_table_util.py +++ b/util/helm_values_to_rst_table_util/helm_values_to_rst_table_util.py @@ -5,7 +5,7 @@ @click.option('--values', help='Path to a values.yaml file', required=True) @click.option('--subcharts', help='Sub chart values to ignore', multiple=True) def cli(values, subcharts): - with open(values, 'r') as stream: + with open(values) as stream: parsed_dict = yaml.safe_load(stream) keys_from_yaml = collect_keys_from_yaml(parsed_dict, subcharts) col_width = 99 @@ -62,7 +62,7 @@ def get_keys(prefix, inp): for child_key in inp.keys(): child = inp[child_key] - if prefix is not "": + if prefix != "": modified_prefix = prefix + "." else: modified_prefix = prefix diff --git a/util/jenkins/add_new_xqueues_to_dashboard/add_xqueue_to_dashboard.py b/util/jenkins/add_new_xqueues_to_dashboard/add_xqueue_to_dashboard.py index dddbfcc41a2..fef21151bbf 100644 --- a/util/jenkins/add_new_xqueues_to_dashboard/add_xqueue_to_dashboard.py +++ b/util/jenkins/add_new_xqueues_to_dashboard/add_xqueue_to_dashboard.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import pprint import re @@ -11,7 +9,7 @@ MAX_TRIES = 1 -class CwBotoWrapper(object): +class CwBotoWrapper: def __init__(self): self.client = boto3.client('cloudwatch') @@ -83,7 +81,7 @@ def generate_dashboard_widget( return {'type': 'metric', 'height': height, 'width': width, 'x': x, 'y': y, 'properties': { 'period': period, 'view': 'timeSeries', 'stacked': stacked, 'region': region, - 'title': "{} (auto-generated)".format(title), + 'title': f"{title} (auto-generated)", 'metrics': generate_dashboard_widget_metrics(cloudwatch, namespace, metric_name, dimension_name, metrics_properties, right_axis_items=right_axis_items) } @@ -98,8 +96,8 @@ def generate_dashboard(environment, deploy): pp = pprint.PrettyPrinter(indent=4) cloudwatch = CwBotoWrapper() - dashboard_name = "{}-{}-xqueues".format(environment, deploy) - xqueue_namespace = "xqueue/{}-{}".format(environment, deploy) + dashboard_name = f"{environment}-{deploy}-xqueues" + xqueue_namespace = f"xqueue/{environment}-{deploy}" widgets = [] y_cord = 0 @@ -110,7 +108,7 @@ def generate_dashboard(environment, deploy): height = 9 widgets.append(generate_dashboard_widget(cloudwatch, y=y_cord, height=height, - title="{}-{} Xqueue Queues".format(environment, deploy), + title=f"{environment}-{deploy} Xqueue Queues", namespace=xqueue_namespace, metric_name="queue_length", dimension_name="queue", ) diff --git a/util/jenkins/check-ses-limits.py b/util/jenkins/check-ses-limits.py index 838c63afbe2..5bd0472f6d8 100755 --- a/util/jenkins/check-ses-limits.py +++ b/util/jenkins/check-ses-limits.py @@ -2,8 +2,6 @@ # This script is used by the monioring/check-seslimits Jenkins job -from __future__ import absolute_import -from __future__ import print_function import boto3 import argparse import sys @@ -29,9 +27,9 @@ def __call__(self, parser, namespace, values, option_string=None): args = parser.parse_args() if args.warning and args.warning >= args.critical: - warn_str = "Warning threshold ({})".format(args.warning) - crit_str = "Critical threshold ({})".format(args.critical) - print(("ERROR: {} >= {}".format(warn_str, crit_str))) + warn_str = f"Warning threshold ({args.warning})" + crit_str = f"Critical threshold ({args.critical})" + print(f"ERROR: {warn_str} >= {crit_str}") sys.exit(1) exit_code = 0 @@ -51,8 +49,8 @@ def __call__(self, parser, namespace, values, option_string=None): level = "WARNING" if level: - print(("{} {}/{} ({}%) - {}".format(region, current, limit, percent, - level))) + print("{} {}/{} ({}%) - {}".format(region, current, limit, percent, + level)) exit_code += 1 sys.exit(exit_code) diff --git a/util/jenkins/check_table_size/check_table_size.py b/util/jenkins/check_table_size/check_table_size.py index 48d1c1c5142..bf047da354f 100644 --- a/util/jenkins/check_table_size/check_table_size.py +++ b/util/jenkins/check_table_size/check_table_size.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -45,7 +43,7 @@ def rds_extractor(): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: client = RDSBotoWrapper(region_name=region["RegionName"]) @@ -142,9 +140,9 @@ def controller(username, password, threshold, rdsthreshold, rdsignore): table_list = check_table_growth(filtered_rds_list, username, password, threshold, rds_threshold) if len(table_list) > 0: format_string = "{:<40}{:<20}{:<50}{}" - print((format_string.format("RDS Name","Database Name", "Table Name", "Size"))) + print(format_string.format("RDS Name","Database Name", "Table Name", "Size")) for items in table_list: - print((format_string.format(items["rds"], items["db"], items["table"], str(items["size"]) + " MB"))) + print(format_string.format(items["rds"], items["db"], items["table"], str(items["size"]) + " MB")) exit(1) exit(0) diff --git a/util/jenkins/cloudflare-hit-rate.py b/util/jenkins/cloudflare-hit-rate.py index 4bb2c68dd30..203649cdcff 100644 --- a/util/jenkins/cloudflare-hit-rate.py +++ b/util/jenkins/cloudflare-hit-rate.py @@ -3,8 +3,6 @@ https://api.cloudflare.com/#zone-analytics-dashboard """ -from __future__ import absolute_import -from __future__ import print_function import requests import argparse import sys @@ -32,7 +30,7 @@ def calcualte_cache_hit_rate(zone_id, auth_key, email, threshold): sys.exit(1) except Exception as error: - print(("JSON Error: {} \n Content returned from API call: {}".format(error, res.text))) + print(f"JSON Error: {error} \n Content returned from API call: {res.text}") diff --git a/util/jenkins/export_dead_locks/export_dead_locks.py b/util/jenkins/export_dead_locks/export_dead_locks.py index bd5de355d3d..a2d048c5899 100644 --- a/util/jenkins/export_dead_locks/export_dead_locks.py +++ b/util/jenkins/export_dead_locks/export_dead_locks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -49,7 +47,7 @@ def rds_extractor(environment): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: rds_client = RDSBotoWrapper(region_name=region["RegionName"]) diff --git a/util/jenkins/export_slow_logs/export_slow_query_logs.py b/util/jenkins/export_slow_logs/export_slow_query_logs.py index e91b8e3726b..fa26b72b86a 100644 --- a/util/jenkins/export_slow_logs/export_slow_query_logs.py +++ b/util/jenkins/export_slow_logs/export_slow_query_logs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -64,7 +62,7 @@ def rds_extractor(environment): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: client = RDSBotoWrapper(region_name=region["RegionName"]) @@ -89,7 +87,7 @@ def rds_controller(rds_list, username, password): connection = pymysql.connect(host=rds_host_endpoint, port=rds_port, user=username, password=password) cursor = connection.cursor() - print("Checking slow log on RDS HOST {0}".format(rds_host_endpoint)) + print(f"Checking slow log on RDS HOST {rds_host_endpoint}") cursor.execute(""" SELECT * FROM mysql.slow_log diff --git a/util/jenkins/list_mysql_process/list_mysql_process.py b/util/jenkins/list_mysql_process/list_mysql_process.py index 9b0e780558e..f6293ef2199 100644 --- a/util/jenkins/list_mysql_process/list_mysql_process.py +++ b/util/jenkins/list_mysql_process/list_mysql_process.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -45,7 +43,7 @@ def rds_extractor(environment): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: client = RDSBotoWrapper(region_name=region["RegionName"]) @@ -132,10 +130,10 @@ def controller(username, password, environment, rdsignore): process_list = check_queries_running(filtered_rds_list, username, password) if len(process_list) > 0: format_string = "{:<20}{:<20}{:<30}{:<20}{:<20}{:<70}{}" - print((format_string.format("Query ID", "User Name", "Host", "Command", "Time Executed", "State", "Info"))) + print(format_string.format("Query ID", "User Name", "Host", "Command", "Time Executed", "State", "Info")) for items in process_list: - print((format_string.format(items["id"], items["user"], items["host"], items["command"], - str(items["time"]) + " sec", items["state"], items["info"]))) + print(format_string.format(items["id"], items["user"], items["host"], items["command"], + str(items["time"]) + " sec", items["state"], items["info"])) exit(0) diff --git a/util/jenkins/missing_alerts_checker/missing_alerts_checker.py b/util/jenkins/missing_alerts_checker/missing_alerts_checker.py index dec759040d2..ec283bb26f2 100644 --- a/util/jenkins/missing_alerts_checker/missing_alerts_checker.py +++ b/util/jenkins/missing_alerts_checker/missing_alerts_checker.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 import requests import click @@ -34,7 +32,7 @@ def new_relic_policies_extractor(self): try: alert_policies = response.json() except ValueError: - print(("Failed to parse response json. Got:\n{}".format(response.text))) + print(f"Failed to parse response json. Got:\n{response.text}") sys.exit(1) return alert_policies @@ -74,7 +72,7 @@ def edc_extractor(self): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list['Regions']: client = boto3.resource('ec2', region_name=region['RegionName']) @@ -134,7 +132,7 @@ def new_relic_app_extractor(self): try: apps_list = response.json() except ValueError: - print(("Failed to parse response json. Got:\n{}".format(response.text))) + print(f"Failed to parse response json. Got:\n{response.text}") sys.exit(1) return apps_list["applications"] @@ -180,7 +178,7 @@ def new_relic_browser_extractor(self): try: browser_list = response.json() except ValueError: - raise Exception("Failed to parse response json. Got:\n{}".format(response.text)) + raise Exception(f"Failed to parse response json. Got:\n{response.text}") return browser_list["browser_applications"] def missing_alerts_checker(self, browser_list, alert_policies): @@ -230,9 +228,9 @@ def controller(new_relic_api_key,ignore): missing_alerts_list = infracheck.missing_alerts_checker(instance_list, alert_policies) filtered_missing_alerts_list = list([x for x in missing_alerts_list if not any(re.search(r, x['name']) for r in ignore)]) format_string = "{:<30}{}" - print((format_string.format("Instance ID", "Instance Name"))) + print(format_string.format("Instance ID", "Instance Name")) for instance_wo_alerts in filtered_missing_alerts_list: - print((format_string.format(instance_wo_alerts["ID"], instance_wo_alerts["name"]))) + print(format_string.format(instance_wo_alerts["ID"], instance_wo_alerts["name"])) flag = 1 # Initializing object of classes @@ -245,9 +243,9 @@ def controller(new_relic_api_key,ignore): filtered_missing_alerts_list_app = list([x for x in missing_alerts_list_app if not any(re.search(r, x['name']) for r in ignore)]) format_string = "{:<20}{}" print("") - print((format_string.format("Application ID", "Application Name"))) + print(format_string.format("Application ID", "Application Name")) for instance_wo_alerts in filtered_missing_alerts_list_app: - print((format_string.format(instance_wo_alerts["id"], instance_wo_alerts["name"]))) + print(format_string.format(instance_wo_alerts["id"], instance_wo_alerts["name"])) flag = 1 # Initializing object of classes @@ -260,9 +258,9 @@ def controller(new_relic_api_key,ignore): filtered_missing_alerts_list_browser = list([x for x in missing_alerts_list_browser if not any(re.search(r, x['name']) for r in ignore)]) format_string = "{:<20}{}" print("") - print((format_string.format("Browser ID", "Browser Name"))) + print(format_string.format("Browser ID", "Browser Name")) for instance_wo_alerts in filtered_missing_alerts_list_browser: - print((format_string.format(instance_wo_alerts["id"], instance_wo_alerts["name"]))) + print(format_string.format(instance_wo_alerts["id"], instance_wo_alerts["name"])) flag = 1 sys.exit(flag) diff --git a/util/jenkins/primary_keys/check_primary_keys.py b/util/jenkins/primary_keys/check_primary_keys.py index 4a211a1fe43..e7f6673038e 100644 --- a/util/jenkins/primary_keys/check_primary_keys.py +++ b/util/jenkins/primary_keys/check_primary_keys.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -7,7 +5,6 @@ import pymysql import click from datetime import datetime, timedelta, timezone -from six.moves import range MAX_TRIES = 5 PERIOD = 360 @@ -97,7 +94,7 @@ def send_an_email(to_addr, from_addr, primary_keys_message, region): ) message += """""" - print(("Sending the following as email to {}".format(to_addr))) + print(f"Sending the following as email to {to_addr}") print(message) ses_client.send_email( Source=from_addr, @@ -143,10 +140,10 @@ def get_rds_from_all_regions(): try: regions_list = ec2_client.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: - print(("Getting RDS instances in region {}".format(region["RegionName"]))) + print("Getting RDS instances in region {}".format(region["RegionName"])) rds_client = RDSBotoWrapper(region_name=region["RegionName"]) response = rds_client.describe_db_instances() for instance in response.get('DBInstances'): @@ -178,13 +175,13 @@ def check_primary_keys(rds_list, username, password, environment, deploy): """ cloudwatch = CwBotoWrapper() metric_name = 'used_key_space' - namespace = "rds-primary-keys/{}-{}".format(environment, deploy) + namespace = f"rds-primary-keys/{environment}-{deploy}" try: table_list = [] metric_data = [] tables_reaching_exhaustion_limit = [] for rds_instance in rds_list: - print(("Checking rds instance {}".format(rds_instance["name"]))) + print("Checking rds instance {}".format(rds_instance["name"])) rds_host_endpoint = rds_instance["Endpoint"] rds_port = rds_instance["Port"] connection = pymysql.connect(host=rds_host_endpoint, @@ -242,11 +239,11 @@ def check_primary_keys(rds_list, username, password, environment, deploy): table_data = {} db_name = result_table[0] table_name = result_table[1] - table_name_combined = "{}.{}".format(db_name, table_name) + table_name_combined = f"{db_name}.{table_name}" table_percent = result_table[6] if table_percent > 70: - print(("RDS {} Table {}: Primary keys {}% full".format( - rds_instance["name"], table_name_combined, table_percent))) + print("RDS {} Table {}: Primary keys {}% full".format( + rds_instance["name"], table_name_combined, table_percent)) metric_data.append({ 'MetricName': metric_name, 'Dimensions': [{ @@ -319,9 +316,9 @@ def get_metrics_and_calcuate_diff(namespace, metric_name, dimension, value, curr no_of_days = time_diff.days increase_over_time_period = current_usage/no_of_days days_remaining_before_exhaustion = consumed_keys_percentage/increase_over_time_period - print(("Days remaining for {table} table on db {db}: {days}".format(table=value, + print("Days remaining for {table} table on db {db}: {days}".format(table=value, db=dimension, - days=days_remaining_before_exhaustion))) + days=days_remaining_before_exhaustion)) return days_remaining_before_exhaustion diff --git a/util/jenkins/rds_alarms_checker/missing_rds_alarms.py b/util/jenkins/rds_alarms_checker/missing_rds_alarms.py index f34aa603314..e7518d6c22d 100644 --- a/util/jenkins/rds_alarms_checker/missing_rds_alarms.py +++ b/util/jenkins/rds_alarms_checker/missing_rds_alarms.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import boto3 from botocore.exceptions import ClientError import sys @@ -53,7 +51,7 @@ def rds_extractor(): try: regions_list = client_region.describe_regions() except ClientError as e: - print(("Unable to connect to AWS with error :{}".format(e))) + print(f"Unable to connect to AWS with error :{e}") sys.exit(1) for region in regions_list["Regions"]: client = RDSBotoWrapper(region_name=region["RegionName"]) diff --git a/util/old/import_xml_courses.py b/util/old/import_xml_courses.py index 20436b40ffc..7a7cdd66779 100644 --- a/util/old/import_xml_courses.py +++ b/util/old/import_xml_courses.py @@ -13,8 +13,6 @@ # version: #} -from __future__ import absolute_import -from __future__ import print_function import argparse from os.path import basename import yaml @@ -26,7 +24,7 @@ help="A CSV of xml courses to import.") args = parser.parse_args() - courses = open(args.courses_csv, 'r') + courses = open(args.courses_csv) all_course_data = [] all_xml_mappings = {} diff --git a/util/parsefiles.py b/util/parsefiles.py index 9c088bef06f..de1184d7c11 100644 --- a/util/parsefiles.py +++ b/util/parsefiles.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import -from __future__ import print_function import os import pathlib2 import logging @@ -336,7 +334,7 @@ def _get_role_name(role): """ if isinstance(role, dict): return role['role'] - elif isinstance(role, six.string_types): + elif isinstance(role, str): return role else: LOGGER.warning("role %s could not be resolved to a role name." % role) diff --git a/util/pingdom/create_pingdom_alerts.py b/util/pingdom/create_pingdom_alerts.py index cb122a27908..0140a79cfbd 100644 --- a/util/pingdom/create_pingdom_alerts.py +++ b/util/pingdom/create_pingdom_alerts.py @@ -1,12 +1,9 @@ -from __future__ import absolute_import -from __future__ import print_function import json import click import yaml import requests import json -from six.moves import map class PingdomInvalidResponse(Exception): @@ -30,7 +27,7 @@ class PingdomInvalidResponse(Exception): def main(noop, pingdom_email, pingdom_password, pingdom_api_key, alert_config_file): - with open(alert_config_file, 'r') as stream: + with open(alert_config_file) as stream: config_file_content = yaml.safe_load(stream) config_file_content = replace_user_names_with_userids(pingdom_email, pingdom_password, @@ -45,10 +42,10 @@ def main(noop, pingdom_email, pingdom_password, if (alert_config['name'], alert_config['host']) not in checks_by_hostname.items(): # Create new check if noop: - print(("Would CREATE: {0}, but you set the noop flag.".format( - alert_config))) + print("Would CREATE: {}, but you set the noop flag.".format( + alert_config)) else: - print(("CREATE: {0}".format(alert_config))) + print(f"CREATE: {alert_config}") create_check(pingdom_email, pingdom_password, pingdom_api_key, alert_config) @@ -56,12 +53,12 @@ def main(noop, pingdom_email, pingdom_password, # Updating existing check existing_check = check_for_update[alert_config['name']] if noop: - print((""" - Has changes, would UPDATE: {0}, + print(""" + Has changes, would UPDATE: {}, but you set the noop flag. - """.format(alert_config))) + """.format(alert_config)) else: - print(("Attempting UPDATE: {0}".format(alert_config))) + print(f"Attempting UPDATE: {alert_config}") # We always update because the parameters to POST check # and the paramters returned by GET check differ. # It would be difficult to figure out if changes @@ -87,7 +84,7 @@ def replace_user_names_with_userids(pingdom_email, [x.strip() for x in alert['userids'].split(',')]) if user not in user_ids_by_name: raise PingdomInvalidResponse( - 'Pingdom has no user with the name {0}'.format(user)) + f'Pingdom has no user with the name {user}') user_id = user_ids_by_name[user] user_ids.append(user_id) del alert['users'] @@ -147,7 +144,7 @@ def update_check(pingdom_email, pingdom_password, if('type' in payload): del(payload['type']) try: - url = "https://api.pingdom.com/api/2.1/checks/{0}".format(id) + url = f"https://api.pingdom.com/api/2.1/checks/{id}" response = requests.put(url, headers={ 'app-key': pingdom_api_key @@ -216,15 +213,15 @@ def build_userid_by_name(pingdom_email, pingdom_password, pingdom_api_key): def print_request_and_response(response): print("Request:") for key in response.request.headers: - print(("{0}: {1}".format(key, response.request.headers[key]))) + print(f"{key}: {response.request.headers[key]}") print("") - print((response.request.body)) + print(response.request.body) print("------------------") print("Response:") for key in response.headers: - print(("{0}: {1}".format(key, response.headers[key]))) + print(f"{key}: {response.headers[key]}") print("") - print((response.content.decode('utf-8'))) + print(response.content.decode('utf-8')) print("------------------") diff --git a/util/publish_rds_logs_to_cloudwatch.py b/util/publish_rds_logs_to_cloudwatch.py index 1c1d1923275..a36972c54f1 100755 --- a/util/publish_rds_logs_to_cloudwatch.py +++ b/util/publish_rds_logs_to_cloudwatch.py @@ -6,8 +6,6 @@ ./publish_rds_logs_to_cloudwatch --db_engine aurora --db_identifier edx-aurora-cluster """ -from __future__ import absolute_import -from __future__ import print_function import boto3 import argparse @@ -32,8 +30,8 @@ def publish_rds_logs_to_cloudwatch(db_engine,db_identifier,logs_to_publish): if response["ResponseMetadata"]["HTTPStatusCode"] == 200: id=response["DBInstance"]["DBInstanceIdentifier"] logs_exports_to_cloudwatch=response["DBInstance"]["EnabledCloudwatchLogsExports"] - print(("RDS MySQL DB {} logs {} are enabled to exports to cloudwatch" \ - .format(id,logs_exports_to_cloudwatch))) + print("RDS MySQL DB {} logs {} are enabled to exports to cloudwatch" \ + .format(id,logs_exports_to_cloudwatch)) elif db_engine == "aurora": response = client.modify_db_cluster( DBClusterIdentifier=db_identifier, @@ -46,8 +44,8 @@ def publish_rds_logs_to_cloudwatch(db_engine,db_identifier,logs_to_publish): if response["ResponseMetadata"]["HTTPStatusCode"] == 200: id=response["DBCluster"]["DBClusterIdentifier"] logs_exports_to_cloudwatch=response["DBCluster"]["EnabledCloudwatchLogsExports"] - print(("RDS Aurora Cluster {} logs {} are enabled to exports to cloudwatch" \ - .format(id,logs_exports_to_cloudwatch))) + print("RDS Aurora Cluster {} logs {} are enabled to exports to cloudwatch" \ + .format(id,logs_exports_to_cloudwatch)) else: print("db_engine valid options are: mysql or aurora") exit() diff --git a/util/rabbitmq/shovel.py b/util/rabbitmq/shovel.py index dfddd99d7ba..0d1538dfd7a 100644 --- a/util/rabbitmq/shovel.py +++ b/util/rabbitmq/shovel.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -from __future__ import absolute_import -from __future__ import print_function import argparse import subprocess import requests @@ -18,35 +16,35 @@ args=parser.parse_args() -src_uri='amqp://{}:{}@{}'.format(args.src_user,args.src_user_pass,args.src_host) -dest_uri='amqp://{}:{}@{}'.format(args.dest_user,args.dest_user_pass,args.dest_host) +src_uri=f'amqp://{args.src_user}:{args.src_user_pass}@{args.src_host}' +dest_uri=f'amqp://{args.dest_user}:{args.dest_user_pass}@{args.dest_host}' port=15672 def list_vhosts(): - url='http://{}:{}/api/vhosts'.format(args.src_host,port) + url=f'http://{args.src_host}:{port}/api/vhosts' try: response=requests.get(url,auth=(args.src_user,args.src_user_pass)) response.raise_for_status() vhosts=[v['name'] for v in response.json() if v['name'].startswith('/')] except Exception as ex: - print("Failed to get vhosts: {}".format(ex)) + print(f"Failed to get vhosts: {ex}") sys.exit(1) return vhosts def list_queues(): for vhost in list_vhosts(): - url='http://{}:{}/api/queues/{}'.format(args.src_host,port,vhost) + url=f'http://{args.src_host}:{port}/api/queues/{vhost}' try: response=requests.get(url,auth=(args.src_user,args.src_user_pass)) response.raise_for_status() queues=[q['name'] for q in response.json()] except Exception as ex: - print("Failed to get queues: {}".format(ex)) + print(f"Failed to get queues: {ex}") sys.exit(1) return queues def create_shovel(shovel,arg): - cmd="/usr/sbin/rabbitmqctl set_parameter shovel {} '{}'".format(shovel,arg) + cmd=f"/usr/sbin/rabbitmqctl set_parameter shovel {shovel} '{arg}'" try: subprocess.check_output( cmd,stderr=subprocess.STDOUT,shell=True) @@ -67,11 +65,11 @@ def create_shovel(shovel,arg): """ q=queue.split('.') if (q[0]!='celeryev' and q[-1]!='pidbox'): - args='{{"src-uri": "{}", "src-queue": "{}","dest-uri": "{}","dest-queue": "{}"}}'.format(src_uri,queue,dest_uri,queue) - print("Running shovel for queue:{}".format(queue)) + args=f'{{"src-uri": "{src_uri}", "src-queue": "{queue}","dest-uri": "{dest_uri}","dest-queue": "{queue}"}}' + print(f"Running shovel for queue:{queue}") shovel_output=create_shovel(queue,args) if shovel_output is not None: - content=six.text_type(shovel_output,"utf-8") + content=str(shovel_output,"utf-8") output[queue]=content for k,v in output.items(): print(k,v) diff --git a/util/rds_sgs/rds_sgs.py b/util/rds_sgs/rds_sgs.py index 0d718f14afa..0ae32d944fb 100755 --- a/util/rds_sgs/rds_sgs.py +++ b/util/rds_sgs/rds_sgs.py @@ -1,7 +1,5 @@ #!/usr/bin/python3 -from __future__ import absolute_import -from __future__ import print_function import boto3 import click @@ -71,12 +69,12 @@ def command(mode): for sg_id in sorted(sources[source].keys()): output = sg_id if sources[source][sg_id]: - output = "{} ({})".format(output, sources[source][sg_id]) + output = f"{output} ({sources[source][sg_id]})" sgs.append(output) - print(("{: <40} {: <11} {: <70} {}".format(db['DBInstanceIdentifier'], ports, source, ", ".join(sgs)))) + print("{: <40} {: <11} {: <70} {}".format(db['DBInstanceIdentifier'], ports, source, ", ".join(sgs))) if mode == 'by_sg': for sg,dbs in dbs_by_sg.items(): - print(("{: <70} {: <4} {}".format(sg, len(dbs), ", ".join(dbs)))) + print("{: <70} {: <4} {}".format(sg, len(dbs), ", ".join(dbs))) if __name__ == '__main__': command() diff --git a/util/s3_acl.py b/util/s3_acl.py index 6d38040517e..581338515e8 100644 --- a/util/s3_acl.py +++ b/util/s3_acl.py @@ -100,7 +100,7 @@ def set_acl_private(acl_list, bucket_name, exclude): if e.response['Error']['Code'] == 'NoSuchKey': logger.warning("No such key in S3: " + key) # Will send the errors to the file else: - logger.error(("Unexpected error :{}".format(e))) + logger.error(f"Unexpected error :{e}") sys.exit(1) @@ -122,7 +122,7 @@ def revert_s3_acl(acl_list, bucket_name, exclude): if e.response['Error']['Code'] == 'NoSuchKey': logger.warning("No such key in S3: " + key) # Will send the errors to the file else: - logger.error(("Unexpected error :{}".format(e))) + logger.error(f"Unexpected error :{e}") sys.exit(1) @@ -132,7 +132,7 @@ def get_s3_acl(s3_bucket, exclude): try: s3_objects_key = get_all_s3_keys(s3_bucket, region, exclude) except ClientError as e: - logger.error(("Unable to connect to AWS with error :{}".format(e))) + logger.error(f"Unable to connect to AWS with error :{e}") sys.exit(1) for object_key in s3_objects_key: try: @@ -146,7 +146,7 @@ def get_s3_acl(s3_bucket, exclude): elif e.response['Error']['Code'] == 'NoSuchKey': logger.warning("No such key in S3: " + object_key) # Will send the errors to the file else: - logger.error(("Unexpected error :{}".format(e))) + logger.error(f"Unexpected error :{e}") sys.exit(1) return response_list @@ -165,21 +165,21 @@ def controller(bucketname, operation, exclude): elif operation == 'setaclprivate': try: data = [] - with open(file_to_write, "r") as inFile: + with open(file_to_write) as inFile: data = json.load(inFile) set_acl_private(data, bucketname, exclude) logger.info("Task completed. ACL of " + bucketname + " objects set to private.") - except IOError: + except OSError: logger.error("File not accessible") sys.exit(1) elif operation == 'revertacl': try: data = [] - with open(file_to_write, "r") as inFile: + with open(file_to_write) as inFile: data = json.load(inFile) revert_s3_acl(data, bucketname, exclude) logger.info("Task completed. ACL of " + bucketname + " objects reverted to given state") - except IOError: + except OSError: logger.error("File not accessible") sys.exit(1) else: diff --git a/util/vpc-tools/asg_lifcycle_watcher.py b/util/vpc-tools/asg_lifcycle_watcher.py index 15f1ab8d512..dc4e72ab419 100644 --- a/util/vpc-tools/asg_lifcycle_watcher.py +++ b/util/vpc-tools/asg_lifcycle_watcher.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import __author__ = 'e0d' """ @@ -51,7 +50,7 @@ def __init__(self, region, queue, hook, dry_run, bin_directory=None): aws_bin=self.aws_bin) if self.region: - self.base_cli_command += "--region {region} ".format(region=self.region) + self.base_cli_command += f"--region {self.region} " self.dry_run = args.dry_run self.ec2_con = boto3.client('ec2',region_name=self.region) @@ -65,7 +64,7 @@ def process_lifecycle_messages(self): WaitTimeSeconds=LifecycleHandler.WAIT_TIME_SECONDS).get('Messages', []): body = json.loads(sqs_message['Body']) as_message = json.loads(body['Message']) - logging.info("Proccessing message {message}.".format(message=as_message)) + logging.info(f"Proccessing message {as_message}.") if 'LifecycleTransition' in as_message and as_message['LifecycleTransition'] \ == LifecycleHandler.INSTANCE_TERMINATION: @@ -107,10 +106,10 @@ def process_lifecycle_messages(self): def delete_sqs_message(self, queue, sqs_message, as_message, dry_run): if not dry_run: - logging.info("Deleting message with body {message}".format(message=as_message)) + logging.info(f"Deleting message with body {as_message}") self.sqs_con.delete_message(QueueUrl=queue.url, ReceiptHandle=sqs_message['ReceiptHandle']) else: - logging.info("Would have deleted message with body {message}".format(message=as_message)) + logging.info(f"Would have deleted message with body {as_message}") def record_lifecycle_action_heartbeat(self, asg, token, hook): @@ -132,18 +131,18 @@ def continue_lifecycle(self, asg, token, hook): def run_subprocess_command(self, command, dry_run): - message = "Running command {command}.".format(command=command) + message = f"Running command {command}." if not dry_run: logging.info(message) try: output = subprocess.check_output(command.split(' ')) - logging.info("Output was {output}".format(output=output)) + logging.info(f"Output was {output}") except Exception as e: logging.exception(e) raise e else: - logging.info("Dry run: {message}".format(message=message)) + logging.info(f"Dry run: {message}") def get_ec2_instance_by_id(self, instance_id): """ @@ -171,10 +170,10 @@ def verify_ok_to_retire(self, instance_id): for t in instance['Tags']: tags_dict[t['Key']] = t['Value'] if 'safe_to_retire' in tags_dict and tags_dict['safe_to_retire'].lower() == 'true': - logging.info("Instance with id {id} is safe to retire.".format(id=instance_id)) + logging.info(f"Instance with id {instance_id} is safe to retire.") return True else: - logging.info("Instance with id {id} is not safe to retire.".format(id=instance_id)) + logging.info(f"Instance with id {instance_id} is not safe to retire.") return False else: # No instance for id in SQS message this can happen if something else diff --git a/util/vpc-tools/tag-old-ebs.py b/util/vpc-tools/tag-old-ebs.py index 90fdae41335..5ed5b3c3683 100644 --- a/util/vpc-tools/tag-old-ebs.py +++ b/util/vpc-tools/tag-old-ebs.py @@ -3,7 +3,6 @@ For a given aws account, go through all un-attached volumes and tag them. """ -from __future__ import absolute_import import boto import boto.utils import argparse @@ -15,7 +14,6 @@ import yaml # needs to be pip installed import netaddr -from six.moves import filter LOG_FORMAT = "%(asctime)s %(levelname)s - %(filename)s:%(lineno)s - %(message)s" TIMEOUT = 300 @@ -23,7 +21,7 @@ log_level = logging.INFO def tags_for_hostname(hostname, mapping): - logging.debug("Hostname is {}".format(hostname)) + logging.debug(f"Hostname is {hostname}") if not hostname.startswith('ip-'): return {} @@ -54,7 +52,7 @@ def potential_devices(root_device): all_devices = os.listdir(device_dir) all_devices = list(filter(relevant_devices, all_devices)) - logging.info("Potential devices on {}: {}".format(root_device, all_devices)) + logging.info(f"Potential devices on {root_device}: {all_devices}") if len(all_devices) > 1: all_devices.remove(basename(root_device)) @@ -73,7 +71,7 @@ def get_tags_for_disk(mountpoint): edx_dir = join(mountpoint, 'edx', 'app') if exists(hostname_file): # This means this was a root volume. - with open(hostname_file, 'r') as f: + with open(hostname_file) as f: hostname = f.readline().strip() tag_data['hostname'] = hostname @@ -118,7 +116,7 @@ def get_tags_for_disk(mountpoint): args = parser.parse_args() - mappings = yaml.safe_load(open(args.config,'r')) + mappings = yaml.safe_load(open(args.config)) # Setup Logging if args.verbose: @@ -139,7 +137,7 @@ def get_tags_for_disk(mountpoint): # Find all unattached volumes filters = { "status": "available", "availability-zone": az } potential_volumes = ec2.get_all_volumes(filters=filters) - logging.debug("Found {} unattached volumes in {}".format(len(potential_volumes), az)) + logging.debug(f"Found {len(potential_volumes)} unattached volumes in {az}") for vol in potential_volumes: if "cluster" in vol.tags: @@ -158,7 +156,7 @@ def get_tags_for_disk(mountpoint): logging.debug(waiting_msg.format(vol.id, root_device)) timeout -= 2 if timeout <= 0: - logging.critical("Timed out while attaching {}.".format(vol.id)) + logging.critical(f"Timed out while attaching {vol.id}.") exit(1) @@ -168,8 +166,8 @@ def get_tags_for_disk(mountpoint): vol.add_tag("devices_on_volume", str(devices_on_volume)) # Don't tag in this case because the different devices # may have conflicting tags. - logging.info("Skipping {} because it has multiple mountpoints.".format(vol.id)) - logging.info("{} has mountpoints {}".format(vol.id, str(devices_on_volume))) + logging.info(f"Skipping {vol.id} because it has multiple mountpoints.") + logging.info(f"{vol.id} has mountpoints {str(devices_on_volume)}") else: device = devices_on_volume[0] try: @@ -182,9 +180,9 @@ def get_tags_for_disk(mountpoint): # If they are found tag the instance with them if args.noop: - logging.info("Would have tagged {} with: \n{}".format(vol.id, str(tag_data))) + logging.info(f"Would have tagged {vol.id} with: \n{str(tag_data)}") else: - logging.info("Tagging {} with: \n{}".format(vol.id, str(tag_data))) + logging.info(f"Tagging {vol.id} with: \n{str(tag_data)}") vol.add_tags(tag_data) finally: # Un-mount the volume @@ -203,7 +201,7 @@ def get_tags_for_disk(mountpoint): time.sleep(2) timeout -= 2 if timeout <= 0: - logging.critical("Timed out while detaching {}.".format(vol.id)) + logging.critical(f"Timed out while detaching {vol.id}.") exit(1) - logging.debug("Waiting for {} to be detached.".format(vol.id)) + logging.debug(f"Waiting for {vol.id} to be detached.") diff --git a/util/vpc-tools/vpc_dns.py b/util/vpc-tools/vpc_dns.py index 3f34b7793e9..29f7b238642 100644 --- a/util/vpc-tools/vpc_dns.py +++ b/util/vpc-tools/vpc_dns.py @@ -22,8 +22,6 @@ # python vpc_dns.py -s stage-stack -z vpc.example.com # -from __future__ import absolute_import -from __future__ import print_function import argparse import boto import datetime @@ -80,13 +78,13 @@ def add_or_update_record(dns_records): """.format(record.record_name, record.record_type, record.record_ttl, record.record_values) if args.noop: - print(("Would have updated DNS record:\n{}".format(status_msg))) + print(f"Would have updated DNS record:\n{status_msg}") else: - print(("Updating DNS record:\n{}".format(status_msg))) + print(f"Updating DNS record:\n{status_msg}") if record.record_name in record_names: - print(("Unable to create record for {} with value {} because one already exists!".format( - record.record_values, record.record_name))) + print("Unable to create record for {} with value {} because one already exists!".format( + record.record_values, record.record_name)) sys.exit(1) record_names.add(record.record_name) @@ -102,12 +100,12 @@ def add_or_update_record(dns_records): if record.record_name in list(old_records.keys()): if record.record_name + "." == old_records[record.record_name].name and \ record.record_type == old_records[record.record_name].type: - print(("Record for {} already exists and is identical, skipping.\n".format( - record.record_name))) + print("Record for {} already exists and is identical, skipping.\n".format( + record.record_name)) continue if args.force: - print(("Deleting record:\n{}".format(status_msg))) + print(f"Deleting record:\n{status_msg}") change = change_set.add_change( 'DELETE', record.record_name, @@ -135,7 +133,7 @@ def add_or_update_record(dns_records): else: print("Submitting the following change set:\n") xml_doc = xml.dom.minidom.parseString(change_set.to_xml()) - print((xml_doc.toprettyxml(newl=''))) # newl='' to remove extra newlines + print(xml_doc.toprettyxml(newl='')) # newl='' to remove extra newlines if not args.noop: r53.change_rrsets(zone_id, change_set.to_xml()) @@ -154,21 +152,21 @@ def get_or_create_hosted_zone(zone_name): if args.noop: if parent_zone: - print(("Would have created/updated zone: {} parent: {}".format( - zone_name, parent_zone_name))) + print("Would have created/updated zone: {} parent: {}".format( + zone_name, parent_zone_name)) else: - print(("Would have created/updated zone: {}".format( - zone_name, parent_zone_name))) + print("Would have created/updated zone: {}".format( + zone_name, parent_zone_name)) return zone if not zone: - print(("zone {} does not exist, creating".format(zone_name))) + print(f"zone {zone_name} does not exist, creating") ts = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%SUTC') zone = r53.create_hosted_zone( - zone_name, comment="Created by vpc_dns script - {}".format(ts)) + zone_name, comment=f"Created by vpc_dns script - {ts}") if parent_zone: - print(("Updating parent zone {}".format(parent_zone_name))) + print(f"Updating parent zone {parent_zone_name}") dns_records = set() dns_records.add(DNSRecord(parent_zone, zone_name, 'NS', 900, zone.NameServers)) @@ -190,7 +188,7 @@ def get_dns_from_instances(elb): instance = ec2_con.get_all_instances( instance_ids=[inst.id])[0].instances[0] except IndexError: - print(("instance {} attached to elb {}".format(inst, elb))) + print(f"instance {inst} attached to elb {elb}") sys.exit(1) try: env_tag = instance.tags['environment'] @@ -202,8 +200,8 @@ def get_dns_from_instances(elb): play_tag = instance.tags['role'] break # only need the first instance for tag info except KeyError: - print(("Instance {}, attached to elb {} does not " - "have a tag for environment, play or deployment".format(inst, elb))) + print("Instance {}, attached to elb {} does not " + "have a tag for environment, play or deployment".format(inst, elb)) sys.exit(1) return env_tag, deployment_tag, play_tag @@ -240,13 +238,13 @@ def update_elb_rds_dns(zone): if key in elb.name: play_tag = ELB_PLAY_MAPPINGS[key] break - fqdn = "{}-{}-{}.{}".format(env_tag, deployment_tag, play_tag, zone_name) + fqdn = f"{env_tag}-{deployment_tag}-{play_tag}.{zone_name}" # Skip over ELBs if a substring of the ELB name is in # the ELB_BAN_LIST if any(name in elb.name for name in ELB_BAN_LIST): - print(("Skipping {} because it is on the ELB ban list".format(elb.name))) + print(f"Skipping {elb.name} because it is on the ELB ban list") continue dns_records.add(DNSRecord(zone, fqdn, 'CNAME', 600, [elb.dns_name])) diff --git a/util/vpc-tools/vpcutil.py b/util/vpc-tools/vpcutil.py index 05e1110116c..b2ab9ae51a3 100644 --- a/util/vpc-tools/vpcutil.py +++ b/util/vpc-tools/vpcutil.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import boto import boto.rds2 import boto.rds @@ -19,7 +18,7 @@ def stack_name_for_vpc(vpc_name, aws_id, aws_secret): if CFN_TAG_KEY in resource.tags: return resource.tags[CFN_TAG_KEY] else: - msg = "VPC({}) is not part of a cloudformation stack.".format(vpc_name) + msg = f"VPC({vpc_name}) is not part of a cloudformation stack." raise Exception(msg) From da4136f535b6774b31d23a1196c4240e4375e5f3 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Fri, 22 Apr 2022 16:14:25 -0400 Subject: [PATCH 300/664] chore: Remove microsite-related config. These have been deprecated for a while and are not being used. --- docker/build/edxapp/lms.yml | 2 -- docker/build/edxapp/studio.yml | 2 -- playbooks/roles/edxapp/defaults/main.yml | 6 ------ 3 files changed, 10 deletions(-) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 6572821b61b..fe809bb5513 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -353,8 +353,6 @@ MAILCHIMP_NEW_USER_LIST_ID: null MAINTENANCE_BANNER_TEXT: Sample banner message MEDIA_ROOT: /edx/var/edxapp/media/ MEDIA_URL: /media/ -MICROSITE_CONFIGURATION: {} -MICROSITE_ROOT_DIR: /edx/app/edxapp/edx-microsite MKTG_URLS: {} MKTG_URL_LINK_MAP: {} MOBILE_STORE_URLS: {} diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 6ca98c522bc..037e80ed31e 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -315,8 +315,6 @@ LOG_DIR: /edx/var/log/edx MAINTENANCE_BANNER_TEXT: Sample banner message MEDIA_ROOT: /edx/var/edxapp/media/ MEDIA_URL: /media/ -MICROSITE_CONFIGURATION: {} -MICROSITE_ROOT_DIR: /edx/app/edxapp/edx-microsite MKTG_URLS: {} MKTG_URL_LINK_MAP: {} MOBILE_STORE_URLS: {} diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 82e4a44677e..4523b79fee3 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -613,10 +613,6 @@ EDXAPP_VERIFY_STUDENT: EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS: {} EDXAPP_BULK_EMAIL_EMAILS_PER_TASK: 500 -# If using microsites this should point to the microsite repo -EDXAPP_MICROSITE_ROOT_DIR: "{{ edxapp_app_dir }}/edx-microsite" -# this dictionary defines what microsites are configured -EDXAPP_MICROSITE_CONFIGURATION: {} # Instructor code that will not be run in the code sandbox EDXAPP_COURSES_WITH_UNSAFE_CODE: [] EDXAPP_SESSION_COOKIE_DOMAIN: "" @@ -1340,8 +1336,6 @@ generic_env_config: &edxapp_generic_env ZENDESK_CUSTOM_FIELDS: "{{ EDXAPP_ZENDESK_CUSTOM_FIELDS }}" COURSES_WITH_UNSAFE_CODE: "{{ EDXAPP_COURSES_WITH_UNSAFE_CODE }}" BULK_EMAIL_EMAILS_PER_TASK: "{{ EDXAPP_BULK_EMAIL_EMAILS_PER_TASK }}" - MICROSITE_ROOT_DIR: "{{ EDXAPP_MICROSITE_ROOT_DIR }}" - MICROSITE_CONFIGURATION: "{{ EDXAPP_MICROSITE_CONFIGURATION }}" DEFAULT_FILE_STORAGE: "{{ EDXAPP_DEFAULT_FILE_STORAGE }}" GRADES_DOWNLOAD: STORAGE_CLASS: "{{ EDXAPP_GRADE_STORAGE_CLASS | default(None) }}" From e9a270bd536313a6e2c9301834dd8fcaa4029517 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 25 Apr 2022 16:38:03 -0400 Subject: [PATCH 301/664] fix: Mark all directories as safe for git on devstack (#6732) When a user runs make requirements inside devstack's lms-shell (or tries to provision), pip pulls some from dependencies from git URLs and then tries to use git commands against those directories in the virtualenv. However, in devstack, the owner of those repo directories doesn't match the current user (root), and git versions 2.35.2 will refuse to read repositories from a different user (since this could allow code execution). This change tells git to consider all directories safe, at least on devstack. (Listing out specific directories to consider safe isn't really feasible, since the set of git-based dependencies changes on a regular basis.) Notes: - edxapp doesn't use git_clone, but it looks like everything else does, so I had to duplicate the task - There's no one, convenient way of marking something to run only on devstack; apparently the devstack tag isn't used universally when building devstack, but it turns out that either `devstack` or `edx_django_service_is_devstack` is set (and true) in the `docker/build/*/ansible_overrides.yml` files, so we can check if *either* of those is true. This approach is used in various other playbooks already. I also threw on the `devstack` tag for good measure, and again because some other tasks use that approach in combination. ref: ARCHBOM-2096 --- playbooks/roles/edxapp/tasks/deploy.yml | 12 ++++++++++++ playbooks/roles/git_clone/tasks/main.yml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 385cc0a216a..68bf5e85c8c 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -27,6 +27,18 @@ - install - install:base +# On devstack, tell Git that repos owner by other users are safe. +# This is necessary in git 2.35.2 and higher. Devstack uses a mix of +# root and edxapp and git+https pip dependencies end up cloning repos +# into an open-ended set of directories, so our best bet is to just +# say every dir on devstack is safe. +- name: Mark all directories as safe for git on devstack + shell: "git config --global --add safe.directory '*'" + become: true + when: "({{ devstack | default(False) }} or {{ edx_django_service_is_devstack | default(False) }})" + tags: + - devstack + - name: set git fetch.prune to ignore deleted remote refs shell: git config --global fetch.prune true become_user: "{{ edxapp_user }}" diff --git a/playbooks/roles/git_clone/tasks/main.yml b/playbooks/roles/git_clone/tasks/main.yml index 646b6536be5..b3dd6e0cc68 100644 --- a/playbooks/roles/git_clone/tasks/main.yml +++ b/playbooks/roles/git_clone/tasks/main.yml @@ -20,6 +20,18 @@ # Rewrite this task using the ansible git-config module once we'll migrate to Ansible 2.x # https://docs.ansible.com/ansible/git_config_module.html#git-config +# On devstack, tell Git that repos owner by other users are safe. +# This is necessary in git 2.35.2 and higher. Devstack uses a mix of +# root and edxapp and git+https pip dependencies end up cloning repos +# into an open-ended set of directories, so our best bet is to just +# say every dir on devstack is safe. +- name: Mark all directories as safe for git on devstack + shell: "git config --global --add safe.directory '*'" + become: true + when: "({{ devstack | default(False) }} or {{ edx_django_service_is_devstack | default(False) }})" + tags: + - devstack + - name: Set git fetch.prune to ignore deleted remote refs shell: git config --global fetch.prune true become_user: "{{ repo_owner }}" From b0925cebbcb66892600bfa71b6ad8f40dabb8a02 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Fri, 6 May 2022 11:27:19 -0400 Subject: [PATCH 302/664] chore: "Jenkins: ISRE alerts" opsgenie integration migration --- .github/workflows/upgrade-python-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 1b7d2ce51d5..92fa30981d5 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -14,7 +14,7 @@ jobs: with: branch: ${{ github.event.inputs.branch }} team_reviewers: "devops" - email_address: tools-jenkins-isre@edx.opsgenie.net + email_address: tools-jenkins-isre@2u-internal.opsgenie.net send_success_notification: false secrets: requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} From 83188c3695ac4015de83b45952be047f733446e8 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 9 May 2022 14:54:07 +0500 Subject: [PATCH 303/664] feat: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index 726e49710b5..bb416d8579c 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.15.0.7497-1" + version: "11.15.1.7500-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From b3c7f60c43de691770eba68a1866d854fcca383f Mon Sep 17 00:00:00 2001 From: Gabe Mulley Date: Thu, 12 May 2022 20:50:42 -0400 Subject: [PATCH 304/664] feat: allow external sites to embed prospectus /event/ in iframes --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 7243dce5e77..dd72b4a5da3 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -46,6 +46,13 @@ server { add_header Cache-Control "no-store, max-age=0" always; } + location /event/ { + # doesn't have any content, can be aggressively cached + add_header 'Cache-Control' 'public, max-age=86400'; + # this page is designed to be injected into other pages via an iframe + add_header X-Frame-Options ''; + } + # Cache js/css for a long time at the edge, they are versioned in their names location ~ \.(js|css)$ { add_header 'Cache-Control' 'public, max-age=31536000, immutable'; From 64acd164a3560e21ff8cdd7da540322c288dbb18 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Mon, 16 May 2022 15:33:27 +0200 Subject: [PATCH 305/664] fix: use the correct python executable --- playbooks/roles/aws/templates/send-logs-to-s3.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws/templates/send-logs-to-s3.j2 b/playbooks/roles/aws/templates/send-logs-to-s3.j2 index 008f5de3682..7fa6a492086 100644 --- a/playbooks/roles/aws/templates/send-logs-to-s3.j2 +++ b/playbooks/roles/aws/templates/send-logs-to-s3.j2 @@ -97,7 +97,7 @@ onerror() { message_file=/var/tmp/message-$$.json message_string="Error syncing $s3_path: inst_id=$instance_id ip=$ip region={{ aws_region }}" if [[ -r "{{ aws_s3_logfile }}" ]]; then - python -c "import json; d={'Subject':{'Data':'$message_string'},'Body':{'Text':{'Data':open('"{{ aws_s3_logfile }}"').read()}}};print json.dumps(d)" > $message_file + python3 -c "import json; d={'Subject':{'Data':'$message_string'},'Body':{'Text':{'Data':open('"{{ aws_s3_logfile }}"').read()}}};print(json.dumps(d))" > $message_file else cat << EOF > $message_file {"Subject": { "Data": "$message_string" }, "Body": { "Text": { "Data": "!! ERROR !! no logfile" } } } From e0cc0895a598b8a6072e7e8b1e253d665182e948 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Thu, 19 May 2022 17:01:47 +0500 Subject: [PATCH 306/664] Upgrade Node and Npm version for edxapp (#6716) * feat: add edxapp npm pin and upgrade Node Co-authored-by: Saad Ali --- playbooks/roles/edxapp/defaults/main.yml | 9 +++++++-- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- playbooks/roles/edxapp/tasks/main.yml | 10 ++++++++++ playbooks/roles/edxapp/templates/.npmrc.j2 | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 4523b79fee3..8408d014979 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1092,14 +1092,17 @@ edxapp_venv_dir: "{{ edxapp_venvs_dir }}/edxapp" edxapp_venv_bin: "{{ edxapp_venv_dir }}/bin" edxapp_nodeenv_dir: "{{ edxapp_app_dir }}/nodeenvs/edxapp" edxapp_nodeenv_bin: "{{ edxapp_nodeenv_dir }}/bin" +edxapp_npm_dir: "{{ edxapp_app_dir }}/.npm" +edxapp_npm_bin: "{{ edxapp_npm_dir }}/bin" edxapp_settings: '{{ EDXAPP_SETTINGS }}' -EDXAPP_NODE_VERSION: "12" +EDXAPP_NODE_VERSION: "16" +EDXAPP_NPM_VERSION: "8.5.0" # This is where node installs modules, not node itself edxapp_node_bin: "{{ edxapp_code_dir }}/node_modules/.bin" edxapp_user: edxapp edxapp_user_createhome: 'no' edxapp_user_shell: '/bin/false' -edxapp_deploy_path: "{{ edxapp_venv_bin }}:{{ edxapp_code_dir }}/bin:{{ edxapp_node_bin }}:{{ edxapp_nodeenv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +edxapp_deploy_path: "{{ edxapp_venv_bin }}:{{ edxapp_code_dir }}/bin:{{ edxapp_npm_bin }}:{{ edxapp_node_bin }}:{{ edxapp_nodeenv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles" edxapp_media_dir: "{{ edxapp_data_dir }}/media" edxapp_media_dir_s3: "{{ edxapp_media_dir | regex_replace('^\\/', '') }}" @@ -1159,6 +1162,8 @@ edxapp_environment_default: STUDIO_CFG: "{{ edxapp_studio_cfg }}" BOTO_CONFIG: "{{ edxapp_app_dir }}/.boto" REVISION_CFG: "{{ edxapp_revision_cfg }}" + NODE_PATH: "{{ edxapp_npm_dir }}/lib/modules:/usr/lib/node_modules" + MANPATH: "{{ edxapp_npm_dir }}/share/man:$(manpath)" edxapp_environment_extra: {} diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 68bf5e85c8c..2bba7d5aa98 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -250,7 +250,7 @@ - install:app-requirements - name: install node dependencies - shell: "easy_install --version && npm ci" + shell: "easy_install --version && npm install" args: chdir: "{{ edxapp_code_dir }}" environment: "{{ edxapp_environment | combine(git_ssh_environment_mixin) }}" diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index bc902250319..f95d5c58d0d 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -165,6 +165,16 @@ - install - install:base +# This will install npm EDXAPP_NPM_VERSION to edxapp_npm_bin rather than updating the global npm version installed via apt. +# As edxapp_npm_bin is already part of edxapp_environment, the npm command will always pick up the one installed in edxapp_npm_bin. +- name: Pin npm to {{ EDXAPP_NPM_VERSION }} + shell: "npm install -g npm@{{ EDXAPP_NPM_VERSION }}" + environment: "{{ edxapp_environment | combine(git_ssh_environment_mixin) }}" + become_user: "{{ edxapp_user }}" + tags: + - install + - install:base + - name: install python3.8 apt: pkg: diff --git a/playbooks/roles/edxapp/templates/.npmrc.j2 b/playbooks/roles/edxapp/templates/.npmrc.j2 index 3a82b3419dd..70e815aa876 100644 --- a/playbooks/roles/edxapp/templates/.npmrc.j2 +++ b/playbooks/roles/edxapp/templates/.npmrc.j2 @@ -1 +1,2 @@ registry={{ COMMON_NPM_MIRROR_URL }} +prefix={{ edxapp_npm_dir }} From 45865ce455f8571e6a09a435322865c50011cc37 Mon Sep 17 00:00:00 2001 From: Gabe Mulley Date: Thu, 19 May 2022 11:05:45 -0400 Subject: [PATCH 307/664] feat: enable caching of prospectus pages --- .../edx/app/nginx/sites-available/prospectus.j2 | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index dd72b4a5da3..a5d49f31115 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -46,9 +46,7 @@ server { add_header Cache-Control "no-store, max-age=0" always; } - location /event/ { - # doesn't have any content, can be aggressively cached - add_header 'Cache-Control' 'public, max-age=86400'; + location /event { # this page is designed to be injected into other pages via an iframe add_header X-Frame-Options ''; } @@ -58,12 +56,6 @@ server { add_header 'Cache-Control' 'public, max-age=31536000, immutable'; } - # data.json files are used to rehydrate the page, they can be cached for a 30 minutes at the edge. - - location ~ data\.json$ { - add_header 'Cache-Control' 'public, max-age=1800'; - } - # images sometimes change, we want to cache them for an hour at the edge to reduce bandwidth. location /images/ { @@ -141,7 +133,7 @@ server { port_in_redirect off; {% endif %} - add_header 'Cache-Control' 'public, max-age=0, must-revalidate'; + add_header 'Cache-Control' 'public, max-age=1800'; try_files $uri $uri/index.html =404; # PROSPECTUS_STATIC_SITES will be a list of dictionaries which have a: From c92fe0e981ff2c0773d21e380ade5ab05c32b539 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 22 May 2022 21:53:55 -0400 Subject: [PATCH 308/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 12 ++++++------ requirements.txt | 14 +++++++------- requirements/pip-tools.txt | 4 ++-- util/elasticsearch/requirements.txt | 2 +- util/jenkins/requirements-cloudflare.txt | 4 ++-- util/jenkins/requirements.txt | 16 ++++++++-------- util/pingdom/requirements.txt | 2 +- util/vpc-tools/requirements.txt | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index ab0473a5ae5..a675beaf0bd 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,20 +4,20 @@ # # make upgrade # -awscli==1.22.97 +awscli==1.24.5 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.21.42 +boto3==1.23.5 # via -r requirements/aws.in -botocore==1.24.42 +botocore==1.26.5 # via # awscli # boto3 # s3transfer -colorama==0.4.3 +colorama==0.4.4 # via awscli -docutils==0.15.2 +docutils==0.16 # via awscli jmespath==1.0.0 # via @@ -29,7 +29,7 @@ python-dateutil==2.8.2 # via # botocore # s3cmd -python-magic==0.4.25 +python-magic==0.4.26 # via s3cmd pyyaml==5.3.1 # via diff --git a/requirements.txt b/requirements.txt index 6dcbfe4f1be..cf4552dc051 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.8.20 # via -r requirements/base.in -awscli==1.22.97 +awscli==1.24.5 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,14 +14,14 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.21.42 +boto3==1.23.5 # via -r requirements/base.in -botocore==1.24.42 +botocore==1.26.5 # via # awscli # boto3 # s3transfer -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests cffi==1.15.0 # via @@ -30,9 +30,9 @@ cffi==1.15.0 # pynacl charset-normalizer==2.0.12 # via requests -colorama==0.4.3 +colorama==0.4.4 # via awscli -cryptography==36.0.2 +cryptography==37.0.2 # via # ansible # paramiko @@ -44,7 +44,7 @@ decorator==5.1.1 # networkx docopt==0.6.2 # via -r requirements/base.in -docutils==0.15.2 +docutils==0.16 # via awscli ecdsa==0.13.3 # via -r requirements/base.in diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index a31aa926d35..899a014e5f5 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,11 +4,11 @@ # # make upgrade # -click==8.1.2 +click==8.1.3 # via pip-tools pep517==0.12.0 # via pip-tools -pip-tools==6.6.0 +pip-tools==6.6.1 # via -r requirements/pip-tools.in tomli==2.0.1 # via pep517 diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index f7e34bd955d..c284838d599 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -8,7 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -jsonpickle==2.1.0 +jsonpickle==2.2.0 # via deepdiff urllib3==1.26.9 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 4a691480f97..bbfeaaa1cb3 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,11 +4,11 @@ # # make upgrade # -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests charset-normalizer==2.0.12 # via requests -click==8.1.2 +click==8.1.3 # via -r requirements/cloudflare.in idna==3.3 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index bedfe3b822d..17b62de76fc 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.22.97 +awscli==1.24.5 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,16 +16,16 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.21.42 +boto3==1.23.5 # via -r requirements/jenkins.in -botocore==1.24.42 +botocore==1.26.5 # via # awscli # boto3 # s3transfer celery==3.1.25 # via -r requirements/jenkins.in -certifi==2021.10.8 +certifi==2022.5.18.1 # via # opsgenie-sdk # requests @@ -33,9 +33,9 @@ charset-normalizer==2.0.12 # via requests click==6.7 # via -r requirements/jenkins.in -colorama==0.4.3 +colorama==0.4.4 # via awscli -docutils==0.15.2 +docutils==0.16 # via awscli idna==3.3 # via requests @@ -56,9 +56,9 @@ python-dateutil==2.8.2 # botocore # opsgenie-sdk # s3cmd -python-gnupg==0.4.8 +python-gnupg==0.4.9 # via -r requirements/jenkins.in -python-magic==0.4.25 +python-magic==0.4.26 # via s3cmd pytz==2022.1 # via diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index bf934d0b7d6..54560ad3e33 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests charset-normalizer==2.0.12 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 78061aa41d8..6692bd4ad7a 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,7 +6,7 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2021.10.8 +certifi==2022.5.18.1 # via requests charset-normalizer==2.0.12 # via requests From bd52b695adb65e8a66d44619f9ea12e45bc699c2 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Mon, 23 May 2022 12:29:02 -0400 Subject: [PATCH 309/664] chore: Redirect http traffic to DE services back to ELBs --- .../dbt_docs_nginx/templates/nginx/nginx.conf.j2 | 13 ++++++------- .../edx/app/nginx/sites-available/jenkins.j2 | 6 +++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 b/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 index f5ac702b314..f69111a859e 100644 --- a/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 +++ b/playbooks/roles/dbt_docs_nginx/templates/nginx/nginx.conf.j2 @@ -54,12 +54,11 @@ http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/sites-enabled/*; - server{ - - server_name {{ hostname_variable }}.edx.org; - root /usr/share/nginx/html; - + server { + server_name {{ hostname_variable }}.edx.org; + if ($http_x_forwarded_proto = "http") { + return 301 https://$host$request_uri; + } + root /usr/share/nginx/html; } - - } diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 index b130b5e71b8..d5769ff9f44 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/jenkins.j2 @@ -4,7 +4,11 @@ server { listen [::]:{{ jenkins_nginx_port }}; {% endif %} server_name {{ jenkins_server_name }}; - + {%- if jenkins_protocol_https %} + if ($http_x_forwarded_proto = "http") { + return 301 https://$host$request_uri; + } + {%- endif %} location / { proxy_pass http://localhost:{{ jenkins_port }}; From d4e75394ff1ff6d74e2d29614e5972477fe65b53 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Wed, 27 Apr 2022 16:13:26 +0500 Subject: [PATCH 310/664] feat: Added Opensearch role --- playbooks/edx_continuous_integration.yml | 1 + playbooks/openedx_native.yml | 3 + playbooks/opensearch.yml | 45 +++++++++ playbooks/roles/edxapp/defaults/main.yml | 10 ++ playbooks/roles/forum/defaults/main.yml | 9 ++ playbooks/roles/opensearch/defaults/main.yml | 17 ++++ playbooks/roles/opensearch/meta/main.yml | 3 + playbooks/roles/opensearch/tasks/main.yml | 97 +++++++++++++++++++ .../edx/etc/opensearch/jvm.options.j2 | 77 +++++++++++++++ .../edx/etc/opensearch/opensearch.yml.j2 | 97 +++++++++++++++++++ .../lib/systemd/system/opensearch.service.j2 | 51 ++++++++++ 11 files changed, 410 insertions(+) create mode 100644 playbooks/opensearch.yml create mode 100644 playbooks/roles/opensearch/defaults/main.yml create mode 100644 playbooks/roles/opensearch/meta/main.yml create mode 100644 playbooks/roles/opensearch/tasks/main.yml create mode 100644 playbooks/roles/opensearch/templates/edx/etc/opensearch/jvm.options.j2 create mode 100644 playbooks/roles/opensearch/templates/edx/etc/opensearch/opensearch.yml.j2 create mode 100644 playbooks/roles/opensearch/templates/lib/systemd/system/opensearch.service.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index be2bf34d7d6..77f381277a5 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -29,6 +29,7 @@ - testcourses - oraclejdk - elasticsearch + - opensearch - forum - { role: "xqueue", update_users: True } - edx_ansible diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 3342c9f9edb..ec675704522 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -56,6 +56,7 @@ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false EDXAPP_ENABLE_MEMCACHE: true EDXAPP_ENABLE_ELASTIC_SEARCH: true + EDXAPP_ENABLE_OPEN_SEARCH: TRUE # Ecommerce ECOMMERCE_CORS_ORIGIN_WHITELIST: [ "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", @@ -115,6 +116,8 @@ - oauth_client_setup - role: elasticsearch when: EDXAPP_ENABLE_ELASTIC_SEARCH + - role: opensearch + when: EDXAPP_ENABLE_OPEN_SEARCH - forum - role: discovery when: SANDBOX_ENABLE_DISCOVERY diff --git a/playbooks/opensearch.yml b/playbooks/opensearch.yml new file mode 100644 index 00000000000..c499d0b0958 --- /dev/null +++ b/playbooks/opensearch.yml @@ -0,0 +1,45 @@ +- hosts: all + become: True + vars: + # By default take instances in and out of the elb(s) they + # are attached to + # To skip elb operations use "-e elb_pre_post=fase" + elb_pre_post: true + # Number of instances to operate on at a time + serial_count: 1 + CLUSTER_NAME: "commoncluster" + serial: "{{ serial_count }}" + pre_tasks: + - action: ec2_metadata_facts + when: elb_pre_post + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Instance De-register + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + region: us-east-1 + state: absent + wait_timeout: 60 + become: False + when: elb_pre_post + roles: + - common + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - opensearch + post_tasks: + - debug: + var: ansible_ec2_instance_id + when: elb_pre_post + - name: Register instance in the elb + local_action: ec2_elb + args: + instance_id: "{{ ansible_ec2_instance_id }}" + ec2_elbs: "{{ ec2_elbs }}" + region: us-east-1 + state: present + wait_timeout: 60 + become: False + when: elb_pre_post diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 8408d014979..04293f2abe6 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -123,6 +123,10 @@ EDXAPP_SEARCH_HOST: 'localhost' EDXAPP_SEARCH_PORT: 9200 EDXAPP_SEARCH_USE_SSL: false +EDXAPP_OPENSEARCH_HOST: 'localhost' +EDXAPP_OPENSEARCH_PORT: 9202 +EDXAPP_OPENSEARCH_USE_SSL: false + # list of dictionaries of the format # { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' } # http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch @@ -131,6 +135,11 @@ EDXAPP_ELASTIC_SEARCH_CONFIG: port: "{{ EDXAPP_SEARCH_PORT }}" use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" +EDXAPP_OPEN_SEARCH_CONFIG: + - host: "{{ EDXAPP_OPENSEARCH_HOST }}" + port: "{{ EDXAPP_OPENSEARCH_PORT }}" + use_ssl: "{{ EDXAPP_OPENSEARCH_USE_SSL }}" + EDXAPP_SETTINGS: '{{ COMMON_EDXAPP_SETTINGS }}' EDXAPP_LMS_ENV: 'lms.envs.{{ EDXAPP_SETTINGS }}' @@ -1505,6 +1514,7 @@ generic_env_config: &edxapp_generic_env user: '{{ edxapp_sandbox_user }}' AFFILIATE_COOKIE_NAME: "{{ EDXAPP_AFFILIATE_COOKIE_NAME }}" ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" + OPEN_SEARCH_CONFIG: "{{ EDXAPP_OPEN_SEARCH_CONFIG }}" PLATFORM_TWITTER_ACCOUNT: "{{ EDXAPP_PLATFORM_TWITTER_ACCOUNT }}" PLATFORM_FACEBOOK_ACCOUNT: "{{ EDXAPP_PLATFORM_FACEBOOK_ACCOUNT }}" HELP_TOKENS_BOOKS: "{{ EDXAPP_HELP_TOKENS_BOOKS }}" diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index cc9a1909d36..8df0032c3a7 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -37,6 +37,12 @@ FORUM_ELASTICSEARCH_URL: "{{ FORUM_ELASTICSEARCH_PROTOCOL }}://{{ FORUM_ELASTICS FORUM_ELASTICSEARCH_HOST_ES7: "localhost" FORUM_ELASTICSEARCH_URL_ES7: "{{ FORUM_ELASTICSEARCH_PROTOCOL }}://{{ FORUM_ELASTICSEARCH_HOST_ES7 }}:{{ FORUM_ELASTICSEARCH_PORT }}" +#Opensearch config +FORUM_OPENSEARCH_HOST: "localhost" +FORUM_OPENSEARCH_PORT: "9202" +FORUM_OPENSEARCH_PROTOCOL: "http" +FORUM_OPENSEARCH_URL: "{{ FORUM_OPENSEARCH_PROTOCOL }}://{{ FORUM_OPENSEARCH_HOST }}:{{ FORUM_OPENSEARCH_PORT }}" + # This needs to be a string, set to 'false' to disable FORUM_NEW_RELIC_ENABLE: '{{ COMMON_ENABLE_NEWRELIC_APP }}' FORUM_NEW_RELIC_LICENSE_KEY: '{{ NEWRELIC_LICENSE_KEY | default("") }}' @@ -66,6 +72,7 @@ forum_base_env: &forum_base_env API_KEY: "{{ FORUM_API_KEY }}" SEARCH_SERVER: "{{ FORUM_ELASTICSEARCH_URL }}" SEARCH_SERVER_ES7: "{{ FORUM_ELASTICSEARCH_URL_ES7 }}" + SEARCH_SERVER_OS12: "{{ FORUM_OPENSEARCH_URL }}" MONGOHQ_URL: "{{ FORUM_MONGO_URL }}" MONGOID_USE_SSL: "{{ FORUM_MONGO_USE_SSL }}" MONGOID_AUTH_SOURCE: "{{ FORUM_MONGO_AUTH_DB }}" @@ -88,6 +95,7 @@ devstack_forum_env: SINATRA_ENV: "development" SEARCH_SERVER: "http://edx.devstack.elasticsearch:9200/" SEARCH_SERVER_ES7: "http://edx.devstack.elasticsearch710:9200/" + SEARCH_SERVER_OS12: "http://edx.devstack.opensearch12:9202/" MONGOHQ_URL: "mongodb://cs_comments_service:password@edx.devstack.mongo:27017/cs_comments_service" MONGOID_AUTH_MECH: "{{ FORUM_MONGO_AUTH_MECH }}" @@ -108,3 +116,4 @@ FORUM_VERSION: "master" forum_services: - {service: "elasticsearch", host: "{{ FORUM_ELASTICSEARCH_HOST }}", port: "{{ FORUM_ELASTICSEARCH_PORT }}"} - {service: "elasticsearch7", host: "{{ FORUM_ELASTICSEARCH_HOST_ES7 }}", port: "{{ FORUM_ELASTICSEARCH_PORT }}"} + - {service: "opensearch12", host: "{{ FORUM_OPENSEARCH_HOST }}", port: "{{ FORUM_OPENSEARCH_PORT }}"} diff --git a/playbooks/roles/opensearch/defaults/main.yml b/playbooks/roles/opensearch/defaults/main.yml new file mode 100644 index 00000000000..e183b74c6f0 --- /dev/null +++ b/playbooks/roles/opensearch/defaults/main.yml @@ -0,0 +1,17 @@ +--- +opensearch_app_dir: "{{ COMMON_APP_DIR }}/opensearch" +opensearch_data_dir: "{{ COMMON_DATA_DIR }}/opensearch" +opensearch_log_dir: "{{ COMMON_LOG_DIR }}/opensearch" +opensearch_cfg_dir: "{{ COMMON_CFG_DIR }}/opensearch" +opensearch_download_url: "https://artifacts.opensearch.org/releases/bundle/opensearch" +opensearch_user: "opensearch" +opensearch_group: "opensearch" + +# +# Defaults for a single server installation. +OPENSEARCH_CLUSTER_MEMBERS: [] +OPENSEARCH_HEAP_SIZE: "512m" +OPENSEARCH_START_TIMEOUT: "1200" +OPENSEARCH_VERSION: "1.2.0" + +systemctl_path: /etc/systemd/system diff --git a/playbooks/roles/opensearch/meta/main.yml b/playbooks/roles/opensearch/meta/main.yml new file mode 100644 index 00000000000..2083f0e1251 --- /dev/null +++ b/playbooks/roles/opensearch/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - common diff --git a/playbooks/roles/opensearch/tasks/main.yml b/playbooks/roles/opensearch/tasks/main.yml new file mode 100644 index 00000000000..0767f902720 --- /dev/null +++ b/playbooks/roles/opensearch/tasks/main.yml @@ -0,0 +1,97 @@ +--- +- name: Download opensearch {{ OPENSEARCH_VERSION }} + get_url: + url: "{{ opensearch_download_url }}/{{ OPENSEARCH_VERSION }}/opensearch-{{ OPENSEARCH_VERSION }}-linux-x64.tar.gz" + dest: "/tmp/opensearch.tar.gz" + tags: + - install + - install:base + +- name: Create opensearch user + user: + name: "{{ opensearch_user }}" + state: present + shell: /bin/bash + tags: + - install + - install:base + +- name: Create home directory + file: + path: "{{ item }}" + state: directory + owner: "{{ opensearch_user }}" + group: "{{ opensearch_user }}" + with_items: + - "{{ opensearch_app_dir }}" + - "{{ opensearch_data_dir }}" + - "{{ opensearch_log_dir }}" + - "{{ opensearch_cfg_dir }}" + tags: + - install + - install:base + +- name: Extract the tar file + command: chdir=/tmp/ tar -xvzf opensearch.tar.gz -C "{{ opensearch_app_dir }}" --strip-components=1 + tags: + - install + - install:base + register: opensearch_reinstall + +- name: Copy Configuration File + blockinfile: + block: "{{ lookup('template', 'templates/edx/etc/opensearch/opensearch.yml.j2') }}" + dest: "{{ opensearch_app_dir }}/config/opensearch.yml" + backup: yes + state: present + create: yes + marker: "## {mark} opensearch main configuration ##" + owner: "{{ opensearch_user }}" + group: "{{ opensearch_user }}" + mode: 0600 + tags: + - install + - install:base + +- name: Copy jvm.options File for Instance + template: + src: edx/etc/opensearch/jvm.options.j2 + dest: "{{ opensearch_app_dir }}/config/jvm.options" + owner: "{{ opensearch_user }}" + group: "{{ opensearch_user }}" + mode: 0600 + force: yes + tags: + - install + - install:base + +- name: Create systemd service + template: + src: lib/systemd/system/opensearch.service.j2 + dest: "{{ systemctl_path }}/opensearch.service" + tags: + - install + - install:base + +- name: Make sure opensearch is started + service: + name: opensearch + state: started + enabled: yes + tags: + - install + - install:base + +- name: Get all the installed ES plugins + command: "/edx/app/opensearch/bin/opensearch-plugin list" + register: list_plugins + tags: + - install + - install:base + +- name: Show all the installed ES plugins + debug: + msg: "{{ list_plugins.stdout }}" + tags: + - install + - install:base diff --git a/playbooks/roles/opensearch/templates/edx/etc/opensearch/jvm.options.j2 b/playbooks/roles/opensearch/templates/edx/etc/opensearch/jvm.options.j2 new file mode 100644 index 00000000000..130eee4499c --- /dev/null +++ b/playbooks/roles/opensearch/templates/edx/etc/opensearch/jvm.options.j2 @@ -0,0 +1,77 @@ +# {{ ansible_managed }} + +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ OPENSEARCH_HEAP_SIZE }} +-Xmx{{ OPENSEARCH_HEAP_SIZE }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +8-13:-XX:+UseConcMarkSweepGC +8-13:-XX:CMSInitiatingOccupancyFraction=75 +8-13:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1 GC is only supported on JDK version 10 or later +# to use G1GC, uncomment the next two lines and update the version on the +# following three lines to your version of the JDK +# 10-13:-XX:-UseConcMarkSweepGC +# 10-13:-XX:-UseCMSInitiatingOccupancyOnly +14-:-XX:+UseG1GC +14-:-XX:G1ReservePercent=25 +14-:-XX:InitiatingHeapOccupancyPercent=30 + +## JVM temporary directory +-Djava.io.tmpdir=${OPENSEARCH_TMPDIR} + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath={{ opensearch_data_dir }} + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile={{ opensearch_log_dir }}/hs_err_pid%p.log + +## JDK 8 GC logging +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:{{ opensearch_log_dir }}/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ opensearch_log_dir }}/gc.log:utctime,pid,tags:filecount=32,filesize=64m diff --git a/playbooks/roles/opensearch/templates/edx/etc/opensearch/opensearch.yml.j2 b/playbooks/roles/opensearch/templates/edx/etc/opensearch/opensearch.yml.j2 new file mode 100644 index 00000000000..a1795e8dd52 --- /dev/null +++ b/playbooks/roles/opensearch/templates/edx/etc/opensearch/opensearch.yml.j2 @@ -0,0 +1,97 @@ +# {{ ansible_managed }} + +# ======================== Opensearch Configuration ========================= +# +# NOTE: Opensearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +#node.name: node-1 +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: {{ opensearch_data_dir }} +# +# Path to log files: +# +path.logs: {{ opensearch_log_dir }} +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Opensearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +#network.host: 192.168.0.1 +# +# Set a custom port for HTTP: +# +http.port: 9202 +# +{% if vagrant_cluster|bool %} +network.host: {{ ansible_ssh_host }} +{% endif %} +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +{% if OPENSEARCH_CLUSTER_MEMBERS|length > 1 -%} +discovery.seed_hosts: ['{{OPENSEARCH_CLUSTER_MEMBERS|join("\',\'") }}'] +{% endif -%} +# +discovery.type: single-node +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, consult the gateway module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +#action.destructive_requires_name: true +# +# ---------------------------------- Plugins ----------------------------------- +plugins.security.disabled: true \ No newline at end of file diff --git a/playbooks/roles/opensearch/templates/lib/systemd/system/opensearch.service.j2 b/playbooks/roles/opensearch/templates/lib/systemd/system/opensearch.service.j2 new file mode 100644 index 00000000000..b5d2b030fc7 --- /dev/null +++ b/playbooks/roles/opensearch/templates/lib/systemd/system/opensearch.service.j2 @@ -0,0 +1,51 @@ +[Unit] +Description=opensearch +Wants=network-online.target +After=network-online.target + +[Service] +RuntimeDirectory=opensearch +PrivateTmp=true + +WorkingDirectory={{ opensearch_app_dir }} + +User={{ opensearch_user }} +Group={{ opensearch_group }} + +ExecStart={{ opensearch_app_dir }}/bin/opensearch -p {{ opensearch_app_dir }}/opensearch.pid -q + +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +LimitNOFILE=65536 + +# Specifies the memory lock settings +LimitMEMLOCK=infinity + +# Specifies the maximum number of processes +LimitNPROC=4096 + +# Specifies the maximum size of virtual memory +LimitAS=infinity + +# Specifies the maximum file size +LimitFSIZE=infinity + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop the Java process +KillSignal=SIGTERM + +# Send the signal only to the JVM rather than its control group +KillMode=process + +# Java process is never killed +SendSIGKILL=no + +# When a JVM receives a SIGTERM signal it exits with code 143 +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 687dd2de5eccb7ba8cdfc3530cd3d69f0e09300c Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 30 May 2022 13:31:59 +0500 Subject: [PATCH 311/664] chore: Removing unused path from config. --- playbooks/roles/edxapp/templates/code.sandbox.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/edxapp/templates/code.sandbox.j2 b/playbooks/roles/edxapp/templates/code.sandbox.j2 index 911af26174f..719b94db2bd 100644 --- a/playbooks/roles/edxapp/templates/code.sandbox.j2 +++ b/playbooks/roles/edxapp/templates/code.sandbox.j2 @@ -4,7 +4,6 @@ #include {{ edxapp_sandbox_venv_dir }}/** mr, - {{ edxapp_code_dir }}/common/lib/sandbox-packages/** r, /tmp/codejail-*/ rix, /tmp/codejail-*/** wrix, From a8dd4bb5009d3c6b2c4f0063394de4e436223894 Mon Sep 17 00:00:00 2001 From: Ardiea Date: Fri, 27 May 2022 11:27:08 -0400 Subject: [PATCH 312/664] fix: Adding a missing space to the single-beat binary call in beat_scheduler.sh.j2. --- .../roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 index 12068c05450..88d764c41f5 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/beat_scheduler.sh.j2 @@ -14,7 +14,7 @@ if command -v ec2metadata >/dev/null 2>&1; then export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" fi {% else %} -{% set executable = edxapp_venv_bin + '/single-beat' + edxapp_venv_bin + '/celery' %} +{% set executable = edxapp_venv_bin + '/single-beat ' + edxapp_venv_bin + '/celery' %} {% endif %} # We exec so that celery is the child of supervisor and can be managed properly From ce74761cfde8a99797cda8ad5611e2d6f86b8000 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Fri, 15 Apr 2022 16:12:34 +0500 Subject: [PATCH 313/664] feat: Upgrade ansible to 2.9 --- CHANGELOG.md | 3 +++ playbooks/roles/aws/tasks/main.yml | 2 +- playbooks/roles/aws/templates/requirements.txt.j2 | 8 ++++---- playbooks/roles/common/tasks/main.yml | 10 +++++----- playbooks/roles/edx_django_service/tasks/main.yml | 4 ++-- playbooks/roles/edx_service/tasks/main.yml | 6 +++--- playbooks/roles/edxapp/tasks/main.yml | 8 ++++---- playbooks/roles/edxapp_common/tasks/main.yml | 2 +- playbooks/roles/flower/handlers/main.yml | 3 ++- playbooks/roles/ghost/tasks/main.yml | 2 +- playbooks/roles/hotg/tasks/deploy.yml | 9 ++++++--- playbooks/roles/nginx/tasks/main.yml | 6 +++--- playbooks/roles/nltk/tasks/main.yml | 4 ++-- playbooks/roles/openstack/tasks/main.yml | 3 ++- playbooks/roles/prospectus/tasks/main.yml | 6 +++--- playbooks/roles/tableau/tasks/main.yml | 2 +- playbooks/vagrant-cluster.yml | 4 ++-- requirements.txt | 10 +++++----- requirements/base.in | 2 +- requirements/pip-tools.txt | 2 +- util/jenkins/requirements.txt | 8 ++++---- 21 files changed, 56 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa4dd21a40..2f68368365d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-06-01 + - Upgrade ansible to 2.9 + - 2022-04-06 - Role: edxapp - Added a new `EDXAPP_COMPILE_JSI18N` variable to control whether diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml index cd8f8f53c10..8ad20155e64 100644 --- a/playbooks/roles/aws/tasks/main.yml +++ b/playbooks/roles/aws/tasks/main.yml @@ -53,7 +53,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: ansible_distribution in common_debian_variants diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index a675beaf0bd..5fb598be16f 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.24.5 +awscli==1.25.0 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.23.5 +boto3==1.24.0 # via -r requirements/aws.in -botocore==1.26.5 +botocore==1.27.0 # via # awscli # boto3 @@ -39,7 +39,7 @@ rsa==4.7.2 # via awscli s3cmd==2.2.0 # via -r requirements/aws.in -s3transfer==0.5.2 +s3transfer==0.6.0 # via # awscli # boto3 diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index cbb76003cae..a30b14ef309 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -61,7 +61,7 @@ repo: "{{ common_git_ppa }}" update_cache: yes register: add_repo - until: add_repo|success + until: add_repo is success retries: 10 delay: 5 when: ansible_distribution in common_debian_variants @@ -71,7 +71,7 @@ repo: "ppa:linuxuprising/apps" update_cache: yes register: add_repo - until: add_repo|success + until: add_repo is success retries: 10 delay: 5 when: > @@ -112,7 +112,7 @@ repo: "ppa:deadsnakes/ppa" update_cache: yes register: add_repo - until: add_repo|success + until: add_repo is success retries: 10 delay: 5 when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' @@ -128,7 +128,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: ansible_distribution in common_debian_variants @@ -153,7 +153,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: > diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 31feb5b867b..00fe4e7e696 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -68,7 +68,7 @@ - python3.8-distutils update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: edx_django_service_use_python38 and not edx_django_service_enable_experimental_docker_shim @@ -81,7 +81,7 @@ name: "{{ item }}" update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: edx_django_service_use_python3 and not edx_django_service_enable_experimental_docker_shim diff --git a/playbooks/roles/edx_service/tasks/main.yml b/playbooks/roles/edx_service/tasks/main.yml index 1b6991f3354..159ad57e93b 100644 --- a/playbooks/roles/edx_service/tasks/main.yml +++ b/playbooks/roles/edx_service/tasks/main.yml @@ -122,7 +122,7 @@ - python3.8-distutils update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: edx_service_use_python38 @@ -135,7 +135,7 @@ name: "{{ item }}" update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: edx_service_use_python3 and not edx_service_use_python38 @@ -199,7 +199,7 @@ update_cache: true cache_valid_time: 3600 register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: ansible_distribution in common_debian_variants diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index f95d5c58d0d..8c8aacd187d 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -110,7 +110,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 with_flattened: @@ -134,7 +134,7 @@ state: present update_cache: yes register: add_repo - until: add_repo|success + until: add_repo is success retries: 10 delay: 5 tags: @@ -147,7 +147,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 tags: @@ -182,7 +182,7 @@ - python3.8-distutils update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 tags: diff --git a/playbooks/roles/edxapp_common/tasks/main.yml b/playbooks/roles/edxapp_common/tasks/main.yml index b8284df5ab6..b8d0e537776 100644 --- a/playbooks/roles/edxapp_common/tasks/main.yml +++ b/playbooks/roles/edxapp_common/tasks/main.yml @@ -5,7 +5,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 tags: diff --git a/playbooks/roles/flower/handlers/main.yml b/playbooks/roles/flower/handlers/main.yml index c70b4f40766..2ba91d1c454 100644 --- a/playbooks/roles/flower/handlers/main.yml +++ b/playbooks/roles/flower/handlers/main.yml @@ -5,4 +5,5 @@ supervisorctl_path: "{{ supervisor_ctl }}" config: "{{ supervisor_cfg }}" name: "{{ FLOWER_USER }}" - sudo_user: "{{ supervisor_service_user }}" + become: true + become_user: "{{ supervisor_service_user }}" diff --git a/playbooks/roles/ghost/tasks/main.yml b/playbooks/roles/ghost/tasks/main.yml index 0e2082709ba..bbfcc29cf8c 100644 --- a/playbooks/roles/ghost/tasks/main.yml +++ b/playbooks/roles/ghost/tasks/main.yml @@ -17,7 +17,7 @@ - name: Install ghost_package apt: deb="{{ ghost_download_target }}" - sudo: true + become: true when: ghost_correct.rc != 0 - name: "Install percona packages for dropping large tables" diff --git a/playbooks/roles/hotg/tasks/deploy.yml b/playbooks/roles/hotg/tasks/deploy.yml index f97daea5248..dcb4f1ac05b 100644 --- a/playbooks/roles/hotg/tasks/deploy.yml +++ b/playbooks/roles/hotg/tasks/deploy.yml @@ -42,7 +42,8 @@ supervisorctl_path={{ supervisor_ctl }} config={{ supervisor_cfg }} name="{{ HOTG_SERVICE_NAME }}" - sudo_user: "{{ supervisor_service_user }}" + become: true + become_user: "{{ supervisor_service_user }}" tags: - manage - manage:stop @@ -52,7 +53,8 @@ src=edx/app/hotg/Config.groovy.j2 dest={{ hotg_app_dir }}/Config.groovy mode=0644 - sudo_user: "{{ HOTG_USER }}" + become: true + become_user: "{{ HOTG_USER }}" tags: - install - install:configuration @@ -92,7 +94,8 @@ supervisorctl_path={{ supervisor_ctl }} config={{ supervisor_cfg }} name="{{ HOTG_SERVICE_NAME }}" - sudo_user: "{{ supervisor_service_user }}" + become: true + become_user: "{{ supervisor_service_user }}" tags: - manage - manage:start diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index bde5eb34ed9..bb7bb59ba8c 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -46,7 +46,7 @@ state: present update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 notify: restart nginx @@ -69,7 +69,7 @@ state: present update_cache: yes register: add_repo - until: add_repo|success + until: add_repo is success retries: 10 delay: 5 notify: restart nginx @@ -97,7 +97,7 @@ state: latest update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 notify: restart nginx diff --git a/playbooks/roles/nltk/tasks/main.yml b/playbooks/roles/nltk/tasks/main.yml index 025257edea0..9681c67f10e 100644 --- a/playbooks/roles/nltk/tasks/main.yml +++ b/playbooks/roles/nltk/tasks/main.yml @@ -3,7 +3,7 @@ - name: Install unzip apt: pkg=unzip state=present update_cache=yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 @@ -28,6 +28,6 @@ args: chdir: "{{ NLTK_DATA_DIR }}/{{ item.path|dirname }}" with_items: "{{ NLTK_DATA }}" - when: nltk_download|changed + when: nltk_download is changed tags: - deploy diff --git a/playbooks/roles/openstack/tasks/main.yml b/playbooks/roles/openstack/tasks/main.yml index 7e1a87bba1a..8a9a7df417f 100644 --- a/playbooks/roles/openstack/tasks/main.yml +++ b/playbooks/roles/openstack/tasks/main.yml @@ -58,7 +58,8 @@ command: "{{ edxapp_venv_dir }}/bin/pip install {{ COMMON_PIP_VERBOSITY }} -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ openstack_requirements_file }}" args: chdir: "{{ edxapp_code_dir }}" - sudo_user: "{{ edxapp_user }}" + become: true + become_user: "{{ edxapp_user }}" environment: "{{ edxapp_environment }}" when: edxapp_code_dir is defined tags: diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index c65618664f9..75caf9f3a76 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -66,7 +66,7 @@ - python3.8-distutils update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 when: prospectus_use_python3 @@ -89,7 +89,7 @@ become_user: "{{ prospectus_user }}" environment: "{{ prospectus_env_vars }}" register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 tags: @@ -164,7 +164,7 @@ name: "pngquant" update_cache: yes register: install_pkgs - until: install_pkgs|success + until: install_pkgs is success retries: 10 delay: 5 tags: diff --git a/playbooks/roles/tableau/tasks/main.yml b/playbooks/roles/tableau/tasks/main.yml index 99b08f3ec46..32925860a55 100644 --- a/playbooks/roles/tableau/tasks/main.yml +++ b/playbooks/roles/tableau/tasks/main.yml @@ -116,7 +116,7 @@ state: "present" regexp: '^%{{ tableau_user }} ALL=' line: '%{{ tableau_user }} ALL=(ALL) NOPASSWD: ALL' - sudo: true + become: true tags: - install - install:code diff --git a/playbooks/vagrant-cluster.yml b/playbooks/vagrant-cluster.yml index 5c964441077..c50b1797138 100644 --- a/playbooks/vagrant-cluster.yml +++ b/playbooks/vagrant-cluster.yml @@ -23,7 +23,7 @@ # Rabbit needs to be built serially - name: Configure group cluster serial roles hosts: all - sudo: True + become: True serial: 1 gather_facts: True vars: @@ -40,7 +40,7 @@ # but will also show as failed - name: Configure group with tasks that will always fail hosts: all - sudo: True + become: True gather_facts: True vars: MARIADB_CLUSTERED: yes diff --git a/requirements.txt b/requirements.txt index cf4552dc051..ec54e8f6c72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ # # make upgrade # -ansible==2.8.20 +ansible==2.9.0 # via -r requirements/base.in -awscli==1.24.5 +awscli==1.25.0 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,9 +14,9 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.23.5 +boto3==1.24.0 # via -r requirements/base.in -botocore==1.26.5 +botocore==1.27.0 # via # awscli # boto3 @@ -97,7 +97,7 @@ requests==2.27.1 # datadog rsa==4.7.2 # via awscli -s3transfer==0.5.2 +s3transfer==0.6.0 # via # awscli # boto3 diff --git a/requirements/base.in b/requirements/base.in index 7e12eaefb18..e429c2e6393 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,7 +1,7 @@ -c constraints.txt # Standard dependencies for Ansible runs -ansible<2.9.0 +ansible==2.9.0 awscli boto boto3 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 899a014e5f5..ce56f4c7f38 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ click==8.1.3 # via pip-tools pep517==0.12.0 # via pip-tools -pip-tools==6.6.1 +pip-tools==6.6.2 # via -r requirements/pip-tools.in tomli==2.0.1 # via pep517 diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 17b62de76fc..fe1f416d5c5 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu -awscli==1.24.5 +awscli==1.25.0 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,9 +16,9 @@ billiard==3.3.0.23 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.23.5 +boto3==1.24.0 # via -r requirements/jenkins.in -botocore==1.26.5 +botocore==1.27.0 # via # awscli # boto3 @@ -76,7 +76,7 @@ rsa==4.7.2 # via awscli s3cmd==2.2.0 # via -r requirements/jenkins.in -s3transfer==0.5.2 +s3transfer==0.6.0 # via # awscli # boto3 From ff26c97db02616b7228d35a8b1f7ca570deed096 Mon Sep 17 00:00:00 2001 From: Soban Javed Date: Thu, 26 May 2022 16:47:16 +0500 Subject: [PATCH 314/664] build: remove ppa for watchman as it is available by default in ubuntu 20.04 --- playbooks/roles/common/tasks/main.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index a30b14ef309..927a0c0e42c 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -66,20 +66,6 @@ delay: 5 when: ansible_distribution in common_debian_variants -- name: Add ppa for watchman package - apt_repository: - repo: "ppa:linuxuprising/apps" - update_cache: yes - register: add_repo - until: add_repo is success - retries: 10 - delay: 5 - when: > - ansible_distribution in common_debian_variants and - ({{ devstack | default(False) }} or {{ edx_django_service_is_devstack | default(False) }}) - tags: - - "devstack" - # Ensure that we can install old software if need be. - name: Add edX PPA apt key apt_key: @@ -140,7 +126,7 @@ state: present update_cache: yes when: > - ansible_distribution in common_debian_variants and + ansible_distribution in common_debian_variants and ({{ devstack | default(False) }} or {{ edx_django_service_is_devstack | default(False) }}) tags: - "devstack" From aa68c9e69136ebf722903d2ff4bf3de32cee427a Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> Date: Mon, 6 Jun 2022 13:40:34 +0500 Subject: [PATCH 315/664] doc: added changelog entry for watchman PPA removal --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f68368365d..fe09ad67828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-06-06 + - Role: common + - Remove PPA for `watchman` as we have shifted to Ubuntu 20.04, + if you are running this against Ubuntu version less than 20.04 + then installation will fail. + - 2022-06-01 - Upgrade ansible to 2.9 From 33ab71c2276db0a19536d447608dcaaca03ae652 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Wed, 15 Jun 2022 18:00:51 +0200 Subject: [PATCH 316/664] feat: extend aws_cloudwatch_agent to collect logs --- playbooks/openedx_native.yml | 5 +++++ .../aws_cloudwatch_agent/defaults/main.yml | 6 ++++++ .../roles/aws_cloudwatch_agent/tasks/main.yml | 4 ++-- ...ent.json => amazon-cloudwatch-agent.json.j2} | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) rename playbooks/roles/aws_cloudwatch_agent/templates/{amazon-cloudwatch-agent.json => amazon-cloudwatch-agent.json.j2} (57%) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index ec675704522..4ecd54d3044 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -68,6 +68,9 @@ # For the mfe role. COMMON_ECOMMERCE_BASE_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}' ECOMMERCE_ENABLE_PAYMENT_MFE: true + + EDXAPP_ENABLE_CLOUDWATCH: false + cloudwatch_logs_enabled: '{{ EDXAPP_ENABLE_CLOUDWATCH }}' roles: - role: swapfile SWAPFILE_SIZE: 4GB @@ -128,6 +131,8 @@ when: COMMON_ENABLE_DATADOG - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER + - role: aws_cloudwatch_agent + when: EDXAPP_ENABLE_CLOUDWATCH - role: postfix_queue when: POSTFIX_QUEUE_EXTERNAL_SMTP_HOST != '' - role: datadog-uninstall diff --git a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml index 341a074d3dc..1e00a202189 100644 --- a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml +++ b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml @@ -14,3 +14,9 @@ cloudwatch_namespace: Analytics/Monitor # Collectd installation parameters collectd_version: "5.9.2.g-1ubuntu5" collectd_install_recommends: yes + +# CloudWatch logs configuration +cloudwatch_logs_enabled: false +# List of objects with `file_path`, `log_group_name` and `log_stream_name` keys. +# See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html#CloudWatch-Agent-Configuration-File-Logssection +cloudwatch_logs_collect_list: [] diff --git a/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml index 72f136a3aef..6353adbdec7 100644 --- a/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml +++ b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml @@ -39,7 +39,7 @@ dest: "{{ item.dest }}" mode: "{{ item.mode }}" with_items: - - { src: 'amazon-cloudwatch-agent.json', dest: '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', mode: '0644' } + - { src: 'amazon-cloudwatch-agent.json.j2', dest: '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json', mode: '0644' } tags: - install - install:base @@ -53,7 +53,7 @@ - install:base - name: Run AWS CloudWatch Agent - shell: " /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json" + shell: "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json" become: yes tags: - install diff --git a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 similarity index 57% rename from playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json rename to playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 index 742d0767e6a..f2a7307b941 100644 --- a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json +++ b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 @@ -33,5 +33,22 @@ "service_address": ":8125" } } + }{% if cloudwatch_logs_enabled %}, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + {%- for log_config in cloudwatch_logs_collect_list -%} + { + "file_path": "{{ log_config.file_path }}", + "log_group_name": "{{ log_config.log_group_name }}", + "log_stream_name": "{{ log_config.log_stream_name }}" + }{{ ", " if not loop.last else "" }} + {%- endfor -%} + ] + } + }, + "log_stream_name": "default_server_log_stream" } + {% endif %} } From 410add980444661f096c084e15bac847460c7728 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Tue, 5 Jul 2022 07:27:54 -0400 Subject: [PATCH 317/664] chore: Add procstat plugin to cloudwatch config files with nginx, jenkins, cloudwatch as patterns --- .../aws_cloudwatch_agent/defaults/main.yml | 4 ++++ .../roles/aws_cloudwatch_agent/tasks/main.yml | 7 +++++++ .../templates/amazon-cloudwatch-agent.json.j2 | 18 ++++++++++++++++++ .../roles/dbt_docs_nginx/defaults/main.yml | 2 ++ .../jenkins_data_engineering/defaults/main.yml | 3 +++ .../defaults/main.yml | 3 +++ 6 files changed, 37 insertions(+) diff --git a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml index 1e00a202189..64e5d37c2b6 100644 --- a/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml +++ b/playbooks/roles/aws_cloudwatch_agent/defaults/main.yml @@ -11,6 +11,10 @@ cloudwatch_url: https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/lat # Default cloudwatch namespace cloudwatch_namespace: Analytics/Monitor +# Populate the cloudwatch_procstat_patterns with patterns that you want to pass to the procstat config. +# If the list is empty, then the procstat is not enabled. +cloudwatch_procstat_patterns: [] + # Collectd installation parameters collectd_version: "5.9.2.g-1ubuntu5" collectd_install_recommends: yes diff --git a/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml index 6353adbdec7..243e8a7b7d9 100644 --- a/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml +++ b/playbooks/roles/aws_cloudwatch_agent/tasks/main.yml @@ -8,6 +8,7 @@ tags: - install - install:base + - install:cloudwatch - name: Install xz-utils (required when using deb parameter of apt module) apt: @@ -16,6 +17,7 @@ tags: - install - install:base + - install:cloudwatch - name: Download the AWS CloudWatch Agent Debian package get_url: @@ -24,6 +26,7 @@ tags: - install - install:base + - install:cloudwatch - name: Install AWS CloudWatch Agent Debian package apt: @@ -32,6 +35,7 @@ tags: - install - install:base + - install:cloudwatch - name: Copy amazon-cloudwatch-agent template template: @@ -43,6 +47,7 @@ tags: - install - install:base + - install:cloudwatch - name: Enable AWS CloudWatch Agent service: @@ -51,6 +56,7 @@ tags: - install - install:base + - install:cloudwatch - name: Run AWS CloudWatch Agent shell: "/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json" @@ -58,3 +64,4 @@ tags: - install - install:code + - install:cloudwatch diff --git a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 index f2a7307b941..4662d8ea2c7 100644 --- a/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 +++ b/playbooks/roles/aws_cloudwatch_agent/templates/amazon-cloudwatch-agent.json.j2 @@ -9,6 +9,24 @@ "InstanceId": "${aws:InstanceId}" }, "metrics_collected": { + {% if cloudwatch_procstat_patterns -%} + "procstat": [ + {%- for procstat_pattern in cloudwatch_procstat_patterns -%} + { + "pattern": "{{ procstat_pattern }}", + "measurement": [ + "cpu_time", + "cpu_time_system", + "cpu_time_user" + ] + {%- if not loop.last -%} + }, + {%- else -%} + } + {% endif -%} + {% endfor -%} + ], + {% endif -%} "collectd": { "metrics_aggregation_interval": 60 }, diff --git a/playbooks/roles/dbt_docs_nginx/defaults/main.yml b/playbooks/roles/dbt_docs_nginx/defaults/main.yml index e69de29bb2d..56770b09eec 100644 --- a/playbooks/roles/dbt_docs_nginx/defaults/main.yml +++ b/playbooks/roles/dbt_docs_nginx/defaults/main.yml @@ -0,0 +1,2 @@ +# Populate the cloudwatch_procstat_patterns with patterns that you want to pass to the procstat config. +cloudwatch_procstat_patterns: ['nginx', 'cloudwatch-agent'] diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index ac6ca73d54d..f5aaf080599 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -254,3 +254,6 @@ jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" # use ZZ for Jenkins < 2.222.x # use XX for Jenkins >= 2.222.x de_jenkins_timestamper_system_time: "''HH:mm:ssZZ' '" + +# Populate the cloudwatch_procstat_patterns with patterns that you want to pass to the procstat config. +cloudwatch_procstat_patterns: ['nginx', 'jenkins.war', 'cloudwatch-agent'] diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 49848704e00..f9e68e97cc2 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -299,3 +299,6 @@ jenkins_public_keyfile: "{{ jenkins_private_keyfile }}.pub" # use ZZ for Jenkins < 2.222.x # use XX for Jenkins >= 2.222.x de_jenkins_timestamper_system_time: "''HH:mm:ssXX' '" + +# Populate the cloudwatch_procstat_patterns with patterns that you want to pass to the procstat config. +cloudwatch_procstat_patterns: ['nginx', 'jenkins.war', 'cloudwatch-agent'] From 64e5f679060228705f12bbd748b62673ca41c162 Mon Sep 17 00:00:00 2001 From: Kaleb Davenport Date: Wed, 6 Jul 2022 17:12:31 -0400 Subject: [PATCH 318/664] chore: add PROSPECTUS_ALGOLIA_ADMIN_KEY variable to environment This change allows future builds using the PROSPECTUS_ALGOLIA_ADMIN_KEY to build properly using updated key name --- playbooks/roles/prospectus/defaults/main.yml | 1 + playbooks/roles/prospectus/templates/.env.environment.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 25e484118da..b7a9183e19e 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -23,6 +23,7 @@ PROSPECTUS_CONTENTFUL_SPACE_ID: 'fake-key' PROSPECTUS_CONTENTFUL_ACCESS_TOKEN: 'fake-key' PROSPECTUS_SEGMENT_WRITE_KEY: 'fake-key' PROSPECTUS_ALGOLIA_BROWSE_KEY: 'fake-key' +PROSPECTUS_ALGOLIA_ADMIN_KEY: 'fake-key' # nginx vars PROSPECTUS_DATA_DIR: '/edx/var/prospectus' diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index ccb2ead6cab..4dda25acc96 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -10,3 +10,4 @@ GATSBY_CONTENTFUL_SPACE_ID={{ PROSPECTUS_CONTENTFUL_SPACE_ID }} GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} +PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} From 1089a3e1c8d2540af518c4524d679becea7268b9 Mon Sep 17 00:00:00 2001 From: Maxim Beder Date: Thu, 7 Jul 2022 12:33:40 +0200 Subject: [PATCH 319/664] fix: don't create user for edxapp when service name is specified When creating a django user for IDA (i.e. when service name is specified), we only want to create it for that IDA, so we add a predicate that checks that service name is empty for the tasks that create django users in edxapp. --- playbooks/manage_edxapp_users_and_groups.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index a0dd9c0abf2..2641b4e3e7f 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -129,7 +129,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (not group_environment) or group_environment in item.environments + when: (not service) and (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -142,7 +142,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (not group_environment) or group_environment in item.environments + when: (not service) and (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -159,7 +159,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: not item.get('unusable_password') + when: (not service) and (not item.get('unusable_password')) register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 @@ -180,7 +180,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: item.get('unusable_password') + when: (not service) and item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 From 732ac04e35f9eec95460224bde6e30c98c01d28d Mon Sep 17 00:00:00 2001 From: Maxim Beder Date: Thu, 7 Jul 2022 12:41:01 +0200 Subject: [PATCH 320/664] fix: try to create django users for IDAs only when service name is specified When creating a django user for edxapp and not for IDA (i.e. when service name is not specified), we don't want the tasks that create django users in IDAs to be executed, because they don't pass `lms` and/or `cms` to `manage.py`, so these tasks fail for edxapp. To fix that we add a predicate that checks that service name has been specified to these tasks, so they only run when creating a django user for an IDA. --- playbooks/manage_edxapp_users_and_groups.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 2641b4e3e7f..fcf15727b43 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -197,7 +197,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (not group_environment) or group_environment in item.environments + when: (service | length > 0) and (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -214,7 +214,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: not item.get('unusable_password') + when: (service | length > 0) and not item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 @@ -235,7 +235,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: item.get('unusable_password') + when: (service | length > 0) and item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 From 451eb73e3020b6e6c8a5c3681854493d9d9a1e7e Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Thu, 14 Apr 2022 12:09:59 +0530 Subject: [PATCH 321/664] refactor: add CELERY_RESULT_BACKEND with default django-cache --- CHANGELOG.md | 5 +++++ docker/build/edxapp/lms.yml | 1 + docker/build/edxapp/studio.yml | 1 + playbooks/roles/edxapp/defaults/main.yml | 2 ++ 4 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe09ad67828..e6c1bfb6706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-06-08 + - Role: edxapp + - Added a new `CELERY_RESULT_BACKEND` setting to allow operators to + override the default celery result backend. + - 2022-06-06 - Role: common - Remove PPA for `watchman` as we have shifted to Ubuntu 20.04, diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index fe809bb5513..9463740bac5 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -108,6 +108,7 @@ CELERY_BROKER_USE_SSL: false CELERY_BROKER_VHOST: '' CELERY_EVENT_QUEUE_TTL: null CELERY_TIMEZONE: UTC +CELERY_RESULT_BACKEND: django-cache CERTIFICATE_TEMPLATE_LANGUAGES: en: English es: Español diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 037e80ed31e..0339423054a 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -89,6 +89,7 @@ CELERY_BROKER_USE_SSL: false CELERY_BROKER_VHOST: '' CELERY_EVENT_QUEUE_TTL: null CELERY_TIMEZONE: UTC +CELERY_RESULT_BACKEND: django-cache CERTIFICATE_TEMPLATE_LANGUAGES: en: English es: Español diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 04293f2abe6..d1048eb809a 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -190,6 +190,7 @@ EDXAPP_CELERY_BROKER_USE_SSL: false EDXAPP_CELERY_EVENT_QUEUE_TTL: !!null EDXAPP_CELERY_TIMEZONE: "UTC" EDXAPP_CELERYBEAT_SCHEDULER: "celery.beat:PersistentScheduler" +EDXAPP_CELERY_RESULT_BACKEND: "django-cache" EDXAPP_ENABLE_CELERY_BEAT: false EDXAPP_SINGLE_BEAT_LOCK_TIME: 60 # EDXAPP_SINGLE_BEAT_HEARTBEAT_INTERVAL must be smaller than EDXAPP_SINGLE_BEAT_LOCK_TIME / 2 @@ -1345,6 +1346,7 @@ generic_env_config: &edxapp_generic_env CELERY_BROKER_VHOST: "{{ EDXAPP_CELERY_BROKER_VHOST }}" CELERY_BROKER_USE_SSL: "{{ EDXAPP_CELERY_BROKER_USE_SSL }}" CELERY_EVENT_QUEUE_TTL: "{{ EDXAPP_CELERY_EVENT_QUEUE_TTL }}" + CELERY_RESULT_BACKEND: "{{ EDXAPP_CELERY_RESULT_BACKEND }}" PAYMENT_SUPPORT_EMAIL: "{{ EDXAPP_PAYMENT_SUPPORT_EMAIL }}" ZENDESK_URL: "{{ EDXAPP_ZENDESK_URL }}" ZENDESK_CUSTOM_FIELDS: "{{ EDXAPP_ZENDESK_CUSTOM_FIELDS }}" From 9d7482c0b13eb14583d5fcff2720778986a8ff64 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Sat, 23 Jul 2022 11:06:59 -0400 Subject: [PATCH 322/664] fix: Install pip and pip-tools in upgrade script (#6770) * fix: Install pip and pip-tools in upgrade script * fix: upgrade Co-authored-by: Jawayria --- Makefile | 11 +++++++- .../roles/aws/templates/requirements.txt.j2 | 10 ++++---- requirements.txt | 12 ++++----- requirements/common_constraints.txt | 25 +++++++++++++++++++ requirements/constraints.txt | 2 ++ requirements/jenkins.in | 2 +- requirements/pip-tools.in | 1 + requirements/pip-tools.txt | 14 ++++++++--- requirements/pip.in | 7 ++++++ requirements/pip.txt | 16 ++++++++++++ util/jenkins/requirements-cloudflare.txt | 4 +-- util/pingdom/requirements.txt | 4 +-- util/vpc-tools/requirements.txt | 4 +-- 13 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 requirements/common_constraints.txt create mode 100644 requirements/pip.in create mode 100644 requirements/pip.txt diff --git a/Makefile b/Makefile index c642ea22cd8..f3918dc8869 100755 --- a/Makefile +++ b/Makefile @@ -20,12 +20,21 @@ requirements: pip install -qr pre-requirements.txt --exists-action w pip install -qr requirements.txt --exists-action w +COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt +.PHONY: $(COMMON_CONSTRAINTS_TXT) +$(COMMON_CONSTRAINTS_TXT): + wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)" + upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints +upgrade: $(COMMON_CONSTRAINTS_TXT) + ## update the pip requirements files to use the latest releases satisfying our constraints pip install -qr pre-requirements.txt --exists-action w pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! + pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + pip install -qr requirements/pip.txt + pip install -qr requirements/pip-tools.txt pip-compile --upgrade -o requirements.txt requirements/base.in pip-compile --upgrade -o playbooks/roles/aws/templates/requirements.txt.j2 requirements/aws.in pip-compile --upgrade -o util/elasticsearch/requirements.txt requirements/elasticsearch.in diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 5fb598be16f..9249c77305c 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.25.0 +awscli==1.25.18 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.24.0 +boto3==1.24.18 # via -r requirements/aws.in -botocore==1.27.0 +botocore==1.27.18 # via # awscli # boto3 @@ -19,7 +19,7 @@ colorama==0.4.4 # via awscli docutils==0.16 # via awscli -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -29,7 +29,7 @@ python-dateutil==2.8.2 # via # botocore # s3cmd -python-magic==0.4.26 +python-magic==0.4.27 # via s3cmd pyyaml==5.3.1 # via diff --git a/requirements.txt b/requirements.txt index ec54e8f6c72..3cbed8b3238 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.9.0 # via -r requirements/base.in -awscli==1.25.0 +awscli==1.25.18 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,14 +14,14 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.24.0 +boto3==1.24.18 # via -r requirements/base.in -botocore==1.27.0 +botocore==1.27.18 # via # awscli # boto3 # s3transfer -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests cffi==1.15.0 # via @@ -54,7 +54,7 @@ jinja2==2.8 # via # -r requirements/base.in # ansible -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -91,7 +91,7 @@ pyyaml==5.4.1 # -r requirements/base.in # ansible # awscli -requests==2.27.1 +requests==2.28.0 # via # -r requirements/base.in # datadog diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt new file mode 100644 index 00000000000..cca3ccd308c --- /dev/null +++ b/requirements/common_constraints.txt @@ -0,0 +1,25 @@ +# A central location for most common version constraints +# (across edx repos) for pip-installation. +# +# Similar to other constraint files this file doesn't install any packages. +# It specifies version constraints that will be applied if a package is needed. +# When pinning something here, please provide an explanation of why it is a good +# idea to pin this package across all edx repos, Ideally, link to other information +# that will help people in the future to remove the pin when possible. +# Writing an issue against the offending project and linking to it here is good. +# +# Note: Changes to this file will automatically be used by other repos, referencing +# this file from Github directly. It does not require packaging in edx-lint. + + +# using LTS django version +Django<4.0 + +# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. +# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html +elasticsearch<7.14.0 + +setuptools<60 + +# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected +django-simple-history==3.0.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 340665643fe..aaa989ce567 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -1 +1,3 @@ +-c common_constraints.txt + bcrypt<3.2.0 # 3.2.0 dropped support for python 2.7 diff --git a/requirements/jenkins.in b/requirements/jenkins.in index c155d37fa1a..f69c8fe2cea 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -13,4 +13,4 @@ opsgenie-sdk==0.3.1 PyMySQL==0.9.3 python-gnupg redis==2.10.6 -splunk-sdk==1.6.6 +splunk-sdk==1.6.16 # older versions have been yanked diff --git a/requirements/pip-tools.in b/requirements/pip-tools.in index c17d0b41e75..3f1b64ae937 100644 --- a/requirements/pip-tools.in +++ b/requirements/pip-tools.in @@ -1,3 +1,4 @@ # Just the dependencies to run pip-tools, mainly for the "upgrade" make target +-c constraints.txt pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ce56f4c7f38..97f4ed93dc9 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,14 +4,22 @@ # # make upgrade # +build==0.8.0 + # via pip-tools click==8.1.3 # via pip-tools +packaging==21.3 + # via build pep517==0.12.0 - # via pip-tools -pip-tools==6.6.2 + # via build +pip-tools==6.7.0 # via -r requirements/pip-tools.in +pyparsing==3.0.9 + # via packaging tomli==2.0.1 - # via pep517 + # via + # build + # pep517 wheel==0.37.1 # via pip-tools diff --git a/requirements/pip.in b/requirements/pip.in new file mode 100644 index 00000000000..715478cdc0c --- /dev/null +++ b/requirements/pip.in @@ -0,0 +1,7 @@ +-c constraints.txt +# Core dependencies for installing other packages + +pip +setuptools +wheel + diff --git a/requirements/pip.txt b/requirements/pip.txt new file mode 100644 index 00000000000..8a667c4a29e --- /dev/null +++ b/requirements/pip.txt @@ -0,0 +1,16 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +wheel==0.37.1 + # via -r requirements/pip.in + +# The following packages are considered to be unsafe in a requirements file: +pip==22.1.2 + # via -r requirements/pip.in +setuptools==59.8.0 + # via + # -c requirements/common_constraints.txt + # -r requirements/pip.in diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index bbfeaaa1cb3..a3ee4e40405 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -12,7 +12,7 @@ click==8.1.3 # via -r requirements/cloudflare.in idna==3.3 # via requests -requests==2.27.1 +requests==2.28.0 # via -r requirements/cloudflare.in urllib3==1.26.9 # via requests diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 54560ad3e33..f2befe795ab 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -14,7 +14,7 @@ idna==3.3 # via requests pyyaml==6.0 # via -r requirements/pingdom.in -requests==2.27.1 +requests==2.28.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 6692bd4ad7a..e3fca741449 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,7 +6,7 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -14,7 +14,7 @@ docopt==0.6.2 # via -r requirements/vpc-tools.in idna==3.3 # via requests -requests==2.27.1 +requests==2.28.0 # via -r requirements/vpc-tools.in urllib3==1.26.9 # via requests From ab66296fd4127cb2361b92aa89b8a1c0c2e9f69c Mon Sep 17 00:00:00 2001 From: jansenk Date: Mon, 25 Jul 2022 14:28:34 -0400 Subject: [PATCH 323/664] temp: single learner course regrade queue --- docker/build/edxapp/lms.yml | 1 + docker/build/edxapp/studio.yml | 1 + playbooks/roles/edxapp/defaults/main.yml | 2 ++ 3 files changed, 4 insertions(+) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 9463740bac5..70983473f39 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -442,6 +442,7 @@ PLATFORM_FACEBOOK_ACCOUNT: http://www.facebook.com/YourPlatformFacebookAccount PLATFORM_NAME: Your Platform Name Here PLATFORM_TWITTER_ACCOUNT: '@YourPlatformTwitterAccount' POLICY_CHANGE_GRADES_ROUTING_KEY: edx.lms.core.default +SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY: edx.lms.core.default PRESS_EMAIL: press@example.com PROCTORING_BACKENDS: DEFAULT: 'null' diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 0339423054a..5e1e4443698 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -377,6 +377,7 @@ PLATFORM_FACEBOOK_ACCOUNT: http://www.facebook.com/YourPlatformFacebookAccount PLATFORM_NAME: Your Platform Name Here PLATFORM_TWITTER_ACCOUNT: '@YourPlatformTwitterAccount' POLICY_CHANGE_GRADES_ROUTING_KEY: edx.lms.core.default +SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY: edx.lms.core.default PRESS_EMAIL: press@example.com PROCTORING_BACKENDS: DEFAULT: 'null' diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index d1048eb809a..65c768b404e 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -715,6 +715,7 @@ EDXAPP_CELERY_WORKERS: prefetch_optimization: default EDXAPP_RECALCULATE_GRADES_ROUTING_KEY: 'edx.lms.core.default' EDXAPP_POLICY_CHANGE_GRADES_ROUTING_KEY: 'edx.lms.core.default' +EDXAPP_SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY: 'edx.lms.core.default' EDXAPP_BULK_EMAIL_ROUTING_KEY_SMALL_JOBS: 'edx.lms.core.default' EDXAPP_PROGRAM_CERTIFICATES_ROUTING_KEY: 'edx.lms.core.default' EDXAPP_LMS_CELERY_QUEUES: "{{ edxapp_workers|selectattr('service_variant', 'equalto', 'lms')|map(attribute='queue')|map('regex_replace', '^(.*)$', 'edx.lms.core.\\1')|list }}" @@ -1527,6 +1528,7 @@ generic_env_config: &edxapp_generic_env BASE_COOKIE_DOMAIN: "{{ EDXAPP_BASE_COOKIE_DOMAIN }}" POLICY_CHANGE_GRADES_ROUTING_KEY: "{{ EDXAPP_POLICY_CHANGE_GRADES_ROUTING_KEY }}" + SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY: "{{ EDXAPP_SINGLE_LEARNER_COURSE_REGRADE_ROUTING_KEY }}" PROCTORING_SETTINGS: "{{ EDXAPP_PROCTORING_SETTINGS }}" EXTRA_MIDDLEWARE_CLASSES: "{{ EDXAPP_EXTRA_MIDDLEWARE_CLASSES }}" MAINTENANCE_BANNER_TEXT: "{{ EDXAPP_MAINTENANCE_BANNER_TEXT }}" From 82166acd472566599c9adff1c639975425af2dc3 Mon Sep 17 00:00:00 2001 From: jansenk Date: Mon, 25 Jul 2022 14:30:31 -0400 Subject: [PATCH 324/664] docs: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6c1bfb6706..444110d661a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2022-07-25 + - Add new routing key for individual learner course regrade queue - 2022-06-08 - Role: edxapp - Added a new `CELERY_RESULT_BACKEND` setting to allow operators to From 1e3fd0bc4eb084482cffdd23bc941a62f105b89c Mon Sep 17 00:00:00 2001 From: Kaustav Banerjee Date: Wed, 3 Aug 2022 12:29:27 +0530 Subject: [PATCH 325/664] feat: add default analytics api elasticsearch index alias --- playbooks/roles/analytics_api/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index 16595bd7e13..8bee6c37942 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -103,6 +103,7 @@ ANALYTICS_DB_CONFIG_ALIAS: 'reports' ANALYTICS_API_ELASTICSEARCH_LEARNERS_HOST: 'localhost' ANALYTICS_API_ELASTICSEARCH_LEARNERS_INDEX: 'roster_1_2' +ANALYTICS_API_ELASTICSEARCH_LEARNERS_INDEX_ALIAS: 'roster_1_2' ANALYTICS_API_ELASTICSEARCH_LEARNERS_UPDATE_INDEX: 'index_updates' ANALYTICS_API_ELASTICSEARCH_CONNECTION_CLASS: !!null ANALYTICS_API_ELASTICSEARCH_AWS_ACCESS_KEY_ID: !!null @@ -161,6 +162,7 @@ analytics_api_service_config_overrides: DATABASES: '{{ ANALYTICS_API_DATABASES }}' ELASTICSEARCH_LEARNERS_HOST: '{{ ANALYTICS_API_ELASTICSEARCH_LEARNERS_HOST }}' ELASTICSEARCH_LEARNERS_INDEX: '{{ ANALYTICS_API_ELASTICSEARCH_LEARNERS_INDEX }}' + ELASTICSEARCH_LEARNERS_INDEX_ALIAS: '{{ ANALYTICS_API_ELASTICSEARCH_LEARNERS_INDEX_ALIAS }}' ELASTICSEARCH_LEARNERS_UPDATE_INDEX: '{{ ANALYTICS_API_ELASTICSEARCH_LEARNERS_UPDATE_INDEX }}' ELASTICSEARCH_CONNECTION_CLASS: '{{ ANALYTICS_API_ELASTICSEARCH_CONNECTION_CLASS }}' ELASTICSEARCH_AWS_ACCESS_KEY_ID: '{{ ANALYTICS_API_ELASTICSEARCH_AWS_ACCESS_KEY_ID }}' From 04532a3fbd565c4c6a954853210ffc6354a81d96 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Fri, 5 Aug 2022 06:35:59 -0400 Subject: [PATCH 326/664] chore: Update the default retirement states to add new ones --- playbooks/roles/edxapp/defaults/main.yml | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 65c768b404e..afcc75e8a4c 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -867,6 +867,32 @@ EDXAPP_RETIRED_USER_SALTS: # These are the required states, environmental overrides are in edx-internal. EDXAPP_RETIREMENT_STATES: - "PENDING" + - "RETIRING_FORUMS" + - "FORUMS_COMPLETE" + - "RETIRING_SALESFORCE_LEADS" + - "SALESFORCE_LEADS_COMPLETE" + - "RETIRING_SEGMENT" + - "SEGMENT_COMPLETE" + - "RETIRING_HUBSPOT" + - "HUBSPOT_COMPLETE" + - "RETIRING_BRAZE" + - "BRAZE_COMPLETE" + - "RETIRING_ENROLLMENTS" + - "ENROLLMENTS_COMPLETE" + - "RETIRING_NOTES" + - "NOTES_COMPLETE" + - "RETIRING_PROCTORING" + - "PROCTORING_COMPLETE" + - "RETIRING_DEMOGRAPHICS" + - "DEMOGRAPHICS_COMPLETE" + - "RETIRING_LICENSE_MANAGER" + - "LICENSE_MANAGER_COMPLETE" + - "RETIRING_LMS_MISC" + - "LMS_MISC_COMPLETE" + - "RETIRING_LMS" + - "LMS_COMPLETE" + - "ADDING_TO_PARTNER_QUEUE" + - "PARTNER_QUEUE_COMPLETE" - "ERRORED" - "ABORTED" - "COMPLETE" From 7be176ab97e1d037b3c537a92dfa2f69b4a538df Mon Sep 17 00:00:00 2001 From: Alex Dusenbery Date: Fri, 5 Aug 2022 08:53:28 -0400 Subject: [PATCH 327/664] Revert "fix: try to create django users for IDAs only when service name is specified" This reverts commit 732ac04e35f9eec95460224bde6e30c98c01d28d. --- playbooks/manage_edxapp_users_and_groups.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index fcf15727b43..2641b4e3e7f 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -197,7 +197,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (service | length > 0) and (not group_environment) or group_environment in item.environments + when: (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -214,7 +214,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: (service | length > 0) and not item.get('unusable_password') + when: not item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 @@ -235,7 +235,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: (service | length > 0) and item.get('unusable_password') + when: item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 From 44670188cf9e5fd4144d7aee3163e2670c69a260 Mon Sep 17 00:00:00 2001 From: Alex Dusenbery Date: Fri, 5 Aug 2022 08:53:37 -0400 Subject: [PATCH 328/664] Revert "fix: don't create user for edxapp when service name is specified" This reverts commit 1089a3e1c8d2540af518c4524d679becea7268b9. --- playbooks/manage_edxapp_users_and_groups.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 2641b4e3e7f..a0dd9c0abf2 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -129,7 +129,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (not service) and (not group_environment) or group_environment in item.environments + when: (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -142,7 +142,7 @@ {% if item.get('permissions', []) | length %}--permissions {{ item.permissions | default([]) | map('quote') | join(' ') }}{% endif %} {% if item.get('remove') %}--remove{% endif %} with_items: "{{ django_groups }}" - when: (not service) and (not group_environment) or group_environment in item.environments + when: (not group_environment) or group_environment in item.environments become: true become_user: "{{ common_web_user }}" @@ -159,7 +159,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: (not service) and (not item.get('unusable_password')) + when: not item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 @@ -180,7 +180,7 @@ {% if item.get('unusable_password') %}--unusable-password{% endif %} {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} with_items: "{{ django_users }}" - when: (not service) and item.get('unusable_password') + when: item.get('unusable_password') register: manage_users_result failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) retries: 3 From e46b7776464145a2ce935dd9e7955c8e5fae14d5 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 24 Aug 2022 12:35:43 +0500 Subject: [PATCH 329/664] feat: add commerce coordinator ansible configs for sandbox --- playbooks/commerce_coordinator.yml | 22 +++ playbooks/edx_continuous_integration.yml | 2 + .../commerce_coordinator/defaults/main.yml | 168 ++++++++++++++++++ .../roles/commerce_coordinator/meta/main.yml | 56 ++++++ .../roles/commerce_coordinator/tasks/main.yml | 23 +++ playbooks/roles/edxapp/defaults/main.yml | 8 + playbooks/roles/edxlocal/defaults/main.yml | 6 + playbooks/roles/launch_ec2/tasks/main.yml | 2 +- .../oauth_client_setup/defaults/main.yml | 10 ++ util/jenkins/ansible-provision.sh | 19 +- 10 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 playbooks/commerce_coordinator.yml create mode 100644 playbooks/roles/commerce_coordinator/defaults/main.yml create mode 100644 playbooks/roles/commerce_coordinator/meta/main.yml create mode 100644 playbooks/roles/commerce_coordinator/tasks/main.yml diff --git a/playbooks/commerce_coordinator.yml b/playbooks/commerce_coordinator.yml new file mode 100644 index 00000000000..5968f528ff8 --- /dev/null +++ b/playbooks/commerce_coordinator.yml @@ -0,0 +1,22 @@ +- name: Deploy edX Commerce Coordinator + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: True + CLUSTER_NAME: 'commerce_coordinator' + REGISTRAR_ENABLED: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - role: nginx + nginx_default_sites: + - commerce_coordinator + - commerce_coordinator + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: hermes + HERMES_TARGET_SERVICE: 'commerce_coordinator' + when: COMMERCE_COORDINATOR__HERMES_ENABLED diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 77f381277a5..abc1ded71d1 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -41,6 +41,8 @@ when: REGISTRAR_ENABLED - role: license_manager when: LICENSE_MANAGER_ENABLED + - role: commerce_coordinator + when: COMMERCE_COORDINATOR_ENABLED - role: enterprise_catalog when: ENTERPRISE_CATALOG_ENABLED - role: learner_portal diff --git a/playbooks/roles/commerce_coordinator/defaults/main.yml b/playbooks/roles/commerce_coordinator/defaults/main.yml new file mode 100644 index 00000000000..9069ec1c11d --- /dev/null +++ b/playbooks/roles/commerce_coordinator/defaults/main.yml @@ -0,0 +1,168 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +## +# Defaults for role commerce_coordinator +# + +COMMERCE_COORDINATOR_GIT_IDENTITY: !!null + +COMMERCE_COORDINATOR_ENABLED: True + +# +# vars are namespace with the module name. +# +commerce_coordinator_service_name: 'commerce_coordinator' + +commerce_coordinator_user: "{{ commerce_coordinator_service_name }}" +commerce_coordinator_home: "{{ COMMON_APP_DIR }}/{{ commerce_coordinator_service_name }}" +commerce_coordinator_app_dir: "{{ COMMON_APP_DIR }}/{{ commerce_coordinator_service_name }}" +commerce_coordinator_code_dir: "{{ commerce_coordinator_app_dir }}/{{ commerce_coordinator_service_name }}" +commerce_coordinator_venvs_dir: "{{ commerce_coordinator_app_dir }}/venvs" +commerce_coordinator_venv_dir: "{{ commerce_coordinator_venvs_dir }}/commerce_coordinator" +commerce_coordinator_celery_default_queue: 'commerce_coordinator.default' +commerce_coordinator_hostname: "commerce-coordinator" + +COMMERCE_COORDINATOR_USE_PYTHON38: True + +COMMERCE_COORDINATOR_CELERY_ALWAYS_EAGER: false +COMMERCE_COORDINATOR_CELERY_BROKER_TRANSPORT: '' +COMMERCE_COORDINATOR_CELERY_BROKER_USER: '' +COMMERCE_COORDINATOR_CELERY_BROKER_PASSWORD: '' +COMMERCE_COORDINATOR_CELERY_BROKER_HOSTNAME: '' +COMMERCE_COORDINATOR_CELERY_BROKER_VHOST: '' + +commerce_coordinator_environment: + COMMERCE_COORDINATOR_CFG: '{{ COMMON_CFG_DIR }}/{{ commerce_coordinator_service_name }}.yml' + +commerce_coordinator_gunicorn_port: 18170 + +commerce_coordinator_debian_pkgs: [] + +COMMERCE_COORDINATOR_REPOS: + - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' + DOMAIN: '{{ COMMON_GIT_MIRROR }}' + PATH: '{{ COMMON_GIT_PATH }}' + REPO: 'commerce-coordinator.git' + VERSION: '{{ COMMERCE_COORDINATOR_VERSION }}' + DESTINATION: "{{ commerce_coordinator_code_dir }}" + SSH_KEY: '{{ COMMERCE_COORDINATOR_GIT_IDENTITY }}' + +COMMERCE_COORDINATOR_NGINX_PORT: '1{{ commerce_coordinator_gunicorn_port }}' +COMMERCE_COORDINATOR_SSL_NGINX_PORT: '4{{ commerce_coordinator_gunicorn_port }}' + +COMMERCE_COORDINATOR_DEFAULT_DB_NAME: 'commerce-coordinator' +COMMERCE_COORDINATOR_MYSQL_HOST: 'localhost' +# MySQL usernames are limited to 16 characters +COMMERCE_COORDINATOR_MYSQL_USER: 'commerce-coordinator001' +COMMERCE_COORDINATOR_MYSQL_PASSWORD: 'password' +COMMERCE_COORDINATOR_MYSQL_CONN_MAX_AGE: 60 + +COMMERCE_COORDINATOR_MEMCACHE: [ 'memcache' ] + +COMMERCE_COORDINATOR_DJANGO_SETTINGS_MODULE: 'commerce_coordinator.settings.production' +COMMERCE_COORDINATOR_DOMAIN: 'localhost' +COMMERCE_COORDINATOR_URL_ROOT: 'http://{{ COMMERCE_COORDINATOR_DOMAIN }}:{{ COMMERCE_COORDINATOR_NGINX_PORT }}' +COMMERCE_COORDINATOR_API_ROOT: '{{ COMMERCE_COORDINATOR_URL_ROOT }}/api' +COMMERCE_COORDINATOR_LOGOUT_URL: '{{ COMMERCE_COORDINATOR_URL_ROOT }}/logout/' + +COMMERCE_COORDINATOR_LANG: 'en_US.UTF-8' +COMMERCE_COORDINATOR_LANGUAGE_CODE: 'en' +COMMERCE_COORDINATOR_LANGUAGE_COOKIE_NAME: 'openedx-language-preference' + +COMMERCE_COORDINATOR_SERVICE_USER: 'commerce_coordinator_service_user' + +COMMERCE_COORDINATOR_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ commerce_coordinator_service_name }}' +COMMERCE_COORDINATOR_MEDIA_ROOT: '{{ COMMERCE_COORDINATOR_DATA_DIR }}/media' +COMMERCE_COORDINATOR_MEDIA_URL: '/api/media/' + +COMMERCE_COORDINATOR_MEDIA_STORAGE_BACKEND: + DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage' + MEDIA_ROOT: '{{ COMMERCE_COORDINATOR_MEDIA_ROOT }}' + MEDIA_URL: '{{ COMMERCE_COORDINATOR_MEDIA_URL }}' + +# TODO: Let edx_django_service manage COMMERCE_COORDINATOR_STATIC_ROOT in phase 2. +COMMERCE_COORDINATOR_STATIC_ROOT: '{{ COMMERCE_COORDINATOR_DATA_DIR }}/staticfiles' +COMMERCE_COORDINATOR_STATIC_URL: '/static/' + +COMMERCE_COORDINATOR_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage' + +COMMERCE_COORDINATOR_CORS_ORIGIN_ALLOW_ALL: false +COMMERCE_COORDINATOR_CORS_ORIGIN_WHITELIST: [] + +COMMERCE_COORDINATOR_CSRF_COOKIE_SECURE: false +COMMERCE_COORDINATOR_CSRF_TRUSTED_ORIGINS: [] + +COMMERCE_COORDINATOR_VERSION: 'main' + +COMMERCE_COORDINATOR_GUNICORN_EXTRA: '' + +COMMERCE_COORDINATOR_EXTRA_APPS: [] + +COMMERCE_COORDINATOR_SESSION_EXPIRE_AT_BROWSER_CLOSE: false + +COMMERCE_COORDINATOR_CERTIFICATE_LANGUAGES: + 'en': 'English' + 'es_419': 'Spanish' + +# Used to automatically configure OAuth2 Client +COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_KEY: 'commerce_coordinator-sso-key' +COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_SECRET: 'commerce_coordinator-sso-secret' +COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_KEY: 'commerce_coordinator-backend-service-key' +COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_SECRET: 'commerce_coordinator-backend-service-secret' +COMMERCE_COORDINATOR_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false + +# API key for segment.io +COMMERCE_COORDINATOR_SEGMENT_KEY: !!null + +COMMERCE_COORDINATOR_DISCOVERY_BASE_URL: !!null +COMMERCE_COORDINATOR_LMS_BASE_URL: !!null + +commerce_coordinator_service_config_overrides: + CERTIFICATE_LANGUAGES: '{{ COMMERCE_COORDINATOR_CERTIFICATE_LANGUAGES }}' + COMMERCE_COORDINATOR_SERVICE_USER: '{{ COMMERCE_COORDINATOR_SERVICE_USER }}' + LANGUAGE_COOKIE_NAME: '{{ COMMERCE_COORDINATOR_LANGUAGE_COOKIE_NAME }}' + SEGMENT_KEY: "{{ COMMERCE_COORDINATOR_SEGMENT_KEY }}" + DISCOVERY_BASE_URL: "{{ COMMERCE_COORDINATOR_DISCOVERY_BASE_URL }}" + LMS_BASE_URL: "{{ COMMERCE_COORDINATOR_LMS_BASE_URL }}" + CORS_ORIGIN_WHITELIST: "{{ COMMERCE_COORDINATOR_CORS_ORIGIN_WHITELIST }}" + CSRF_TRUSTED_ORIGINS: "{{ COMMERCE_COORDINATOR_CSRF_TRUSTED_ORIGINS }}" + CSRF_COOKIE_SECURE: "{{ COMMERCE_COORDINATOR_CSRF_COOKIE_SECURE }}" + CELERY_ALWAYS_EAGER: '{{ COMMERCE_COORDINATOR_CELERY_ALWAYS_EAGER }}' + CELERY_BROKER_TRANSPORT: '{{ COMMERCE_COORDINATOR_CELERY_BROKER_TRANSPORT }}' + CELERY_BROKER_USER: '{{ COMMERCE_COORDINATOR_CELERY_BROKER_USER }}' + CELERY_BROKER_PASSWORD: '{{ COMMERCE_COORDINATOR_CELERY_BROKER_PASSWORD }}' + CELERY_BROKER_HOSTNAME: '{{ COMMERCE_COORDINATOR_CELERY_BROKER_HOSTNAME }}' + CELERY_BROKER_VHOST: '{{ COMMERCE_COORDINATOR_CELERY_BROKER_VHOST }}' + CELERY_DEFAULT_EXCHANGE: 'commerce_coordinator' + CELERY_DEFAULT_ROUTING_KEY: 'commerce_coordinator' + CELERY_DEFAULT_QUEUE: '{{ commerce_coordinator_celery_default_queue }}' + +# See edx_django_service_automated_users for an example of what this should be +COMMERCE_COORDINATOR_AUTOMATED_USERS: {} + +# NOTE: These variables are only needed to create the demo site (e.g. for sandboxes) + +COMMERCE_COORDINATOR_ENABLE_NEWRELIC_DISTRIBUTED_TRACING: false + +# Remote config +COMMERCE_COORDINATOR_HERMES_ENABLED: "{{ COMMON_HERMES_ENABLED }}" + +COMMERCE_COORDINATOR_DECRYPT_CONFIG_ENABLED: "{{ COMMON_DECRYPT_CONFIG_ENABLED }}" +COMMERCE_COORDINATOR_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" + +# Worker settings +worker_django_settings_module: "{{ COMMERCE_COORDINATOR_DJANGO_SETTINGS_MODULE }}" +COMMERCE_COORDINATOR_CELERY_WORKERS: + - queue: '{{ commerce_coordinator_celery_default_queue }}' + concurrency: 1 + monitor: True +commerce_coordinator_workers: "{{ COMMERCE_COORDINATOR_CELERY_WORKERS }}" + +commerce_coordinator_post_migrate_commands: [] diff --git a/playbooks/roles/commerce_coordinator/meta/main.yml b/playbooks/roles/commerce_coordinator/meta/main.yml new file mode 100644 index 00000000000..fd5e39f102b --- /dev/null +++ b/playbooks/roles/commerce_coordinator/meta/main.yml @@ -0,0 +1,56 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +## +# Role includes for role commerce_coordinator +# +dependencies: + - role: edx_django_service + edx_django_service_use_python38: '{{ COMMERCE_COORDINATOR_USE_PYTHON38 }}' + edx_django_service_version: '{{ COMMERCE_COORDINATOR_VERSION }}' + edx_django_service_name: '{{ commerce_coordinator_service_name }}' + edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ commerce_coordinator_service_name }}' + edx_django_service_user: '{{ commerce_coordinator_user }}' + edx_django_service_config_overrides: '{{ commerce_coordinator_service_config_overrides }}' + edx_django_service_debian_pkgs_extra: '{{ commerce_coordinator_debian_pkgs }}' + edx_django_service_gunicorn_port: '{{ commerce_coordinator_gunicorn_port }}' + edx_django_service_django_settings_module: '{{ COMMERCE_COORDINATOR_DJANGO_SETTINGS_MODULE }}' + edx_django_service_environment_extra: '{{ commerce_coordinator_environment }}' + edx_django_service_gunicorn_extra: '{{ COMMERCE_COORDINATOR_GUNICORN_EXTRA }}' + edx_django_service_nginx_port: '{{ COMMERCE_COORDINATOR_NGINX_PORT }}' + edx_django_service_ssl_nginx_port: '{{ COMMERCE_COORDINATOR_SSL_NGINX_PORT }}' + edx_django_service_language_code: '{{ COMMERCE_COORDINATOR_LANGUAGE_CODE }}' + edx_django_service_secret_key: '{{ COMMERCE_COORDINATOR_SECRET_KEY }}' + edx_django_service_media_storage_backend: '{{ COMMERCE_COORDINATOR_MEDIA_STORAGE_BACKEND }}' + edx_django_service_staticfiles_storage: '{{ COMMERCE_COORDINATOR_STATICFILES_STORAGE }}' + edx_django_service_memcache: '{{ COMMERCE_COORDINATOR_MEMCACHE }}' + edx_django_service_default_db_host: '{{ COMMERCE_COORDINATOR_MYSQL_HOST }}' + edx_django_service_default_db_name: '{{ COMMERCE_COORDINATOR_DEFAULT_DB_NAME }}' + edx_django_service_default_db_atomic_requests: false + edx_django_service_db_user: '{{ COMMERCE_COORDINATOR_MYSQL_USER }}' + edx_django_service_db_password: '{{ COMMERCE_COORDINATOR_MYSQL_PASSWORD }}' + edx_django_service_default_db_conn_max_age: '{{ COMMERCE_COORDINATOR_MYSQL_CONN_MAX_AGE }}' + edx_django_service_extra_apps: '{{ COMMERCE_COORDINATOR_EXTRA_APPS }}' + edx_django_service_session_expire_at_browser_close: '{{ COMMERCE_COORDINATOR_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' + edx_django_service_social_auth_edx_oauth2_key: '{{ COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_KEY }}' + edx_django_service_social_auth_edx_oauth2_secret: '{{ COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_SECRET }}' + edx_django_service_backend_service_edx_oauth2_key: '{{ COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_KEY }}' + edx_django_service_backend_service_edx_oauth2_secret: '{{ COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_SECRET }}' + edx_django_service_automated_users: '{{ COMMERCE_COORDINATOR_AUTOMATED_USERS }}' + edx_django_service_cors_whitelist: '{{ COMMERCE_COORDINATOR_CORS_ORIGIN_WHITELIST }}' + edx_django_service_post_migrate_commands: '{{ commerce_coordinator_post_migrate_commands }}' + edx_django_service_enable_newrelic_distributed_tracing: '{{ COMMERCE_COORDINATOR_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}' + edx_django_service_api_root: '{{ COMMERCE_COORDINATOR_API_ROOT }}' + edx_django_service_decrypt_config_enabled: '{{ COMMERCE_COORDINATOR_DECRYPT_CONFIG_ENABLED }}' + edx_django_service_copy_config_enabled: '{{ COMMERCE_COORDINATOR_COPY_CONFIG_ENABLED }}' + edx_django_service_migration_check_services: '{{ commerce_coordinator_service_name }},{{ commerce_coordinator_service_name }}-workers' + edx_django_service_enable_celery_workers: true + edx_django_service_workers: '{{ commerce_coordinator_workers }}' + edx_django_service_repos: '{{ COMMERCE_COORDINATOR_REPOS }}' + edx_django_service_hostname: '~^((stage|prod)-)?{{ commerce_coordinator_hostname }}.*' diff --git a/playbooks/roles/commerce_coordinator/tasks/main.yml b/playbooks/roles/commerce_coordinator/tasks/main.yml new file mode 100644 index 00000000000..8ebcd1a0080 --- /dev/null +++ b/playbooks/roles/commerce_coordinator/tasks/main.yml @@ -0,0 +1,23 @@ +--- +# +# edX Configuration +# +# github: https://github.com/edx/configuration +# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS +# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions +# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# +# +# +# Tasks for role commerce_coordinator +# +# Overview: This role's tasks come from edx_django_service. +# +# +# Dependencies: +# +# +# Example play: +# +# + diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index afcc75e8a4c..9fd0c644af0 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1025,6 +1025,10 @@ DESIGNER_SERVICE_USER_NAME: "designer_worker" LICENSE_MANAGER_SERVICE_USER_EMAIL: "license_manager_worker@example.com" LICENSE_MANAGER_SERVICE_USER_NAME: "license_manager_worker" +# Configuration needed for LMS to communicate with the Commerce Coordinator service +COMMERCE_COORDINATOR_SERVICE_USER_EMAIL: "commerce_coordinator_worker@example.com" +COMMERCE_COORDINATOR_SERVICE_USER_NAME: "commerce_coordinator_worker" + ENTERPRISE_CATALOG_SERVICE_USER_EMAIL: "enterprise_catalog_worker@example.com" ENTERPRISE_CATALOG_SERVICE_USER_NAME: "enterprise_catalog_worker" @@ -1868,6 +1872,10 @@ SERVICE_WORKER_USERS: username: "{{ LICENSE_MANAGER_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ COMMERCE_COORDINATOR_SERVICE_USER_EMAIL }}" + username: "{{ COMMERCE_COORDINATOR_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ ENTERPRISE_CATALOG_SERVICE_USER_EMAIL }}" username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME }}" is_staff: true diff --git a/playbooks/roles/edxlocal/defaults/main.yml b/playbooks/roles/edxlocal/defaults/main.yml index 509ff28d6ce..afa5ff878eb 100644 --- a/playbooks/roles/edxlocal/defaults/main.yml +++ b/playbooks/roles/edxlocal/defaults/main.yml @@ -19,6 +19,7 @@ edxlocal_databases: - "{{ REGISTRAR_DEFAULT_DB_NAME | default(None) }}" - "{{ LICENSE_MANAGER_DEFAULT_DB_NAME | default(None) }}" - "{{ ENTERPRISE_CATALOG_DEFAULT_DB_NAME | default(None) }}" + - "{{ COMMERCE_COORDINATOR_DEFAULT_DB_NAME | default(None) }}" edxlocal_database_users: - { @@ -91,3 +92,8 @@ edxlocal_database_users: user: "{{ ENTERPRISE_CATALOG_MYSQL_USER | default(None) }}", pass: "{{ ENTERPRISE_CATALOG_MYSQL_PASSWORD | default(None) }}" } + - { + db: "{{ COMMERCE_COORDINATOR_DEFAULT_DB_NAME | default(None) }}", + user: "{{ COMMERCE_COORDINATOR_MYSQL_USER | default(None) }}", + pass: "{{ COMMERCE_COORDINATOR_MYSQL_PASSWORD | default(None) }}" + } diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index 06c79840053..d4975b76308 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,7 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator'] - name: Add new instance to host group local_action: diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 5d0e583479e..3906b24b31f 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -122,6 +122,16 @@ oauth_client_setup_oauth2_clients: logout_uri: "{{ ENTERPRISE_CATALOG_LOGOUT_URL | default('None') }}", username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME | default('None') }}", } + - { + name: "{{ commerce_coordinator_service_name | default('None') }}", + url_root: "{{ COMMERCE_COORDINATOR_URL_ROOT | default('None') }}", + sso_id: "{{ COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_KEY | default('None') }}", + sso_secret: "{{ COMMERCE_COORDINATOR_SOCIAL_AUTH_EDX_OAUTH2_SECRET | default('None') }}", + backend_service_id: "{{ COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_KEY | default('None') }}", + backend_service_secret: "{{ COMMERCE_COORDINATOR_BACKEND_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", + logout_uri: "{{ COMMERCE_COORDINATOR_LOGOUT_URL | default('None') }}", + username: "{{ COMMERCE_COORDINATOR_SERVICE_USER_NAME | default('None') }}", + } # # OS packages # diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 2419b3ffd95..2ac24205b99 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -186,6 +186,14 @@ if [[ -z $license_manager_version ]]; then LICENSE_MANAGER_VERSION="master" fi +if [[ -z $commerce_coordinator ]]; then + commerce_coordinator="false" +fi + +if [[ -z $commerce_coordinator_version ]]; then + COMMERCE_COORDINATOR_VERSION="master" +fi + if [[ -z $enterprise_catalog_version ]]; then ENTERPRISE_CATALOG_VERSION="master" fi @@ -330,6 +338,13 @@ LICENSE_MANAGER_ENABLED: $license_manager LICENSE_MANAGER_DECRYPT_CONFIG_ENABLED: true LICENSE_MANAGER_COPY_CONFIG_ENABLED: true +COMMERCE_COORDINATOR_NGINX_PORT: 80 +COMMERCE_COORDINATOR_SSL_NGINX_PORT: 443 +COMMERCE_COORDINATOR_VERSION: $commerce_coordinator_version +COMMERCE_COORDINATOR_ENABLED: $commerce_coordinator +COMMERCE_COORDINATOR_DECRYPT_CONFIG_ENABLED: true +COMMERCE_COORDINATOR_COPY_CONFIG_ENABLED: true + ENTERPRISE_CATALOG_NGINX_PORT: 80 ENTERPRISE_CATALOG_SSL_NGINX_PORT: 443 ENTERPRISE_CATALOG_VERSION: $enterprise_catalog_version @@ -486,12 +501,14 @@ VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master} LICENSE_MANAGER_URL_ROOT: "https://license-manager-${deploy_host}" +COMMERCE_COORDINATOR_URL_ROOT: "https://commerce-coordinator-${deploy_host}" + ENTERPRISE_CATALOG_URL_ROOT: "https://enterprise-catalog-${deploy_host}" EOF fi -encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api discovery credentials registrar edx_notes_api license_manager) +encrypted_config_apps=(edxapp ecommerce ecommerce_worker analytics_api discovery credentials registrar edx_notes_api license_manager commerce_coordinator) for app in ${encrypted_config_apps[@]}; do eval app_decrypt_and_copy_config_enabled=\${${app}_decrypt_and_copy_config_enabled} From 0d03eadf06fdc6b82acdea85092c87759bc83f7e Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 25 Aug 2022 15:56:25 -0400 Subject: [PATCH 330/664] fix: Quote pip package specs on command line (#6795) This is required to avoid interpreting `<`, `#`, and other characters in the shell. (These can appear in URLs or pip version constraints.) --- playbooks/roles/edxapp/tasks/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 2bba7d5aa98..6acdaaddbb3 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -164,13 +164,13 @@ - install - install:app-requirements -# Private requriements require a ssh key to install, use the same key as the private key for edx-platform +# Private requirements require a ssh key to install, use the same key as the private key for edx-platform # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true - name: install python private requirements # Need to use shell rather than pip so that we can maintain the context of our current working directory; some # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly # installs everything into that virtual environment. - shell: "{{ edxapp_venv_dir }}/bin/pip install {{ COMMON_PIP_VERBOSITY }} -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w {{ item.extra_args|default('') }} {{ item.name }}" + shell: "{{ edxapp_venv_dir }}/bin/pip install {{ COMMON_PIP_VERBOSITY }} -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w {{ item.extra_args|default('') }} {{ item.name|quote }}" args: chdir: "{{ edxapp_code_dir }}" with_items: From 1931dfb6ae22f78ea89e944658bedfa29b0b2fad Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 16 Aug 2022 15:23:04 -0400 Subject: [PATCH 331/664] chore: Update redis to 6.2.6 ISRE-1212 --- playbooks/roles/redis/defaults/main.yml | 6 +- playbooks/roles/redis/tasks/main.yml | 27 +- .../redis/templates/etc/redis/redis.conf.j2 | 1544 ++++++++++++++--- 3 files changed, 1357 insertions(+), 220 deletions(-) diff --git a/playbooks/roles/redis/defaults/main.yml b/playbooks/roles/redis/defaults/main.yml index 97095d3708a..a8c061f8724 100644 --- a/playbooks/roles/redis/defaults/main.yml +++ b/playbooks/roles/redis/defaults/main.yml @@ -27,7 +27,11 @@ redis_group: redis # OS packages # +REDIS_REPO: "deb https://packages.redis.io/deb {{ ansible_distribution_release }} main" +REDIS_VERSION: "6:6.2.6-3rl1~focal1" + redis_debian_pkgs: - - "redis-server" + - "redis-tools={{ REDIS_VERSION }}" + - "redis-server={{ REDIS_VERSION }}" redis_redhat_pkgs: [] diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml index 9f454cd1c48..f799cdd8113 100644 --- a/playbooks/roles/redis/tasks/main.yml +++ b/playbooks/roles/redis/tasks/main.yml @@ -21,7 +21,24 @@ # # -- name: Install redis system packages +- name: add the redis repo signing key + apt_key: + url: "https://packages.redis.io/gpg" + state: present + retries: 3 + tags: + - "install" + - "install:app-requirements" + +- name: add the redis repo to the sources list + apt_repository: + repo: "{{ REDIS_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + +- name: Install redis packages apt: name: "{{ redis_debian_pkgs }}" install_recommends: yes @@ -30,6 +47,14 @@ notify: - reload redis +- name: Pin redis package version + loop: "{{ redis_debian_pkgs }}" + dpkg_selections: + # Dpkg selection just wants the package name, not the package and version + # This turns "redis=6:6.2.6-3rl1~focal1" into just "redis" + name: "{{ item.split('=')[0] }}" + selection: hold + - name: Update redis configuration template: src: "etc/redis/redis.conf.j2" diff --git a/playbooks/roles/redis/templates/etc/redis/redis.conf.j2 b/playbooks/roles/redis/templates/etc/redis/redis.conf.j2 index 56df8636db1..2bd794ba495 100644 --- a/playbooks/roles/redis/templates/etc/redis/redis.conf.j2 +++ b/playbooks/roles/redis/templates/etc/redis/redis.conf.j2 @@ -24,7 +24,7 @@ # to customize a few per-server settings. Include files can include # other files, so use this wisely. # -# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# Note that option "include" won't be rewritten by command "CONFIG REWRITE" # from admin or Redis Sentinel. Since Redis always uses the last processed # line as value of a configuration directive, you'd better put includes # at the beginning of this file to avoid overwriting config change at runtime. @@ -35,44 +35,84 @@ # include /path/to/local.conf # include /path/to/other.conf -################################ GENERAL ##################################### +################################## MODULES ##################################### -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize yes +# Load modules at startup. If the server is not able to load modules +# it will abort. It is possible to use multiple loadmodule directives. +# +# loadmodule /path/to/my_module.so +# loadmodule /path/to/other_module.so + +################################## NETWORK ##################################### + +# By default, if no "bind" configuration directive is specified, Redis listens +# for connections from all available network interfaces on the host machine. +# It is possible to listen to just one or multiple selected interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# Each address can be prefixed by "-", which means that redis will not fail to +# start if the address is not available. Being not available only refers to +# addresses that does not correspond to any network interfece. Addresses that +# are already in use will always fail, and unsupported protocols will always BE +# silently skipped. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses +# bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 +# bind * -::* # like the default, all available interfaces +# +# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the +# internet, binding to all the interfaces is dangerous and will expose the +# instance to everybody on the internet. So by default we uncomment the +# following bind directive, that will force Redis to listen only on the +# IPv4 and IPv6 (if available) loopback interface addresses (this means Redis +# will only be able to accept client connections from the same host that it is +# running on). +# +# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES +# JUST COMMENT OUT THE FOLLOWING LINE. +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +bind {{ REDIS_BIND_IP }} -# When running daemonized, Redis writes a pid file in /var/run/redis.pid by -# default. You can specify a custom pid file location here. -pidfile /var/run/redis/redis-server.pid +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode yes -# Accept connections on the specified port, default is 6379. +# Accept connections on the specified port, default is 6379 (IANA #815344). # If port 0 is specified Redis will not listen on a TCP socket. port 6379 # TCP listen() backlog. # -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel +# In high requests-per-second environments you need a high backlog in order +# to avoid slow clients connection issues. Note that the Linux kernel # will silently truncate it to the value of /proc/sys/net/core/somaxconn so # make sure to raise both the value of somaxconn and tcp_max_syn_backlog # in order to get the desired effect. tcp-backlog 511 -# By default Redis listens for connections from all the network interfaces -# available on the server. It is possible to listen to just one or multiple -# interfaces using the "bind" configuration directive, followed by one or -# more IP addresses. +# Unix socket. # -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -bind {{ REDIS_BIND_IP }} - # Specify the path for the Unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # -# unixsocket /var/run/redis/redis.sock +# unixsocket /run/redis.sock # unixsocketperm 700 # Close the connection after a client is idle for N seconds (0 to disable) @@ -84,15 +124,169 @@ timeout 0 # of communication. This is useful for two reasons: # # 1) Detect dead peers. -# 2) Take the connection alive from the point of view of network -# equipment in the middle. +# 2) Force network equipment in the middle to consider the connection to be +# alive. # # On Linux, the specified value (in seconds) is the period used to send ACKs. # Note that to close the connection the double of the time is needed. # On other kernels the period depends on the kernel configuration. # -# A reasonable value for this option is 60 seconds. -tcp-keepalive 0 +# A reasonable value for this option is 300 seconds, which is the new +# Redis default starting with Redis 3.2.1. +tcp-keepalive 300 + +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt +# tls-key-file redis.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Normally Redis uses the same certificate for both server functions (accepting +# connections) and client functions (replicating from a master, establishing +# cluster bus connections, etc.). +# +# Sometimes certificates are issued with attributes that designate them as +# client-only or server-only certificates. In that case it may be desired to use +# different certificates for incoming (server) and outgoing (client) +# connections. To do that, use the following directives: +# +# tls-client-cert-file client.crt +# tls-client-key-file client.key +# +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-client-key-file-pass secret + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients (including replica servers) on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, a Redis replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +# By default, the Redis Cluster bus uses a plain TCP connection. To enable +# TLS for the bus protocol, use the following directive: +# +# tls-cluster yes + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + +################################# GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +# When Redis is supervised by upstart or systemd, this parameter has no impact. +daemonize yes + +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# requires "expect stop" in your upstart job config +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# on startup, and updating Redis status on a regular +# basis. +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous pings back to your supervisor. +# +# The default is "no". To run under upstart/systemd, you can simply uncomment +# the line below: +# +supervised auto + +# If a pid file is specified, Redis writes it where specified at startup +# and removes it at exit. +# +# When the server runs non daemonized, no pid file is created if none is +# specified in the configuration. When the server is daemonized, the pid file +# is used even if not specified, defaulting to "/var/run/redis.pid". +# +# Creating a pid file is best effort: if Redis is not able to create it +# nothing bad happens, the server will start and run normally. +# +# Note that on modern Linux systems "/run/redis.pid" is more conforming +# and should be used instead. +pidfile /run/redis/redis-server.pid # Specify the server verbosity level. # This can be one of: @@ -117,36 +311,76 @@ logfile /var/log/redis/redis-server.log # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 +# To disable the built in crash log, which will possibly produce cleaner core +# dumps when they are needed, uncomment the following: +# +# crash-log-enabled no + +# To disable the fast memory check that's run as part of the crash log, which +# will possibly let redis terminate sooner, uncomment the following: +# +# crash-memcheck-enabled no + # Set the number of databases. The default database is DB 0, you can select # a different one on a per-connection basis using SELECT where # dbid is a number between 0 and 'databases'-1 databases 16 +# By default Redis shows an ASCII art logo only when started to log to the +# standard output and if the standard output is a TTY and syslog logging is +# disabled. Basically this means that normally a logo is displayed only in +# interactive sessions. +# +# However it is possible to force the pre-4.0 behavior and always show a +# ASCII art logo in startup logs by setting the following option to yes. +always-show-logo no + +# By default, Redis modifies the process title (as seen in 'top' and 'ps') to +# provide some runtime information. It is possible to disable this and leave +# the process name as executed by setting the following to no. +set-proc-title yes + +# When changing the process title, Redis uses the following template to construct +# the modified title. +# +# Template variables are specified in curly brackets. The following variables are +# supported: +# +# {title} Name of process as executed if parent, or type of child process. +# {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or +# Unix socket if only that's available. +# {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". +# {port} TCP port listening on, or 0. +# {tls-port} TLS port listening on, or 0. +# {unixsocket} Unix domain socket listening on, or "". +# {config-file} Name of configuration file used. +# +proc-title-template "{title} {listen-addr} {server-mode}" + ################################ SNAPSHOTTING ################################ + +# Save the DB to disk. # -# Save the DB on disk: +# save # -# save +# Redis will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. # -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. +# Snapshotting can be completely disabled with a single empty string argument +# as in following example: # -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed +# save "" # -# Note: you can disable saving completely by commenting out all "save" lines. +# Unless specified otherwise, by default Redis will save the DB: +# * After 3600 seconds (an hour) if at least 1 key changed +# * After 300 seconds (5 minutes) if at least 100 keys changed +# * After 60 seconds if at least 10000 keys changed # -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: +# You can set these explicitly by uncommenting the three following lines. # -# save "" - -save 900 1 -save 300 10 -save 60 10000 +# save 3600 1 +# save 300 100 +# save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. @@ -164,7 +398,7 @@ save 60 10000 stop-writes-on-bgsave-error yes # Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. +# By default compression is enabled as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes @@ -178,9 +412,37 @@ rdbcompression yes # tell the loading code to skip the check. rdbchecksum yes +# Enables or disables full sanitation checks for ziplist and listpack etc when +# loading an RDB or RESTORE payload. This reduces the chances of a assertion or +# crash later on while processing commands. +# Options: +# no - Never perform full sanitation +# yes - Always perform full sanitation +# clients - Perform full sanitation only for user connections. +# Excludes: RDB files, RESTORE commands received from the master +# connection, and client connections which have the +# skip-sanitize-payload ACL flag. +# The default should be 'clients' but since it currently affects cluster +# resharding via MIGRATE, it is temporarily set to 'no' by default. +# +# sanitize-dump-payload no + # The filename where to dump the DB dbfilename dump.rdb +# Remove RDB files used by replication in instances without persistence +# enabled. By default this option is disabled, however there are environments +# where for regulations or other security concerns, RDB files persisted on +# disk by masters in order to feed replicas, or stored on disk by replicas +# in order to load them for the initial synchronization, should be deleted +# ASAP. Note that this option ONLY WORKS in instances that have both AOF +# and RDB persistence disabled, otherwise is completely ignored. +# +# An alternative (and sometimes better) way to obtain the same effect is +# to use diskless replication on both master and replicas instances. However +# in the case of replicas, diskless is not always an option. +rdb-del-sync-files no + # The working directory. # # The DB will be written inside this directory, with the filename specified @@ -193,84 +455,100 @@ dir {{ REDIS_PERSISTENCE_DIR }} ################################# REPLICATION ################################# -# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# Master-Replica replication. Use replicaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# # 1) Redis replication is asynchronous, but you can configure a master to # stop accepting writes if it appears to be not connected with at least -# a given number of slaves. -# 2) Redis slaves are able to perform a partial resynchronization with the +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the # master if the replication link is lost for a relatively small amount of # time. You may want to configure the replication backlog size (see the next # sections of this file) with a sensible value depending on your needs. # 3) Replication is automatic and does not need user intervention. After a -# network partition slaves automatically try to reconnect to masters +# network partition replicas automatically try to reconnect to masters # and resynchronize with them. # -# slaveof +# replicaof # If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before +# directive below) it is possible to tell the replica to authenticate before # starting the replication synchronization process, otherwise the master will -# refuse the slave request. +# refuse the replica request. # # masterauth +# +# However this is not enough if you are using Redis ACLs (for Redis version +# 6 or greater), and the default user is not capable of running the PSYNC +# command and/or other commands needed for replication. In this case it's +# better to configure a special user to use with replication, and specify the +# masteruser configuration as such: +# +# masteruser +# +# When masteruser is specified, the replica will authenticate against its +# master using the new AUTH form: AUTH . -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: # -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO and SLAVEOF. +# 2) If replica-serve-stale-data is set to 'no' the replica will reply with +# an error "SYNC with master in progress" to all commands except: +# INFO, REPLICAOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, +# UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, +# HOST and LATENCY. # -slave-serve-stale-data yes +replica-serve-stale-data yes -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but # may also cause problems if clients are writing to it because of a # misconfiguration. # -# Since Redis 2.6 by default slaves are read-only. +# Since Redis 2.6 by default replicas are read-only. # -# Note: read only slaves are not designed to be exposed to untrusted clients +# Note: read only replicas are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. -# Still a read only slave exports by default all the administrative commands +# Still a read only replica exports by default all the administrative commands # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only slaves using 'rename-command' to shadow all the +# security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. -slave-read-only yes +replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. # -# New slaves and reconnecting slaves that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the slaves. # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB # file on disk. Later the file is transferred by the parent -# process to the slaves incrementally. +# process to the replicas incrementally. # 2) Diskless: The Redis master creates a new process that directly writes the -# RDB file to slave sockets, without touching the disk at all. +# RDB file to replica sockets, without touching the disk at all. # -# With disk-backed replication, while the RDB file is generated, more slaves -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new slaves arriving will be queued and a new transfer -# will start when the current one terminates. +# With disk-backed replication, while the RDB file is generated, more replicas +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple slaves -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -278,120 +556,347 @@ repl-diskless-sync no # When diskless replication is enabled, it is possible to configure the delay # the server waits in order to spawn the child that transfers the RDB via socket -# to the slaves. +# to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new slaves arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more slaves arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 -# seconds. -# -# repl-ping-slave-period 10 +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if you know what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely +# received from the master. +# +# In many cases the disk is slower than the network, and storing and loading +# the RDB file may increase replication time (and even increase the master's +# Copy on Write memory and salve buffers). +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# +# "disabled" - Don't use diskless load (store the rdb file to the disk first) +# "on-empty-db" - Use diskless load only when it is completely safe. +# "swapdb" - Keep a copy of the current db contents in RAM while parsing +# the data directly from the socket. note that this requires +# sufficient memory, if you don't have it, you risk an OOM kill. +repl-diskless-load disabled + +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. +# +# repl-ping-replica-period 10 # The following option sets the replication timeout for: # -# 1) Bulk transfer I/O during SYNC, from the point of view of slave. -# 2) Master timeout from the point of view of slaves (data, pings). -# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. The default +# value is 60 seconds. # # repl-timeout 60 -# Disable TCP_NODELAY on the slave socket after SYNC? +# Disable TCP_NODELAY on the replica socket after SYNC? # # If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to slaves. But this can add a delay for -# the data to appear on the slave side, up to 40 milliseconds with +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with # Linux kernels using a default configuration. # -# If you select "no" the delay for data to appear on the slave side will +# If you select "no" the delay for data to appear on the replica side will # be reduced but more bandwidth will be used for replication. # # By default we optimize for low latency, but in very high traffic conditions -# or when the master and slaves are many hops away, turning this to "yes" may +# or when the master and replicas are many hops away, turning this to "yes" may # be a good idea. repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# slave data when slaves are disconnected for some time, so that when a slave -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the slave missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # -# The bigger the replication backlog, the longer the time the slave can be -# disconnected and later be able to perform a partial resynchronization. +# The bigger the replication backlog, the longer the replica can endure the +# disconnect and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a slave connected. +# The backlog is only allocated if there is at least one replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected slaves for some time, the backlog -# will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last slave disconnected, for -# the backlog buffer to be freed. +# After a master has no connected replicas for some time, the backlog will be +# freed. The following option configures the amount of seconds that need to +# elapse, starting from the time the last replica disconnected, for the backlog +# buffer to be freed. +# +# Note that replicas never free the backlog for timeout, since they may be +# promoted to masters later, and should be able to correctly "partially +# resynchronize" with other replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The slave priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slaves with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # -# However a special priority of 0 marks the slave as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by # Redis Sentinel for promotion. # # By default the priority is 100. -slave-priority 100 +replica-priority 100 + +# ----------------------------------------------------------------------------- +# By default, Redis Sentinel includes all replicas in its reports. A replica +# can be excluded from Redis Sentinel's announcements. An unannounced replica +# will be ignored by the 'sentinel replicas ' command and won't be +# exposed to Redis Sentinel's clients. +# +# This option does not change the behavior of replica-priority. Even with +# replica-announced set to 'no', the replica can be promoted to master. To +# prevent this behavior, set replica-priority to 0. +# +# replica-announced yes # It is possible for a master to stop accepting writes if there are less than -# N slaves connected, having a lag less or equal than M seconds. +# N replicas connected, having a lag less or equal than M seconds. # -# The N slaves need to be in "online" state. +# The N replicas need to be in "online" state. # # The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the slave, that is usually sent every second. +# the last ping received from the replica, that is usually sent every second. # # This option does not GUARANTEE that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough slaves +# will limit the window of exposure for lost writes in case not enough replicas # are available, to the specified number of seconds. # -# For example to require at least 3 slaves with a lag <= 10 seconds use: +# For example to require at least 3 replicas with a lag <= 10 seconds use: # -# min-slaves-to-write 3 -# min-slaves-max-lag 10 +# min-replicas-to-write 3 +# min-replicas-max-lag 10 # # Setting one or the other to 0 disables the feature. # -# By default min-slaves-to-write is set to 0 (feature disabled) and -# min-slaves-max-lag is set to 10. +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. -################################## SECURITY ################################### +# A Redis master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP address and port normally reported by a replica is +# obtained in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may actually be reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 + +############################### KEYS TRACKING ################################# -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# a radix key indexed by key name, what clients have which keys. In turn +# this is used in order to send invalidation messages to clients. Please +# check this page to understand more about the feature: +# +# https://redis.io/topics/client-side-caching # -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. # -# Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 1M of keys, and once this limit +# is reached, Redis will start to evict keys in the invalidation table +# even if they were not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum size is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and Redis will +# retain as many keys as needed in the invalidation table. +# In the "stats" INFO section, you can find information about the number of +# keys in the invalidation table at every given moment. +# +# Note: when key tracking is used in broadcasting mode, no memory is used +# in the server side so this setting is useless. +# +# tracking-table-max-keys 1000000 + +################################## SECURITY ################################### + +# Warning: since Redis is pretty fast, an outside user can try up to +# 1 million passwords per second against a modern box. This means that you +# should use very strong passwords, otherwise they will be very easy to break. +# Note that because the password is really a shared secret between the client +# and the server, and should not be memorized by any human, the password +# can be easily a long string from /dev/urandom or whatever, so by using a +# long and unguessable password no brute force attack will be possible. + +# Redis ACL users are defined in the following format: +# +# user ... acl rules ... +# +# For example: +# +# user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 +# +# The special username "default" is used for new connections. If this user +# has the "nopass" rule, then new connections will be immediately authenticated +# as the "default" user without the need of any password provided via the +# AUTH command. Otherwise if the "default" user is not flagged with "nopass" +# the connections will start in not authenticated state, and will require +# AUTH (or the HELLO command AUTH option) in order to be authenticated and +# start to work. +# +# The ACL rules that describe what a user can do are the following: +# +# on Enable the user: it is possible to authenticate as this user. +# off Disable the user: it's no longer possible to authenticate +# with this user, however the already authenticated connections +# will still work. +# skip-sanitize-payload RESTORE dump-payload sanitation is skipped. +# sanitize-payload RESTORE dump-payload is sanitized (default). +# + Allow the execution of that command +# - Disallow the execution of that command +# +@ Allow the execution of all the commands in such category +# with valid categories are like @admin, @set, @sortedset, ... +# and so forth, see the full list in the server.c file where +# the Redis command table is described and defined. +# The special category @all means all the commands, but currently +# present in the server, and that will be loaded in the future +# via modules. +# +|subcommand Allow a specific subcommand of an otherwise +# disabled command. Note that this form is not +# allowed as negative like -DEBUG|SEGFAULT, but +# only additive starting with "+". +# allcommands Alias for +@all. Note that it implies the ability to execute +# all the future commands loaded via the modules system. +# nocommands Alias for -@all. +# ~ Add a pattern of keys that can be mentioned as part of +# commands. For instance ~* allows all the keys. The pattern +# is a glob-style pattern like the one of KEYS. +# It is possible to specify multiple patterns. +# allkeys Alias for ~* +# resetkeys Flush the list of allowed keys patterns. +# & Add a glob-style pattern of Pub/Sub channels that can be +# accessed by the user. It is possible to specify multiple channel +# patterns. +# allchannels Alias for &* +# resetchannels Flush the list of allowed channel patterns. +# > Add this password to the list of valid password for the user. +# For example >mypass will add "mypass" to the list. +# This directive clears the "nopass" flag (see later). +# < Remove this password from the list of valid passwords. +# nopass All the set passwords of the user are removed, and the user +# is flagged as requiring no password: it means that every +# password will work against this user. If this directive is +# used for the default user, every new connection will be +# immediately authenticated with the default user without +# any explicit AUTH command required. Note that the "resetpass" +# directive will clear this condition. +# resetpass Flush the list of allowed passwords. Moreover removes the +# "nopass" status. After "resetpass" the user has no associated +# passwords and there is no way to authenticate without adding +# some password (or setting it as "nopass" later). +# reset Performs the following actions: resetpass, resetkeys, off, +# -@all. The user returns to the same state it has immediately +# after its creation. +# +# ACL rules can be specified in any order: for instance you can start with +# passwords, then flags, or key patterns. However note that the additive +# and subtractive rules will CHANGE MEANING depending on the ordering. +# For instance see the following example: +# +# user alice on +@all -DEBUG ~* >somepassword +# +# This will allow "alice" to use all the commands with the exception of the +# DEBUG command, since +@all added all the commands to the set of the commands +# alice can use, and later DEBUG was removed. However if we invert the order +# of two ACL rules the result will be different: +# +# user alice on -DEBUG +@all ~* >somepassword +# +# Now DEBUG was removed when alice had yet no commands in the set of allowed +# commands, later all the commands are added, so the user will be able to +# execute everything. +# +# Basically ACL rules are processed left-to-right. +# +# For more information about ACL configuration please refer to +# the Redis web site at https://redis.io/topics/acl + +# ACL LOG +# +# The ACL Log tracks failed commands and authentication events associated +# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked +# by ACLs. The ACL Log is stored in memory. You can reclaim memory with +# ACL LOG RESET. Define the maximum entry length of the ACL Log below. +acllog-max-len 128 + +# Using an external ACL file +# +# Instead of configuring users here in this file, it is possible to use +# a stand-alone file just listing users. The two methods cannot be mixed: +# if you configure users here and at the same time you activate the external +# ACL file, the server will refuse to start. +# +# The format of the external ACL user file is exactly the same as the +# format that is used inside redis.conf to describe users. +# +# aclfile /etc/redis/users.acl + +# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility +# layer on top of the new ACL system. The option effect will be just setting +# the password for the default user. Clients will still authenticate using +# AUTH as usually, or more explicitly with AUTH default +# if they follow the new protocol: both will work. +# +# The requirepass is not compatable with aclfile option and the ACL LOAD +# command, these will cause requirepass to be ignored. # {% if REDIS_PASSWORD %} {# comment the password incase it has spaces. #} @@ -400,7 +905,34 @@ requirepass "{{ REDIS_PASSWORD }}" # requirepass foobared {% endif %} -# Command renaming. +# New users are initialized with restrictive permissions by default, via the +# equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it +# is possible to manage access to Pub/Sub channels with ACL rules as well. The +# default Pub/Sub channels permission if new users is controlled by the +# acl-pubsub-default configuration directive, which accepts one of these values: +# +# allchannels: grants access to all Pub/Sub channels +# resetchannels: revokes access to all Pub/Sub channels +# +# To ensure backward compatibility while upgrading Redis 6.0, acl-pubsub-default +# defaults to the 'allchannels' permission. +# +# Future compatibility note: it is very likely that in a future version of Redis +# the directive's default of 'allchannels' will be changed to 'resetchannels' in +# order to provide better out-of-the-box Pub/Sub security. Therefore, it is +# recommended that you explicitly define Pub/Sub permissions for all users +# rather then rely on implicit default values. Once you've set explicit +# Pub/Sub for all existing users, you should uncomment the following line. +# +# acl-pubsub-default resetchannels + +# Command renaming (DEPRECATED). +# +# ------------------------------------------------------------------------ +# WARNING: avoid using this option if possible. Instead use ACLs to remove +# commands from the default user, and put them only in some admin user you +# create for administrative purposes. +# ------------------------------------------------------------------------ # # It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something @@ -417,9 +949,9 @@ requirepass "{{ REDIS_PASSWORD }}" # rename-command CONFIG "" # # Please note that changing the name of commands that are logged into the -# AOF file or transmitted to slaves may cause problems. +# AOF file or transmitted to replicas may cause problems. -################################### LIMITS #################################### +################################### CLIENTS #################################### # Set the max number of connected clients at the same time. By default # this limit is set to 10000 clients, however if the Redis server is not @@ -430,9 +962,16 @@ requirepass "{{ REDIS_PASSWORD }}" # Once the limit is reached Redis will close all the new connections sending # an error 'max number of clients reached'. # +# IMPORTANT: When Redis Cluster is used, the max number of connections is also +# shared with the cluster bus: every node in the cluster will use two +# connections, one incoming and another outgoing. It is important to size the +# limit accordingly in case of very large clusters. +# # maxclients 10000 -# Don't use more memory than the specified amount of bytes. +############################## MEMORY MANAGEMENT ################################ + +# Set a memory usage limit to the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys # according to the eviction policy selected (see maxmemory-policy). # @@ -441,56 +980,260 @@ requirepass "{{ REDIS_PASSWORD }}" # that would use more memory, like SET, LPUSH, and so on, and will continue # to reply to read-only commands like GET. # -# This option is usually useful when using Redis as an LRU cache, or to set -# a hard memory limit for an instance (using the 'noeviction' policy). +# This option is usually useful when using Redis as an LRU or LFU cache, or to +# set a hard memory limit for an instance (using the 'noeviction' policy). # -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted # from the used memory count, so that network problems / resyncs will # not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion +# buffer of replicas is full with DELs of keys evicted triggering the deletion # of more keys, and so forth until the database is completely emptied. # -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # maxmemory {{ REDIS_MEMORY_LIMIT }} # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached. You can select among five behaviors: -# -# volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key according to the LRU algorithm -# volatile-random -> remove a random key with an expire set -# allkeys-random -> remove a random key, any key -# volatile-ttl -> remove the key with the nearest expire time (minor TTL) -# noeviction -> don't expire at all, just return an error on write operations -# -# Note: with any of the above policies, Redis will return an error on write -# operations, when there are no suitable keys for eviction. -# -# At the date of writing these commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort +# is reached. You can select one from the following behaviors: +# +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key having an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. +# +# Note: with any of the above policies, when there are no suitable keys for +# eviction, Redis will return an error on write operations that require +# more memory. These are usually commands that create new keys, add data or +# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, +# SORT (due to the STORE argument), and EXEC (if the transaction includes any +# command that requires memory). # # The default is: # maxmemory-policy {{ REDIS_MAX_MEMORY_POLICY }} -# LRU and minimal TTL algorithms are not precise algorithms but approximated +# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated # algorithms (in order to save memory), so you can tune it for speed or -# accuracy. For default Redis will check five keys and pick the one that was -# used less recently, you can change the sample size using the following +# accuracy. By default Redis will check five keys and pick the one that was +# used least recently, you can change the sample size using the following # configuration directive. # # The default of 5 produces good enough results. 10 Approximates very closely -# true LRU but costs a bit more CPU. 3 is very fast but not very accurate. +# true LRU but costs more CPU. 3 is faster but not very accurate. # # maxmemory-samples 5 +# Eviction processing is designed to function well with the default setting. +# If there is an unusually large amount of write traffic, this value may need to +# be increased. Decreasing this value may reduce latency at the risk of +# eviction processing effectiveness +# 0 = minimum latency, 10 = default, 100 = process without regard to latency +# +# maxmemory-eviction-tenacity 10 + +# Starting from Redis 5, by default a replica will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the replica as keys evict in the master side. +# +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). +# +# Note that since the replica by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. +# +# replica-ignore-maxmemory yes + +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tolerate less already expired keys still present +# in the system. It's a tradeoff between memory, CPU and latency. +# +# active-expire-effort 1 + +############################# LAZY FREEING #################################### + +# Redis has two primitives to delete keys. One is called DEL and is a blocking +# deletion of the object. It means that the server stops processing new commands +# in order to reclaim all the memory associated with an object in a synchronous +# way. If the key deleted is associated with a small object, the time needed +# in order to execute the DEL command is very small and comparable to most other +# O(1) or O(log_N) commands in Redis. However if the key is associated with an +# aggregated value containing millions of elements, the server can block for +# a long time (even seconds) in order to complete the operation. +# +# For the above reasons Redis also offers non blocking deletion primitives +# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and +# FLUSHDB commands, in order to reclaim memory in background. Those commands +# are executed in constant time. Another thread will incrementally free the +# object in the background as fast as possible. +# +# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. +# It's up to the design of the application to understand when it is a good +# idea to use one or the other. However the Redis server sometimes has to +# delete keys or flush the whole database as a side effect of other operations. +# Specifically Redis deletes objects independently of a user call in the +# following scenarios: +# +# 1) On eviction, because of the maxmemory and maxmemory policy configurations, +# in order to make room for new data, without going over the specified +# memory limit. +# 2) Because of expire: when a key with an associated time to live (see the +# EXPIRE command) must be deleted from memory. +# 3) Because of a side effect of a command that stores data on a key that may +# already exist. For example the RENAME command may delete the old key +# content when it is replaced with another one. Similarly SUNIONSTORE +# or SORT with STORE option may delete existing keys. The SET command +# itself removes any old content of the specified key in order to replace +# it with the specified string. +# 4) During replication, when a replica performs a full resynchronization with +# its master, the content of the whole database is removed in order to +# load the RDB file just transferred. +# +# In all the above cases the default is to delete objects in a blocking way, +# like if DEL was called. However you can configure each case specifically +# in order to instead release memory in a non-blocking way like if UNLINK +# was called, using the following configuration directives. + +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no + +# It is also possible, for the case when to replace the user code DEL calls +# with UNLINK calls is not easy, to modify the default behavior of the DEL +# command to act exactly like UNLINK, using the following configuration +# directive: + +lazyfree-lazy-user-del no + +# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous +# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the +# commands. When neither flag is passed, this directive will be used to determine +# if the data should be deleted asynchronously. + +lazyfree-lazy-user-flush no + +################################ THREADED I/O ################################# + +# Redis is mostly single threaded, however there are certain threaded +# operations such as UNLINK, slow I/O accesses and other things that are +# performed on side threads. +# +# Now it is also possible to handle Redis clients socket reads and writes +# in different I/O threads. Since especially writing is so slow, normally +# Redis users use pipelining in order to speed up the Redis performances per +# core, and spawn multiple instances in order to scale more. Using I/O +# threads it is possible to easily speedup two times Redis without resorting +# to pipelining nor sharding of the instance. +# +# By default threading is disabled, we suggest enabling it only in machines +# that have at least 4 or more cores, leaving at least one spare core. +# Using more than 8 threads is unlikely to help much. We also recommend using +# threaded I/O only if you actually have performance problems, with Redis +# instances being able to use a quite big percentage of CPU time, otherwise +# there is no point in using this feature. +# +# So for instance if you have a four cores boxes, try to use 2 or 3 I/O +# threads, if you have a 8 cores, try to use 6 threads. In order to +# enable I/O threads use the following configuration directive: +# +# io-threads 4 +# +# Setting io-threads to 1 will just use the main thread as usual. +# When I/O threads are enabled, we only use threads for writes, that is +# to thread the write(2) syscall and transfer the client buffers to the +# socket. However it is also possible to enable threading of reads and +# protocol parsing using the following configuration directive, by setting +# it to yes: +# +# io-threads-do-reads no +# +# Usually threading reads doesn't help much. +# +# NOTE 1: This configuration directive cannot be changed at runtime via +# CONFIG SET. Aso this feature currently does not work when SSL is +# enabled. +# +# NOTE 2: If you want to test the Redis speedup using redis-benchmark, make +# sure you also run the benchmark itself in threaded mode, using the +# --threads option to match the number of Redis threads, otherwise you'll not +# be able to notice the improvements. + +############################ KERNEL OOM CONTROL ############################## + +# On Linux, it is possible to hint the kernel OOM killer on what processes +# should be killed first when out of memory. +# +# Enabling this feature makes Redis actively control the oom_score_adj value +# for all its processes, depending on their role. The default scores will +# attempt to have background child processes killed before all others, and +# replicas killed before masters. +# +# Redis supports three options: +# +# no: Don't make changes to oom-score-adj (default). +# yes: Alias to "relative" see below. +# absolute: Values in oom-score-adj-values are written as is to the kernel. +# relative: Values are used relative to the initial value of oom_score_adj when +# the server starts and are then clamped to a range of -1000 to 1000. +# Because typically the initial value is 0, they will often match the +# absolute values. +oom-score-adj no + +# When oom-score-adj is used, this directive controls the specific values used +# for master, replica and background child processes. Values range -2000 to +# 2000 (higher means more likely to be killed). +# +# Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) +# can freely increase their value, but not decrease it below its initial +# settings. This means that setting oom-score-adj to "relative" and setting the +# oom-score-adj-values to positive values will always succeed. +oom-score-adj-values 0 200 800 + + +#################### KERNEL transparent hugepage CONTROL ###################### + +# Usually the kernel Transparent Huge Pages control is set to "madvise" or +# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which +# case this config has no effect. On systems in which it is set to "always", +# redis will attempt to disable it specifically for the redis process in order +# to avoid latency problems specifically with fork(2) and CoW. +# If for some reason you prefer to keep it enabled, you can set this config to +# "no" and the kernel global to "always". + +disable-thp yes + ############################## APPEND ONLY MODE ############################### # By default Redis asynchronously dumps the dataset on disk. This mode is @@ -509,7 +1252,7 @@ maxmemory-policy {{ REDIS_MAX_MEMORY_POLICY }} # If the AOF is enabled on startup Redis will load the AOF, that is the file # with the better durability guarantees. # -# Please check http://redis.io/topics/persistence for more information. +# Please check https://redis.io/topics/persistence for more information. appendonly no @@ -609,6 +1352,17 @@ auto-aof-rewrite-min-size 64mb # will be found. aof-load-truncated yes +# When rewriting the AOF file, Redis is able to use an RDB preamble in the +# AOF file for faster rewrites and recoveries. When this option is turned +# on the rewritten AOF file is composed of two different stanzas: +# +# [RDB file][AOF tail] +# +# When loading, Redis recognizes that the AOF file starts with the "REDIS" +# string and loads the prefixed RDB file, then continues loading the AOF +# tail. +aof-use-rdb-preamble yes + ################################ LUA SCRIPTING ############################### # Max execution time of a Lua script in milliseconds. @@ -619,7 +1373,7 @@ aof-load-truncated yes # # When a long running script exceeds the maximum execution time only the # SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second +# used to stop a script that did not yet call any write commands. The second # is the only way to shut down the server in the case a write command was # already issued by the script but the user doesn't want to wait for the natural # termination of the script. @@ -628,13 +1382,7 @@ aof-load-truncated yes lua-time-limit 5000 ################################ REDIS CLUSTER ############################### -# -# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however -# in order to mark it as "mature" we need to wait for a non trivial percentage -# of users to deploy it in production. -# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -# + # Normal Redis instances can't be part of a Redis Cluster; only nodes that are # started as cluster nodes can. In order to start a Redis instance as a # cluster node enable the cluster support uncommenting the following: @@ -651,46 +1399,46 @@ lua-time-limit 5000 # Cluster node timeout is the amount of milliseconds a node must be unreachable # for it to be considered in failure state. -# Most other internal time limits are multiple of the node timeout. +# Most other internal time limits are a multiple of the node timeout. # # cluster-node-timeout 15000 -# A slave of a failing master will avoid to start a failover if its data +# A replica of a failing master will avoid to start a failover if its data # looks too old. # -# There is no simple way for a slave to actually have a exact measure of +# There is no simple way for a replica to actually have an exact measure of # its "data age", so the following two checks are performed: # -# 1) If there are multiple slaves able to failover, they exchange messages -# in order to try to give an advantage to the slave with the best +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best # replication offset (more data from the master processed). -# Slaves will try to get their rank by offset, and apply to the start +# Replicas will try to get their rank by offset, and apply to the start # of the failover a delay proportional to their rank. # -# 2) Every single slave computes the time of the last interaction with +# 2) Every single replica computes the time of the last interaction with # its master. This can be the last ping or command received (if the master # is still in the "connected" state), or the time that elapsed since the # disconnection with the master (if the replication link is currently down). -# If the last interaction is too old, the slave will not try to failover +# If the last interaction is too old, the replica will not try to failover # at all. # -# The point "2" can be tuned by user. Specifically a slave will not perform +# The point "2" can be tuned by user. Specifically a replica will not perform # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * slave-validity-factor) + repl-ping-slave-period +# (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the slave-validity-factor -# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the -# slave will not try to failover if it was not able to talk with the master +# So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large slave-validity-factor may allow slaves with too old data to failover +# A large cluster-replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to -# elect a slave at all. +# elect a replica at all. # -# For maximum availability, it is possible to set the slave-validity-factor -# to a value of 0, which means, that slaves will always try to failover the +# For maximum availability, it is possible to set the cluster-replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their # offset rank). @@ -698,29 +1446,38 @@ lua-time-limit 5000 # Zero is the only value able to guarantee that when all the partitions heal # the cluster will always be able to continue. # -# cluster-slave-validity-factor 10 +# cluster-replica-validity-factor 10 -# Cluster slaves are able to migrate to orphaned masters, that are masters -# that are left without working slaves. This improves the cluster ability +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability # to resist to failures as otherwise an orphaned master can't be failed over -# in case of failure if it has no working slaves. +# in case of failure if it has no working replicas. # -# Slaves migrate to orphaned masters only if there are still at least a -# given number of other working slaves for their old master. This number -# is the "migration barrier". A migration barrier of 1 means that a slave -# will migrate only if there is at least 1 other working slave for its master -# and so forth. It usually reflects the number of slaves you want for every +# Replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every # master in your cluster. # -# Default is 1 (slaves migrate only if their masters remain with at least -# one slave). To disable migration just set it to a very large value. +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value or +# set cluster-allow-replica-migration to 'no'. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # # cluster-migration-barrier 1 +# Turning off this option allows to use less automatic cluster configuration. +# It both disables migration to orphaned masters and migration from masters +# that became empty. +# +# Default is 'yes' (allow automatic migrations). +# +# cluster-allow-replica-migration yes + # By default Redis Cluster nodes stop accepting queries if they detect there -# is at least an hash slot uncovered (no available node is serving it). +# is at least a hash slot uncovered (no available node is serving it). # This way if the cluster is partially down (for example a range of hash slots # are no longer covered) all the cluster becomes, eventually, unavailable. # It automatically returns available as soon as all the slots are covered again. @@ -732,8 +1489,73 @@ lua-time-limit 5000 # # cluster-require-full-coverage yes +# This option, when set to yes, prevents replicas from trying to failover its +# master during master failures. However the replica can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-replica-no-failover no + +# This option, when set to yes, allows nodes to serve read traffic while the +# the cluster is in a down state, as long as it believes it owns the slots. +# +# This is useful for two cases. The first case is for when an application +# doesn't require consistency of data during node failures or network partitions. +# One example of this is a cache, where as long as the node has the data it +# should be able to serve it. +# +# The second use case is for configurations that don't meet the recommended +# three shards but want to enable cluster mode and scale later. A +# master outage in a 1 or 2 shard configuration causes a read/write outage to the +# entire cluster without this option set, with it set there is only a write outage. +# Without a quorum of masters, slot ownership will not change automatically. +# +# cluster-allow-reads-when-down no + # In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. +# available at https://redis.io web site. + +########################## CLUSTER DOCKER/NAT support ######################## + +# In certain deployments, Redis Cluster nodes address discovery fails, because +# addresses are NAT-ted or because ports are forwarded (the typical case is +# Docker and other containers). +# +# In order to make Redis Cluster working in such environments, a static +# configuration where each node knows its public address is needed. The +# following four options are used for this scope, and are: +# +# * cluster-announce-ip +# * cluster-announce-port +# * cluster-announce-tls-port +# * cluster-announce-bus-port +# +# Each instructs the node about its address, client ports (for connections +# without and with TLS) and cluster message bus port. The information is then +# published in the header of the bus packets so that other nodes will be able to +# correctly map the address of the node publishing the information. +# +# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set +# to zero, then cluster-announce-port refers to the TLS port. Note also that +# cluster-announce-tls-port has no effect if cluster-tls is set to no. +# +# If the above options are not used, the normal Redis Cluster auto-detection +# will be used instead. +# +# Note that when remapped, the bus port may not be at the fixed offset of +# clients port + 10000, so you can specify any port and bus-port depending +# on how they get remapped. If the bus-port is not set, a fixed offset of +# 10000 will be used as usual. +# +# Example: +# +# cluster-announce-ip 10.1.1.5 +# cluster-announce-tls-port 6379 +# cluster-announce-port 0 +# cluster-announce-bus-port 6380 ################################## SLOW LOG ################################### @@ -783,7 +1605,7 @@ latency-monitor-threshold 0 ############################# EVENT NOTIFICATION ############################## # Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications +# This feature is documented at https://redis.io/topics/notifications # # For instance if keyspace events notification is enabled, and a client # performs a DEL operation on key "foo" stored in the Database 0, two @@ -805,7 +1627,12 @@ latency-monitor-threshold 0 # z Sorted set commands # x Expired events (events generated every time a key expires) # e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# t Stream commands +# d Module key type events +# m Key-miss events (Note: It is not included in the 'A' class) +# A Alias for g$lshzxetd, so that the "AKE" string means all the events +# (Except key-miss events which are excluded from 'A' due to their +# unique nature). # # The "notify-keyspace-events" takes as argument a string that is composed # of zero or multiple characters. The empty string means that notifications @@ -826,6 +1653,64 @@ latency-monitor-threshold 0 # specify at least one of K or E, no events will be delivered. notify-keyspace-events "" +############################### GOPHER SERVER ################################# + +# Redis contains an implementation of the Gopher protocol, as specified in +# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). +# +# The Gopher protocol was very popular in the late '90s. It is an alternative +# to the web, and the implementation both server and client side is so simple +# that the Redis server has just 100 lines of code in order to implement this +# support. +# +# What do you do with Gopher nowadays? Well Gopher never *really* died, and +# lately there is a movement in order for the Gopher more hierarchical content +# composed of just plain text documents to be resurrected. Some want a simpler +# internet, others believe that the mainstream internet became too much +# controlled, and it's cool to create an alternative space for people that +# want a bit of fresh air. +# +# Anyway for the 10nth birthday of the Redis, we gave it the Gopher protocol +# as a gift. +# +# --- HOW IT WORKS? --- +# +# The Redis Gopher support uses the inline protocol of Redis, and specifically +# two kind of inline requests that were anyway illegal: an empty request +# or any request that starts with "/" (there are no Redis commands starting +# with such a slash). Normal RESP2/RESP3 requests are completely out of the +# path of the Gopher protocol implementation and are served as usual as well. +# +# If you open a connection to Redis when Gopher is enabled and send it +# a string like "/foo", if there is a key named "/foo" it is served via the +# Gopher protocol. +# +# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher +# talking), you likely need a script like the following: +# +# https://github.com/antirez/gopher2redis +# +# --- SECURITY WARNING --- +# +# If you plan to put Redis on the internet in a publicly accessible address +# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. +# Once a password is set: +# +# 1. The Gopher server (when enabled, not by default) will still serve +# content via Gopher. +# 2. However other commands cannot be called before the client will +# authenticate. +# +# So use the 'requirepass' option to protect your instance. +# +# Note that Gopher is not currently supported when 'io-threads-do-reads' +# is enabled. +# +# To enable Gopher support, uncomment the following line and set the option +# from no (the default) to yes. +# +# gopher-enabled no + ############################### ADVANCED CONFIG ############################### # Hashes are encoded using a memory efficient data structure when they have a @@ -834,11 +1719,36 @@ notify-keyspace-events "" hash-max-ziplist-entries 512 hash-max-ziplist-value 64 -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 + +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 # Sets have a special encoding in just one case: when a set is composed # of just strings that happen to be integers in radix 10 in the range @@ -867,6 +1777,17 @@ zset-max-ziplist-value 64 # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. hll-sparse-max-bytes 3000 +# Streams macro node max size / items. The stream data structure is a radix +# tree of big nodes that encode multiple items inside. Using this configuration +# it is possible to configure how big a single node can be in bytes, and the +# maximum number of items it may contain before switching to a new node when +# appending new stream entries. If any of the following settings are set to +# zero, the limit is ignored, so for instance it is possible to set just a +# max entries limit by setting max-bytes to 0 and max-entries to the desired +# value. +stream-node-max-bytes 4096 +stream-node-max-entries 100 + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation Redis uses (see dict.c) @@ -895,7 +1816,7 @@ activerehashing yes # The limit can be set differently for the three different classes of clients: # # normal -> normal clients including MONITOR clients -# slave -> slave clients +# replica -> replica clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: @@ -916,14 +1837,28 @@ activerehashing yes # asynchronous clients may create a scenario where data is requested faster # than it can read. # -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. # # Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default in order to avoid that a protocol desynchronization (for +# instance due to a bug in the client) will lead to unbound memory usage in +# the query buffer. However you can configure it here if you have very special +# needs, such us huge multi/exec requests or alike. +# +# client-query-buffer-limit 1gb + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. However you can change this limit +# here, but must be 1mb or greater +# +# proto-max-bulk-len 512mb + # Redis calls an internal function to perform many background tasks, like # closing connections of clients in timeout, purging expired keys that are # never requested, and so forth. @@ -941,8 +1876,181 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporarily raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + # When a child rewrites the AOF file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. aof-rewrite-incremental-fsync yes + +# When redis saves RDB file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +rdb-save-incremental-fsync yes + +# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good +# idea to start with the default settings and only change them after investigating +# how to improve the performances and how the keys LFU change over time, which +# is possible to inspect via the OBJECT FREQ command. +# +# There are two tunable parameters in the Redis LFU implementation: the +# counter logarithm factor and the counter decay time. It is important to +# understand what the two parameters mean before changing them. +# +# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis +# uses a probabilistic increment with logarithmic behavior. Given the value +# of the old counter, when a key is accessed, the counter is incremented in +# this way: +# +# 1. A random number R between 0 and 1 is extracted. +# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). +# 3. The counter is incremented only if R < P. +# +# The default lfu-log-factor is 10. This is a table of how the frequency +# counter changes with a different number of accesses with different +# logarithmic factors: +# +# +--------+------------+------------+------------+------------+------------+ +# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | +# +--------+------------+------------+------------+------------+------------+ +# | 0 | 104 | 255 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 1 | 18 | 49 | 255 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 10 | 10 | 18 | 142 | 255 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# | 100 | 8 | 11 | 49 | 143 | 255 | +# +--------+------------+------------+------------+------------+------------+ +# +# NOTE: The above table was obtained by running the following commands: +# +# redis-benchmark -n 1000000 incr foo +# redis-cli object freq foo +# +# NOTE 2: The counter initial value is 5 in order to give new objects a chance +# to accumulate hits. +# +# The counter decay time is the time, in minutes, that must elapse in order +# for the key counter to be divided by two (or decremented if it has a value +# less <= 10). +# +# The default value for the lfu-decay-time is 1. A special value of 0 means to +# decay the counter every time it happens to be scanned. +# +# lfu-log-factor 10 +# lfu-decay-time 1 + +########################### ACTIVE DEFRAGMENTATION ####################### +# +# What is active defragmentation? +# ------------------------------- +# +# Active (online) defragmentation allows a Redis server to compact the +# spaces left between small allocations and deallocations of data in memory, +# thus allowing to reclaim back memory. +# +# Fragmentation is a natural process that happens with every allocator (but +# less so with Jemalloc, fortunately) and certain workloads. Normally a server +# restart is needed in order to lower the fragmentation, or at least to flush +# away all the data and create it again. However thanks to this feature +# implemented by Oran Agra for Redis 4.0 this process can happen at runtime +# in a "hot" way, while the server is running. +# +# Basically when the fragmentation is over a certain level (see the +# configuration options below) Redis will start to create new copies of the +# values in contiguous memory regions by exploiting certain specific Jemalloc +# features (in order to understand if an allocation is causing fragmentation +# and to allocate it in a better place), and at the same time, will release the +# old copies of the data. This process, repeated incrementally for all the keys +# will cause the fragmentation to drop back to normal values. +# +# Important things to understand: +# +# 1. This feature is disabled by default, and only works if you compiled Redis +# to use the copy of Jemalloc we ship with the source code of Redis. +# This is the default with Linux builds. +# +# 2. You never need to enable this feature if you don't have fragmentation +# issues. +# +# 3. Once you experience fragmentation, you can enable this feature when +# needed with the command "CONFIG SET activedefrag yes". +# +# The configuration parameters are able to fine tune the behavior of the +# defragmentation process. If you are not sure about what they mean it is +# a good idea to leave the defaults untouched. + +# Enabled active defragmentation +# activedefrag no + +# Minimum amount of fragmentation waste to start active defrag +# active-defrag-ignore-bytes 100mb + +# Minimum percentage of fragmentation to start active defrag +# active-defrag-threshold-lower 10 + +# Maximum percentage of fragmentation at which we use maximum effort +# active-defrag-threshold-upper 100 + +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 + +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 + +# Maximum number of set/hash/zset/list fields that will be processed from +# the main dictionary scan +# active-defrag-max-scan-fields 1000 + +# Jemalloc background thread for purging will be enabled by default +jemalloc-bg-thread yes + +# It is possible to pin different threads and processes of Redis to specific +# CPUs in your system, in order to maximize the performances of the server. +# This is useful both in order to pin different Redis threads in different +# CPUs, but also in order to make sure that multiple Redis instances running +# in the same host will be pinned to different CPUs. +# +# Normally you can do this using the "taskset" command, however it is also +# possible to this via Redis configuration directly, both in Linux and FreeBSD. +# +# You can pin the server/IO threads, bio threads, aof rewrite child process, and +# the bgsave child process. The syntax to specify the cpu list is the same as +# the taskset command: +# +# Set redis server/io threads to cpu affinity 0,2,4,6: +# server_cpulist 0-7:2 +# +# Set bio threads to cpu affinity 1,3: +# bio_cpulist 1,3 +# +# Set aof rewrite child process to cpu affinity 8,9,10,11: +# aof_rewrite_cpulist 8-11 +# +# Set bgsave child process to cpu affinity 1,10,11 +# bgsave_cpulist 1,10-11 + +# In some cases redis will emit warnings and even refuse to start if it detects +# that the system is in bad state, it is possible to suppress these warnings +# by setting the following config which takes a space delimited list of warnings +# to suppress +# +# ignore-warnings ARM64-COW-BUG From 09b761eb47608996a858db6d134d213656de13c5 Mon Sep 17 00:00:00 2001 From: MuhammadZakaria Date: Tue, 2 Aug 2022 17:50:47 +0500 Subject: [PATCH 332/664] feat: Added Cron Job to check TSM status and trigger alarm --- playbooks/roles/tableau_de/defaults/main.yml | 3 ++ playbooks/roles/tableau_de/tasks/main.yml | 32 +++++++++++++++++++ .../tableau_de/templates/tsm_status.sh.j2 | 16 ++++++++++ 3 files changed, 51 insertions(+) create mode 100644 playbooks/roles/tableau_de/templates/tsm_status.sh.j2 diff --git a/playbooks/roles/tableau_de/defaults/main.yml b/playbooks/roles/tableau_de/defaults/main.yml index 48c088ab8fb..b6f12ac53f3 100644 --- a/playbooks/roles/tableau_de/defaults/main.yml +++ b/playbooks/roles/tableau_de/defaults/main.yml @@ -9,8 +9,11 @@ tableau_user_group: "tableau" tableau_user: "{{ TABLEAU_ADMIN_USER }}" +tsm_admin_user: "{{ TABLEAU_SERVER_ADMIN_USER }}" +tableau_cron_dir: "/etc/cron" os_default_user: "ubuntu" +centos_default_user: "centos" # Snowflake driver parameter snowflake_odbc_driver_endpoint: "https://sfc-repo.snowflakecomputing.com/odbc/linux" diff --git a/playbooks/roles/tableau_de/tasks/main.yml b/playbooks/roles/tableau_de/tasks/main.yml index 8d4ff7156a2..7957c9c63ff 100644 --- a/playbooks/roles/tableau_de/tasks/main.yml +++ b/playbooks/roles/tableau_de/tasks/main.yml @@ -69,3 +69,35 @@ tags: - ssh - ssh:keys + +- name: Create TSM Service Status Script Directory + file: + path: "{{ tableau_cron_dir }}" + state: directory + mode: "{{ item.mode | default(0755) }}" + owner: "{{ centos_default_user }}" + group: "{{ tableau_user_group }}" + tags: + - install + - install:cron + +- name: Set up Tsm Service Status Script + template: + src: "tsm_status.sh.j2" + dest: "{{ tableau_cron_dir }}/tsm_status.sh" + mode: 0775 + owner: "{{ centos_default_user }}" + group: "{{ tableau_user_group }}" + tags: + - install + - install:cron + +- name: Create Cron Job to collect TSM status metrics every 5 minutes + cron: + name: Check TSM Status and report metric to CloudWatch + minute: "*/5" + owner: "{{ centos_default_user }}" + job: "{{ tableau_cron_dir }}/tsm_status.sh" + tags: + - install + - install:cron diff --git a/playbooks/roles/tableau_de/templates/tsm_status.sh.j2 b/playbooks/roles/tableau_de/templates/tsm_status.sh.j2 new file mode 100644 index 00000000000..5dd6d62e60b --- /dev/null +++ b/playbooks/roles/tableau_de/templates/tsm_status.sh.j2 @@ -0,0 +1,16 @@ +#!/bin/bash + +# Correctly installed aws_cli lives under /usr/local/bin +export PATH="/usr/local/bin:${PATH}" +RUNNING_STATUS="RUNNING" + +TSM_STATUS=$(sudo runuser -u {{ tsm_admin_user }} tsm status) + +if [[ $TSM_STATUS == *"$RUNNING_STATUS"* ]] +then + # TSM status is RUNNING, sending metric value '1' + aws cloudwatch put-metric-data --metric-name tableau_tsm_service_running --namespace AWS/EC2 --value 1 --region us-east-1 +else + # TSM Status is not RUNNING, sendin metric value '0' + aws cloudwatch put-metric-data --metric-name tableau_tsm_service_running --namespace AWS/EC2 --value 0 --region us-east-1 +fi From 9c5732633443ed09cbc2bcfb6d37d27714aac709 Mon Sep 17 00:00:00 2001 From: Abdou Seck Date: Thu, 25 Aug 2022 23:54:29 -0400 Subject: [PATCH 333/664] feat: Add a new jenkins-configuration script to start jenkins in quiet mode The following is carried out: 1. Add a 0StartInQuietMode.groovy script to the jenkins-configuration repo 2. Update the jenkins_data_engineering and jenkins_data_engineering_new playbooks to make sure that when start_jenkins_in_quiet_mode is set to true, only files not matching 0StartInQuietMode.groovy are deleted from the init.groovy.d directory. --- playbooks/roles/jenkins_common/defaults/main.yml | 2 ++ playbooks/roles/jenkins_data_engineering/defaults/main.yml | 2 ++ playbooks/roles/jenkins_data_engineering/meta/main.yml | 1 + playbooks/roles/jenkins_data_engineering/tasks/main.yml | 6 +++++- .../roles/jenkins_data_engineering_new/defaults/main.yml | 2 ++ playbooks/roles/jenkins_data_engineering_new/meta/main.yml | 1 + playbooks/roles/jenkins_data_engineering_new/tasks/main.yml | 6 +++++- 7 files changed, 18 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/jenkins_common/defaults/main.yml b/playbooks/roles/jenkins_common/defaults/main.yml index 621d7e6d14f..530de2bd429 100644 --- a/playbooks/roles/jenkins_common/defaults/main.yml +++ b/playbooks/roles/jenkins_common/defaults/main.yml @@ -11,6 +11,8 @@ jenkins_common_protocol_https: true # When checking if Jenkins is finished initializing, expect a 200 as it should # be publicly available jenkins_common_ready_status_code: 200 +# Always start Jenkins in Quiet/Maintenance mode +start_jenkins_in_quiet_mode: true JENKINS_SERVER_NAME: jenkins.example.org jenkins_node_usage: EXCLUSIVE diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index f5aaf080599..7edb6adf351 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -10,6 +10,8 @@ jenkins_user_home: '/home/{{ jenkins_user }}' jenkins_port: 8080 jenkins_nginx_port: 80 jenkins_protocol_https: true +# Always start Jenkins in Quiet/Maintenance mode +start_jenkins_in_quiet_mode: true AUTOMATION_USER: 'edx-analytics-automation' jenkins_host_name: "{{ JENKINS_SERVER_NAME | default('jenkins') }}" diff --git a/playbooks/roles/jenkins_data_engineering/meta/main.yml b/playbooks/roles/jenkins_data_engineering/meta/main.yml index 72f1aee7772..72456c96c7c 100644 --- a/playbooks/roles/jenkins_data_engineering/meta/main.yml +++ b/playbooks/roles/jenkins_data_engineering/meta/main.yml @@ -25,6 +25,7 @@ dependencies: - 5createLoggers.groovy - 5addSeedJob.groovy - 5configureEmailExtension.groovy + - 9StartInQuietMode.groovy jenkins_common_plugins_list: '{{ de_jenkins_plugins_list }}' jenkins_common_ghprb_white_list_phrase: '{{ de_jenkins_ghprb_white_list_phrase }}' jenkins_common_ghprb_ok_phrase: '{{ de_jenkins_ghprb_ok_phrase }}' diff --git a/playbooks/roles/jenkins_data_engineering/tasks/main.yml b/playbooks/roles/jenkins_data_engineering/tasks/main.yml index 8424df84466..83008c21b62 100644 --- a/playbooks/roles/jenkins_data_engineering/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering/tasks/main.yml @@ -116,8 +116,12 @@ - name: wipe initialization scripts from jenkins_commons file: - path: '{{ jenkins_home }}/init.groovy.d/' + path: '{{ jenkins_home }}/init.groovy.d/{{ item }}' state: absent + # Only delete files that don't match 9StartInQuietMode.groovy when start_jenkins_in_quiet_mode is on. + when: item != "9StartInQuietMode.groovy" and start_jenkins_in_quiet_mode + with_items: "{{ jenkins_common_configuration_scripts }}" + tags: - jenkins-auth diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index f9e68e97cc2..e4373020299 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -12,6 +12,8 @@ jenkins_user_home: '/home/{{ jenkins_user }}' jenkins_port: 8080 jenkins_nginx_port: 80 jenkins_protocol_https: true +# Always start Jenkins in Quiet/Maintenance mode +start_jenkins_in_quiet_mode: true AUTOMATION_USER: 'edx-analytics-automation' jenkins_host_name: "{{ JENKINS_SERVER_NAME | default('jenkins') }}" # We should expect a 403 Forbidden from Jenkins during the init stage, diff --git a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml index 8de3b56e1ef..1903d20353f 100644 --- a/playbooks/roles/jenkins_data_engineering_new/meta/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/meta/main.yml @@ -27,6 +27,7 @@ dependencies: - 5createLoggers.groovy - 5addSeedJob.groovy - 5configureEmailExtension.groovy + - 9StartInQuietMode.groovy jenkins_common_plugins_list: '{{ de_jenkins_plugins_list }}' jenkins_common_ghprb_white_list_phrase: '{{ de_jenkins_ghprb_white_list_phrase }}' jenkins_common_ghprb_ok_phrase: '{{ de_jenkins_ghprb_ok_phrase }}' diff --git a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml index e0e2a7a47ce..2312fb66374 100644 --- a/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/tasks/main.yml @@ -118,8 +118,12 @@ - name: wipe initialization scripts from jenkins_commons file: - path: '{{ jenkins_home }}/init.groovy.d/' + path: '{{ jenkins_home }}/init.groovy.d/{{ item }}' state: absent + # Only delete files that don't match 9StartInQuietMode.groovy when start_jenkins_in_quiet_mode is on. + when: item != "9StartInQuietMode.groovy" and start_jenkins_in_quiet_mode + with_items: "{{ jenkins_common_configuration_scripts }}" + tags: - jenkins-auth From 21d6d21e98b2eda8e3642087916ddb8dd36eec70 Mon Sep 17 00:00:00 2001 From: Ken Clary Date: Thu, 1 Sep 2022 16:21:05 -0400 Subject: [PATCH 334/664] chore: set up role for new relic in coursegraph/neo4j. TNL-10006 --- playbooks/neo4j.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/neo4j.yml b/playbooks/neo4j.yml index 986db32e6a4..d96a3182ad9 100644 --- a/playbooks/neo4j.yml +++ b/playbooks/neo4j.yml @@ -15,3 +15,5 @@ - neo4j - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE \ No newline at end of file From a637ea6a4f1c6d70ae62792e9c13ed2d291c2419 Mon Sep 17 00:00:00 2001 From: aliadnan Date: Thu, 8 Sep 2022 12:56:21 +0500 Subject: [PATCH 335/664] feat: Add cairo dev package in discovery container --- playbooks/roles/discovery/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index efd7158a2c0..3baa55538d3 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -46,6 +46,7 @@ discovery_debian_pkgs: - libxml2-dev - libxslt1-dev - libjpeg-dev + - libcairo2-dev DISCOVERY_NGINX_PORT: "1{{ discovery_gunicorn_port }}" From 3f1af155db176e0fc36d7f0c2a630e231bf6fb56 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 12 Sep 2022 15:33:55 -0400 Subject: [PATCH 336/664] feat: remove unnecessary version pins in jenkins job req. file also fix a typo in docker build docs --- docker/README.rst | 2 +- requirements/jenkins.in | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/README.rst b/docker/README.rst index 6a1c9bdbc69..a68a8f81e80 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -8,7 +8,7 @@ Docker support for edX services is volatile and experimental. We welcome interested testers and contributors. If you are interested in participating, please join us on Slack at https://openedx.slack.com/messages/docker. -We do not and may never run run these images in production. They are not +We do not and may never run these images in production. They are not currently suitable for production use. Tooling diff --git a/requirements/jenkins.in b/requirements/jenkins.in index f69c8fe2cea..aab284fe445 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -7,8 +7,8 @@ futures ; python_version == "2.7" # via s3transfer s3cmd pyyaml backoff==1.4.3 -celery==3.1.25 -click==6.7 +celery +click opsgenie-sdk==0.3.1 PyMySQL==0.9.3 python-gnupg From c2363dfc600d413e9e866c3baa70abb2fd9eb974 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Mon, 12 Sep 2022 15:38:31 -0400 Subject: [PATCH 337/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 8 +-- requirements.txt | 18 +++--- requirements/common_constraints.txt | 2 + requirements/pip-tools.txt | 4 +- requirements/pip.txt | 2 +- util/elasticsearch/requirements.txt | 2 +- util/jenkins/requirements-cloudflare.txt | 8 +-- util/jenkins/requirements.txt | 59 ++++++++++++------- util/pingdom/requirements.txt | 8 +-- util/vpc-tools/requirements.txt | 8 +-- 10 files changed, 70 insertions(+), 49 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 9249c77305c..3f3e985e5a5 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.25.18 +awscli==1.25.72 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.24.18 +boto3==1.24.71 # via -r requirements/aws.in -botocore==1.27.18 +botocore==1.27.71 # via # awscli # boto3 @@ -45,5 +45,5 @@ s3transfer==0.6.0 # boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.9 +urllib3==1.26.12 # via botocore diff --git a/requirements.txt b/requirements.txt index 3cbed8b3238..323779e1783 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.9.0 # via -r requirements/base.in -awscli==1.25.18 +awscli==1.25.72 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,25 +14,25 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.24.18 +boto3==1.24.71 # via -r requirements/base.in -botocore==1.27.18 +botocore==1.27.71 # via # awscli # boto3 # s3transfer -certifi==2022.6.15 +certifi==2022.6.15.1 # via requests -cffi==1.15.0 +cffi==1.15.1 # via # bcrypt # cryptography # pynacl -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests colorama==0.4.4 # via awscli -cryptography==37.0.2 +cryptography==38.0.1 # via # ansible # paramiko @@ -91,7 +91,7 @@ pyyaml==5.4.1 # -r requirements/base.in # ansible # awscli -requests==2.28.0 +requests==2.28.1 # via # -r requirements/base.in # datadog @@ -106,7 +106,7 @@ six==1.16.0 # bcrypt # pathlib2 # python-dateutil -urllib3==1.26.9 +urllib3==1.26.12 # via # botocore # requests diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index cca3ccd308c..b1dfdf0176d 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -19,6 +19,8 @@ Django<4.0 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html elasticsearch<7.14.0 +# setuptools==60.0 had breaking changes and busted several service's pipeline. +# Details can be found here: https://github.com/pypa/setuptools/issues/2940 setuptools<60 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 97f4ed93dc9..ebb8aa6a30f 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -10,9 +10,9 @@ click==8.1.3 # via pip-tools packaging==21.3 # via build -pep517==0.12.0 +pep517==0.13.0 # via build -pip-tools==6.7.0 +pip-tools==6.8.0 # via -r requirements/pip-tools.in pyparsing==3.0.9 # via packaging diff --git a/requirements/pip.txt b/requirements/pip.txt index 8a667c4a29e..35b87891c26 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.37.1 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==22.1.2 +pip==22.2.2 # via -r requirements/pip.in setuptools==59.8.0 # via diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index c284838d599..254db4a8416 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -10,5 +10,5 @@ elasticsearch==0.4.5 # via -r requirements/elasticsearch.in jsonpickle==2.2.0 # via deepdiff -urllib3==1.26.9 +urllib3==1.26.12 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index a3ee4e40405..9ad045870f2 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,15 +4,15 @@ # # make upgrade # -certifi==2022.6.15 +certifi==2022.6.15.1 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests click==8.1.3 # via -r requirements/cloudflare.in idna==3.3 # via requests -requests==2.28.0 +requests==2.28.1 # via -r requirements/cloudflare.in -urllib3==1.26.9 +urllib3==1.26.12 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index fe1f416d5c5..4b610fd138e 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -4,49 +4,60 @@ # # make upgrade # -amqp==1.4.9 +amqp==5.1.1 # via kombu -anyjson==0.3.3 - # via kombu -awscli==1.25.0 +awscli==1.25.72 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in -billiard==3.3.0.23 +billiard==3.6.4.0 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.24.0 +boto3==1.24.71 # via -r requirements/jenkins.in -botocore==1.27.0 +botocore==1.27.71 # via # awscli # boto3 # s3transfer -celery==3.1.25 +celery==5.2.7 # via -r requirements/jenkins.in -certifi==2022.5.18.1 +certifi==2022.6.15.1 # via # opsgenie-sdk # requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests -click==6.7 - # via -r requirements/jenkins.in +click==8.1.3 + # via + # -r requirements/jenkins.in + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.2.0 + # via celery colorama==0.4.4 # via awscli docutils==0.16 # via awscli idna==3.3 # via requests -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore -kombu==3.0.37 +kombu==5.2.4 # via celery opsgenie-sdk==0.3.1 # via -r requirements/jenkins.in +prompt-toolkit==3.0.31 + # via click-repl pyasn1==0.4.8 # via rsa pymysql==0.9.3 @@ -56,11 +67,11 @@ python-dateutil==2.8.2 # botocore # opsgenie-sdk # s3cmd -python-gnupg==0.4.9 +python-gnupg==0.5.0 # via -r requirements/jenkins.in -python-magic==0.4.26 +python-magic==0.4.27 # via s3cmd -pytz==2022.1 +pytz==2022.2.1 # via # celery # opsgenie-sdk @@ -70,7 +81,7 @@ pyyaml==5.4.1 # awscli redis==2.10.6 # via -r requirements/jenkins.in -requests==2.27.1 +requests==2.28.1 # via opsgenie-sdk rsa==4.7.2 # via awscli @@ -82,15 +93,23 @@ s3transfer==0.6.0 # boto3 six==1.16.0 # via + # click-repl # opsgenie-sdk # python-dateutil -splunk-sdk==1.6.6 +splunk-sdk==1.6.16 # via -r requirements/jenkins.in -urllib3==1.26.9 +urllib3==1.26.12 # via # botocore # opsgenie-sdk # requests +vine==5.0.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.5 + # via prompt-toolkit # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index f2befe795ab..04183c2c24a 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,9 +4,9 @@ # # make upgrade # -certifi==2022.6.15 +certifi==2022.6.15.1 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests click==6.7 # via -r requirements/pingdom.in @@ -14,9 +14,9 @@ idna==3.3 # via requests pyyaml==6.0 # via -r requirements/pingdom.in -requests==2.28.0 +requests==2.28.1 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.26.9 +urllib3==1.26.12 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index e3fca741449..635cb6e3dcc 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,15 +6,15 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2022.6.15 +certifi==2022.6.15.1 # via requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.1 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in idna==3.3 # via requests -requests==2.28.0 +requests==2.28.1 # via -r requirements/vpc-tools.in -urllib3==1.26.9 +urllib3==1.26.12 # via requests From d2abb65d4a358994380c418766dcc94689fb0eff Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 15 Sep 2022 18:24:39 -0400 Subject: [PATCH 338/664] fix: Use npm clean-install to match edx-platform --- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 6acdaaddbb3..77d5dffcb51 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -250,7 +250,7 @@ - install:app-requirements - name: install node dependencies - shell: "easy_install --version && npm install" + shell: "easy_install --version && npm clean-install" args: chdir: "{{ edxapp_code_dir }}" environment: "{{ edxapp_environment | combine(git_ssh_environment_mixin) }}" From 1901a9a469971015b814c474a597fd23507e7243 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:08:52 +0500 Subject: [PATCH 339/664] fix: add local requirements --- playbooks/roles/edxapp/tasks/deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 77d5dffcb51..fd0c041c5e7 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -163,6 +163,12 @@ tags: - install - install:app-requirements + +- name: Install Local edxapp Requirements + command: "{{ edxapp_venv_dir }}/bin/pip install -e {{ edxapp_code_dir }}/." + tags: + - install + - install:app-requirements # Private requirements require a ssh key to install, use the same key as the private key for edx-platform # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true From 2ac8228b81423ff5474fbe8a477e336153fffb1b Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Fri, 16 Sep 2022 14:39:47 +0500 Subject: [PATCH 340/664] fix: add edxapp user to local req installation --- playbooks/roles/edxapp/tasks/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index fd0c041c5e7..4af9e3e2932 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -166,6 +166,7 @@ - name: Install Local edxapp Requirements command: "{{ edxapp_venv_dir }}/bin/pip install -e {{ edxapp_code_dir }}/." + become_user: "{{ edxapp_user }}" tags: - install - install:app-requirements From ce2bd25dec46c10b869bd029ccde58b83aa8ea9d Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Fri, 16 Sep 2022 16:52:59 +0500 Subject: [PATCH 341/664] fix: fix name --- playbooks/roles/edxapp/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 4af9e3e2932..c322f9fa38c 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -164,7 +164,7 @@ - install - install:app-requirements -- name: Install Local edxapp Requirements +- name: Install local edxapp requirements command: "{{ edxapp_venv_dir }}/bin/pip install -e {{ edxapp_code_dir }}/." become_user: "{{ edxapp_user }}" tags: From 40ac63a372c8692ae33d0eb7314620660e2f27a7 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 13 Sep 2022 23:14:58 +0500 Subject: [PATCH 342/664] feat: Upgrade ansible to 2.10 --- playbooks/roles/nginx/tasks/main.yml | 2 ++ playbooks/roles/oraclejdk/tasks/main.yml | 3 ++- playbooks/roles/supervisor/tasks/main.yml | 4 ++-- requirements.txt | 14 ++++++++++---- requirements/base.in | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index bb7bb59ba8c..592c22b1d20 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -319,6 +319,7 @@ module: stat path: "{{ NGINX_SSL_CERTIFICATE }}" become: False + connection: local register: ssl_cert tags: - install @@ -328,6 +329,7 @@ module: stat path: "{{ NGINX_SSL_KEY }}" become: False + connection: local register: ssl_key tags: - install diff --git a/playbooks/roles/oraclejdk/tasks/main.yml b/playbooks/roles/oraclejdk/tasks/main.yml index cf1cb84c898..2ef46eb0106 100644 --- a/playbooks/roles/oraclejdk/tasks/main.yml +++ b/playbooks/roles/oraclejdk/tasks/main.yml @@ -19,7 +19,8 @@ - name: Download Oracle Java get_url: url: "{{ oraclejdk_url }}" - headers: 'Cookie:oraclelicense=accept-securebackup-cookie' + headers: + Cookie: oraclelicense=accept-securebackup-cookie dest: "/var/tmp/{{ oraclejdk_file }}" - name: Create jvm dir diff --git a/playbooks/roles/supervisor/tasks/main.yml b/playbooks/roles/supervisor/tasks/main.yml index b4cd1dd939c..7df54ce3ec8 100644 --- a/playbooks/roles/supervisor/tasks/main.yml +++ b/playbooks/roles/supervisor/tasks/main.yml @@ -164,7 +164,7 @@ # this link is deprecated in favor of the shell wrapper - name: Remove a symlink for supervisortctl file: - src: "{{ supervisor_ctl }}" + # src: "{{ supervisor_ctl }}" dest: "{{ COMMON_BIN_DIR }}/{{ supervisor_ctl | basename }}" state: absent when: supervisor_service == "supervisor" @@ -175,7 +175,7 @@ # these links are deprecated in favor of the shell wrapper - name: Remove symlink for supervisor cfg file: - src: "{{ item }}" + # src: "{{ item }}" dest: "{{ COMMON_CFG_DIR }}/{{ item | basename }}" state: absent when: supervisor_service == "supervisor" diff --git a/requirements.txt b/requirements.txt index 323779e1783..ea7312aeb17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,10 @@ # # make upgrade # -ansible==2.9.0 +ansible==2.10.7 # via -r requirements/base.in +ansible-base==2.10.17 + # via ansible awscli==1.25.72 # via -r requirements/base.in bcrypt==3.1.7 @@ -34,7 +36,7 @@ colorama==0.4.4 # via awscli cryptography==38.0.1 # via - # ansible + # ansible-base # paramiko datadog==0.8.0 # via -r requirements/base.in @@ -53,7 +55,7 @@ idna==3.3 jinja2==2.8 # via # -r requirements/base.in - # ansible + # ansible-base jmespath==1.0.1 # via # boto3 @@ -66,6 +68,8 @@ mysqlclient==1.4.6 # via -r requirements/base.in networkx==1.11 # via -r requirements/base.in +packaging==21.3 + # via ansible-base paramiko==2.4.2 # via -r requirements/base.in pathlib2==2.3.0 @@ -84,12 +88,14 @@ pymongo==3.9.0 # via -r requirements/base.in pynacl==1.5.0 # via paramiko +pyparsing==3.0.9 + # via packaging python-dateutil==2.8.2 # via botocore pyyaml==5.4.1 # via # -r requirements/base.in - # ansible + # ansible-base # awscli requests==2.28.1 # via diff --git a/requirements/base.in b/requirements/base.in index e429c2e6393..a1eaa57a6a1 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,7 +1,7 @@ -c constraints.txt # Standard dependencies for Ansible runs -ansible==2.9.0 +ansible<3.0.0 awscli boto boto3 From 77eeab74c3ed969601d277ea8097770bc01f7f99 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Sat, 10 Sep 2022 15:52:13 -0400 Subject: [PATCH 343/664] fix: fix github url strings (org edx -> openedx) --- README.rst | 2 +- docker/build/analytics_pipeline/Dockerfile | 2 +- docker/build/bionic-common/Dockerfile | 2 +- docker/build/focal-common/Dockerfile | 2 +- docker/build/trusty-common/Dockerfile | 2 +- docker/build/xenial-common/Dockerfile | 2 +- docker/build/xqwatcher/ansible_overrides.yml | 2 +- .../rollback_migrations.yml | 2 +- .../continuous_delivery/run_migrations.yml | 2 +- playbooks/manage_edxapp_users_and_groups.yml | 4 ++-- playbooks/ora2.yml | 2 +- playbooks/populate_configuration_model.yml | 4 ++-- .../roles/ad_hoc_reporting/defaults/main.yml | 4 ++-- .../roles/ad_hoc_reporting/meta/main.yml | 4 ++-- .../roles/ad_hoc_reporting/tasks/main.yml | 4 ++-- playbooks/roles/add_user/defaults/main.yml | 4 ++-- playbooks/roles/add_user/meta/main.yml | 4 ++-- playbooks/roles/add_user/tasks/main.yml | 4 ++-- .../roles/analytics_api/defaults/main.yml | 4 ++-- playbooks/roles/analytics_api/meta/main.yml | 4 ++-- playbooks/roles/analytics_api/tasks/main.yml | 4 ++-- .../analytics_pipeline/defaults/main.yml | 4 ++-- .../roles/analytics_pipeline/meta/main.yml | 4 ++-- .../roles/analytics_pipeline/tasks/main.yml | 4 ++-- .../templates/header.j2 | 4 ++-- .../roles/ansible-role/templates/header.j2 | 4 ++-- playbooks/roles/antivirus/defaults/main.yml | 4 ++-- playbooks/roles/antivirus/handlers/main.yml | 4 ++-- playbooks/roles/antivirus/meta/main.yml | 4 ++-- playbooks/roles/antivirus/tasks/main.yml | 4 ++-- playbooks/roles/automated/defaults/main.yml | 8 +++---- playbooks/roles/automated/meta/main.yml | 4 ++-- playbooks/roles/automated/tasks/main.yml | 8 +++---- playbooks/roles/aws/defaults/main.yml | 4 ++-- playbooks/roles/aws/meta/main.yml | 4 ++-- playbooks/roles/aws/tasks/main.yml | 4 ++-- playbooks/roles/blockstore/defaults/main.yml | 2 +- playbooks/roles/blockstore/meta/main.yml | 2 +- playbooks/roles/blockstore/tasks/main.yml | 2 +- playbooks/roles/cassandra/tasks/main.yml | 4 ++-- playbooks/roles/codejail/tasks/main.yml | 2 +- .../commerce_coordinator/defaults/main.yml | 4 ++-- .../roles/commerce_coordinator/meta/main.yml | 4 ++-- .../roles/commerce_coordinator/tasks/main.yml | 4 ++-- playbooks/roles/common_vars/defaults/main.yml | 4 ++-- playbooks/roles/conductor/defaults/main.yml | 4 ++-- playbooks/roles/credentials/defaults/main.yml | 4 ++-- playbooks/roles/credentials/meta/main.yml | 4 ++-- playbooks/roles/credentials/tasks/main.yml | 4 ++-- playbooks/roles/demo/defaults/main.yml | 4 ++-- playbooks/roles/demo/tasks/main.yml | 6 ++--- playbooks/roles/designer/defaults/main.yml | 4 ++-- playbooks/roles/designer/meta/main.yml | 4 ++-- playbooks/roles/designer/tasks/main.yml | 4 ++-- playbooks/roles/devpi/defaults/main.yml | 4 ++-- playbooks/roles/devpi/tasks/main.yml | 4 ++-- playbooks/roles/discovery/defaults/main.yml | 4 ++-- playbooks/roles/discovery/meta/main.yml | 4 ++-- playbooks/roles/discovery/tasks/main.yml | 4 ++-- playbooks/roles/ecommerce/defaults/main.yml | 4 ++-- playbooks/roles/ecommerce/meta/main.yml | 4 ++-- playbooks/roles/ecomworker/defaults/main.yml | 4 ++-- playbooks/roles/ecomworker/meta/main.yml | 4 ++-- playbooks/roles/ecomworker/tasks/main.yml | 4 ++-- playbooks/roles/edx-sandbox/tasks/main.yml | 4 ++-- playbooks/roles/edx_ansible/defaults/main.yml | 6 ++--- playbooks/roles/edx_ansible/meta/main.yml | 4 ++-- playbooks/roles/edx_ansible/tasks/main.yml | 4 ++-- .../roles/edx_django_service/tasks/main.yml | 4 ++-- .../tasks/main.yml | 4 ++-- .../roles/edx_maintenance/defaults/main.yml | 4 ++-- playbooks/roles/edx_maintenance/meta/main.yml | 4 ++-- .../roles/edx_maintenance/tasks/main.yml | 4 ++-- .../roles/edx_notes_api/defaults/main.yml | 4 ++-- playbooks/roles/edx_notes_api/meta/main.yml | 4 ++-- playbooks/roles/edx_notes_api/tasks/main.yml | 6 ++--- playbooks/roles/edx_service/defaults/main.yml | 4 ++-- playbooks/roles/edx_service/meta/main.yml | 4 ++-- playbooks/roles/edx_service/tasks/main.yml | 4 ++-- .../defaults/main.yml | 4 ++-- .../meta/main.yml | 4 ++-- .../tasks/main.yml | 4 ++-- playbooks/roles/edx_themes/defaults/main.yml | 4 ++-- playbooks/roles/edx_themes/meta/main.yml | 4 ++-- playbooks/roles/edx_themes/tasks/main.yml | 4 ++-- playbooks/roles/edxapp/defaults/main.yml | 24 +++++++++---------- .../roles/enhanced_networking/tasks/main.yml | 4 ++-- .../enterprise_catalog/defaults/main.yml | 4 ++-- .../roles/enterprise_catalog/meta/main.yml | 4 ++-- .../roles/enterprise_catalog/tasks/main.yml | 4 ++-- playbooks/roles/forum/defaults/main.yml | 2 +- playbooks/roles/gh_mirror/defaults/main.yml | 4 ++-- playbooks/roles/gh_mirror/tasks/main.yml | 4 ++-- playbooks/roles/git_clone/defaults/main.yml | 4 ++-- playbooks/roles/git_clone/meta/main.yml | 4 ++-- playbooks/roles/git_clone/tasks/main.yml | 4 ++-- playbooks/roles/gitreload/defaults/main.yml | 6 ++--- playbooks/roles/gitreload/handlers/main.yml | 4 ++-- playbooks/roles/gitreload/meta/main.yml | 4 ++-- playbooks/roles/gitreload/tasks/main.yml | 4 ++-- .../go-agent-docker-server/defaults/main.yml | 4 ++-- .../go-agent-docker-server/tasks/main.yml | 4 ++-- playbooks/roles/grafana/tasks/main.yml | 4 ++-- playbooks/roles/graphite/tasks/main.yml | 4 ++-- .../roles/hadoop_common/defaults/main.yml | 4 ++-- playbooks/roles/hadoop_common/meta/main.yml | 4 ++-- playbooks/roles/hadoop_common/tasks/main.yml | 4 ++-- .../roles/hadoop_master/defaults/main.yml | 4 ++-- playbooks/roles/hadoop_master/meta/main.yml | 4 ++-- playbooks/roles/hadoop_master/tasks/main.yml | 4 ++-- playbooks/roles/haproxy/defaults/main.yml | 4 ++-- playbooks/roles/haproxy/handlers/main.yml | 4 ++-- playbooks/roles/haproxy/meta/main.yml | 4 ++-- playbooks/roles/haproxy/tasks/main.yml | 4 ++-- playbooks/roles/hermes/defaults/main.yml | 4 ++-- playbooks/roles/hermes/meta/main.yml | 4 ++-- playbooks/roles/hermes/tasks/main.yml | 4 ++-- playbooks/roles/hive/defaults/main.yml | 4 ++-- playbooks/roles/hive/meta/main.yml | 4 ++-- playbooks/roles/hive/tasks/main.yml | 4 ++-- playbooks/roles/hotg/defaults/main.yml | 8 +++---- playbooks/roles/hotg/meta/main.yml | 8 +++---- playbooks/roles/hotg/tasks/main.yml | 8 +++---- playbooks/roles/insights/defaults/main.yml | 4 ++-- playbooks/roles/insights/meta/main.yml | 4 ++-- playbooks/roles/insights/tasks/main.yml | 4 ++-- .../roles/jenkins_admin/defaults/main.yml | 4 ++-- .../roles/jenkins_admin/handlers/main.yml | 4 ++-- playbooks/roles/jenkins_admin/meta/main.yml | 4 ++-- playbooks/roles/jenkins_admin/tasks/main.yml | 4 ++-- .../tasks/loadtest_driver_worker.yml | 2 +- .../tasks/python_platform_worker.yml | 2 +- .../roles/learner_portal/defaults/main.yml | 6 ++--- .../roles/license_manager/defaults/main.yml | 4 ++-- playbooks/roles/license_manager/meta/main.yml | 4 ++-- .../roles/license_manager/tasks/main.yml | 4 ++-- playbooks/roles/loadtest_driver/meta/main.yml | 4 ++-- playbooks/roles/locust/defaults/main.yml | 4 ++-- playbooks/roles/locust/meta/main.yml | 4 ++-- playbooks/roles/locust/tasks/main.yml | 4 ++-- playbooks/roles/mariadb/defaults/main.yml | 4 ++-- playbooks/roles/mariadb/meta/main.yml | 4 ++-- playbooks/roles/mariadb/tasks/main.yml | 4 ++-- playbooks/roles/mfe/tasks/main.yml | 4 ++-- playbooks/roles/mfe_deployer/README.rst | 4 ++-- .../roles/mfe_flags_setup/tasks/main.yml | 4 ++-- playbooks/roles/minos/defaults/main.yml | 4 ++-- playbooks/roles/minos/meta/main.yml | 4 ++-- playbooks/roles/minos/tasks/main.yml | 4 ++-- playbooks/roles/mount_ebs/tasks/main.yml | 4 ++-- playbooks/roles/munin_node/defaults/main.yml | 4 ++-- playbooks/roles/munin_node/tasks/main.yml | 4 ++-- playbooks/roles/neo4j/defaults/main.yml | 6 ++--- playbooks/roles/neo4j/tasks/main.yml | 4 ++-- .../newrelic_infrastructure/defaults/main.yml | 4 ++-- .../newrelic_infrastructure/tasks/main.yml | 4 ++-- .../edx/app/nginx/sites-available/maps.j2 | 2 +- .../oauth_client_setup/defaults/main.yml | 4 ++-- .../roles/oauth_client_setup/tasks/main.yml | 4 ++-- playbooks/roles/openstack/defaults/main.yml | 8 +++---- playbooks/roles/openstack/meta/main.yml | 8 +++---- playbooks/roles/openstack/tasks/main.yml | 8 +++---- .../roles/program_console/defaults/main.yml | 6 ++--- playbooks/roles/prospectus/defaults/main.yml | 4 ++-- playbooks/roles/rabbitmq/defaults/main.yml | 8 +++---- playbooks/roles/redis/defaults/main.yml | 4 ++-- playbooks/roles/redis/handlers/main.yml | 4 ++-- playbooks/roles/redis/meta/main.yml | 4 ++-- playbooks/roles/redis/tasks/main.yml | 4 ++-- playbooks/roles/registrar/defaults/main.yml | 4 ++-- playbooks/roles/registrar/meta/main.yml | 4 ++-- playbooks/roles/registrar/tasks/main.yml | 4 ++-- playbooks/roles/s3fs/defaults/main.yml | 4 ++-- playbooks/roles/s3fs/tasks/main.yml | 4 ++-- playbooks/roles/security/defaults/main.yml | 4 ++-- playbooks/roles/security/tasks/main.yml | 4 ++-- .../roles/server_utils/defaults/main.yml | 4 ++-- playbooks/roles/server_utils/meta/main.yml | 4 ++-- playbooks/roles/server_utils/tasks/main.yml | 4 ++-- .../roles/simple_theme/defaults/main.yml | 4 ++-- playbooks/roles/simple_theme/meta/main.yml | 4 ++-- playbooks/roles/simple_theme/tasks/deploy.yml | 4 ++-- .../roles/splunk-server/defaults/main.yml | 4 ++-- playbooks/roles/splunk-server/meta/main.yml | 4 ++-- playbooks/roles/splunk-server/tasks/main.yml | 4 ++-- .../roles/splunkforwarder/defaults/main.yml | 4 ++-- .../roles/splunkforwarder/handlers/main.yml | 4 ++-- .../roles/splunkforwarder/tasks/main.yml | 4 ++-- playbooks/roles/sqoop/defaults/main.yml | 4 ++-- playbooks/roles/sqoop/meta/main.yml | 4 ++-- playbooks/roles/sqoop/tasks/main.yml | 4 ++-- .../stop_all_edx_services/defaults/main.yml | 4 ++-- .../stop_all_edx_services/handlers/main.yml | 4 ++-- .../stop_all_edx_services/tasks/main.yml | 4 ++-- playbooks/roles/supervisor/defaults/main.yml | 4 ++-- playbooks/roles/supervisor/tasks/main.yml | 4 ++-- .../roles/test_build_server/defaults/main.yml | 4 ++-- .../roles/test_build_server/meta/main.yml | 4 ++-- .../roles/test_build_server/tasks/main.yml | 6 ++--- playbooks/roles/testcourses/defaults/main.yml | 4 ++-- playbooks/roles/testcourses/tasks/main.yml | 4 ++-- playbooks/roles/tinymce_plugins/README.rst | 2 +- playbooks/roles/user/defaults/main.yml | 4 ++-- playbooks/roles/user/tasks/main.yml | 4 ++-- .../defaults/main.yml | 4 ++-- .../veda_delivery_worker/defaults/main.yml | 4 ++-- .../roles/veda_delivery_worker/meta/main.yml | 4 ++-- .../roles/veda_delivery_worker/tasks/main.yml | 4 ++-- .../veda_encode_worker/defaults/main.yml | 4 ++-- .../roles/veda_encode_worker/meta/main.yml | 4 ++-- .../roles/veda_encode_worker/tasks/main.yml | 4 ++-- playbooks/roles/veda_ffmpeg/defaults/main.yml | 4 ++-- playbooks/roles/veda_ffmpeg/meta/main.yml | 4 ++-- playbooks/roles/veda_ffmpeg/tasks/main.yml | 4 ++-- .../veda_pipeline_worker/defaults/main.yml | 4 ++-- .../roles/veda_pipeline_worker/meta/main.yml | 4 ++-- .../roles/veda_pipeline_worker/tasks/main.yml | 4 ++-- .../roles/veda_web_frontend/defaults/main.yml | 4 ++-- .../roles/veda_web_frontend/meta/main.yml | 4 ++-- .../roles/veda_web_frontend/tasks/main.yml | 4 ++-- playbooks/roles/vhost/defaults/main.yml | 4 ++-- playbooks/roles/vhost/meta/main.yml | 4 ++-- playbooks/roles/vhost/tasks/main.yml | 4 ++-- .../video_pipeline_base/defaults/main.yml | 4 ++-- .../defaults/main.yml | 4 ++-- .../video_pipeline_integration/tasks/main.yml | 4 ++-- playbooks/roles/xqwatcher/defaults/main.yml | 4 ++-- playbooks/roles/xqwatcher/meta/main.yml | 4 ++-- playbooks/roles/xqwatcher/tasks/code_jail.yml | 2 +- playbooks/roles/xqwatcher/tasks/main.yml | 6 ++--- playbooks/sample_vars/server_vars.yml | 2 +- .../assign_czar_org_groups.py | 4 +--- util/install/ansible-bootstrap.sh | 2 +- util/install/native.sh | 2 +- util/jenkins/get-rc-branches.sh | 2 +- 235 files changed, 482 insertions(+), 484 deletions(-) diff --git a/README.rst b/README.rst index 7f7c6a09c2c..011fa1a6e82 100644 --- a/README.rst +++ b/README.rst @@ -66,4 +66,4 @@ Contributing .. _Open EdX Installation options: https://open.edx.org/installation-options .. _Ansible: http://ansible.com/ .. _OpenEdX Wiki: https://openedx.atlassian.net/wiki/display/OpenOPS/Open+edX+Operations+Home -.. _change log: https://github.com/edx/configuration/blob/master/CHANGELOG.md +.. _change log: https://github.com/openedx/configuration/blob/master/CHANGELOG.md diff --git a/docker/build/analytics_pipeline/Dockerfile b/docker/build/analytics_pipeline/Dockerfile index 3f6e8f1aa89..8cccf22c2e1 100644 --- a/docker/build/analytics_pipeline/Dockerfile +++ b/docker/build/analytics_pipeline/Dockerfile @@ -123,7 +123,7 @@ RUN chown hadoop:hadoop /etc/bootstrap.sh \ # Analytics pipeline ARG OPENEDX_RELEASE=master ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -RUN git clone https://github.com/edx/edx-analytics-pipeline \ +RUN git clone https://github.com/openedx/edx-analytics-pipeline \ && cd edx-analytics-pipeline \ && git checkout ${OPENEDX_RELEASE} \ && cd .. \ diff --git a/docker/build/bionic-common/Dockerfile b/docker/build/bionic-common/Dockerfile index 95cd32cb8a3..611f22fe54c 100644 --- a/docker/build/bionic-common/Dockerfile +++ b/docker/build/bionic-common/Dockerfile @@ -12,7 +12,7 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git" +ENV CONFIGURATION_REPO="https://github.com/openedx/configuration.git" ARG OPENEDX_RELEASE=master ENV CONFIGURATION_VERSION="${OPENEDX_RELEASE}" diff --git a/docker/build/focal-common/Dockerfile b/docker/build/focal-common/Dockerfile index 042f894f300..e999496cee6 100644 --- a/docker/build/focal-common/Dockerfile +++ b/docker/build/focal-common/Dockerfile @@ -16,7 +16,7 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git" +ENV CONFIGURATION_REPO="https://github.com/openedx/configuration.git" ARG OPENEDX_RELEASE=master ENV CONFIGURATION_VERSION="${OPENEDX_RELEASE}" diff --git a/docker/build/trusty-common/Dockerfile b/docker/build/trusty-common/Dockerfile index c1033ca8021..dd58c8e3490 100644 --- a/docker/build/trusty-common/Dockerfile +++ b/docker/build/trusty-common/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:trusty LABEL maintainer="edxops" -ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git" +ENV CONFIGURATION_REPO="https://github.com/openedx/configuration.git" ENV CONFIGURATION_VERSION="master" ADD util/install/ansible-bootstrap.sh /tmp/ansible-bootstrap.sh diff --git a/docker/build/xenial-common/Dockerfile b/docker/build/xenial-common/Dockerfile index a5ba3648752..a2c6cadc529 100644 --- a/docker/build/xenial-common/Dockerfile +++ b/docker/build/xenial-common/Dockerfile @@ -15,7 +15,7 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git" +ENV CONFIGURATION_REPO="https://github.com/openedx/configuration.git" ARG OPENEDX_RELEASE=master ENV CONFIGURATION_VERSION="${OPENEDX_RELEASE}" diff --git a/docker/build/xqwatcher/ansible_overrides.yml b/docker/build/xqwatcher/ansible_overrides.yml index a83502e55fd..ac9fe7dd2f9 100644 --- a/docker/build/xqwatcher/ansible_overrides.yml +++ b/docker/build/xqwatcher/ansible_overrides.yml @@ -5,7 +5,7 @@ FLOCK_TLD: "edx" # there isn't a grader provided in the edx-demo-course yet. XQWATCHER_COURSES: - COURSE: "demo" - GIT_REPO: "https://github.com/edx/edx-demo-course" + GIT_REPO: "https://github.com/openedx/edx-demo-course" GIT_REF: "master" QUEUE_NAME: "test-pull" QUEUE_CONFIG: diff --git a/playbooks/continuous_delivery/rollback_migrations.yml b/playbooks/continuous_delivery/rollback_migrations.yml index 95fb8ef086c..d63d74e6a21 100644 --- a/playbooks/continuous_delivery/rollback_migrations.yml +++ b/playbooks/continuous_delivery/rollback_migrations.yml @@ -2,7 +2,7 @@ # Django project. The provided input file will determine what migrations should be rolled back. # # The playbook uses the Django management commands found in this Django app repo: -# https://github.com/edx/edx-django-release-util +# https://github.com/openedx/edx-django-release-util # So the Django app above needs to be installed in the Django project. # # Required variables for this playbook: diff --git a/playbooks/continuous_delivery/run_migrations.yml b/playbooks/continuous_delivery/run_migrations.yml index 5a32dd81529..a6c73846579 100644 --- a/playbooks/continuous_delivery/run_migrations.yml +++ b/playbooks/continuous_delivery/run_migrations.yml @@ -2,7 +2,7 @@ # Django application. If migrations exist, it will run the migrations while saving the output as an artifact. # # The playbook uses the Django management commands found in this Django app repo: -# https://github.com/edx/edx-django-release-util +# https://github.com/openedx/edx-django-release-util # So the Django app above needs to be installed in the Django app being checked for migrations. # # Required variables for this playbook: diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index a0dd9c0abf2..0e7b12a9747 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Usage: ansible-playbook -i edxapp-host-1, -e@/path/to/configfile-of-users-andor-groups # -e 'group_environment=prod-edge' diff --git a/playbooks/ora2.yml b/playbooks/ora2.yml index 0b5d5378142..4fd52106a8a 100644 --- a/playbooks/ora2.yml +++ b/playbooks/ora2.yml @@ -17,7 +17,7 @@ - edxapp_mysql_password: "password" - supervisorctl_path: "/edx/bin/supervisorctl" - ora2_version: "master" - - ora2_pip_req: "git+https://github.com/edx/edx-ora2.git@{{ ora2_version }}#egg=edx-ora2" + - ora2_pip_req: "git+https://github.com/openedx/edx-ora2.git@{{ ora2_version }}#egg=edx-ora2" tasks: - name: install edx-ora2 diff --git a/playbooks/populate_configuration_model.yml b/playbooks/populate_configuration_model.yml index a674f68d206..a76f224fb5b 100644 --- a/playbooks/populate_configuration_model.yml +++ b/playbooks/populate_configuration_model.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Usage: ansible-playbook -i lms-host-1, ./populate_configuration_model.yml -e "file=/path/to/json/file" -e "user=username" # diff --git a/playbooks/roles/ad_hoc_reporting/defaults/main.yml b/playbooks/roles/ad_hoc_reporting/defaults/main.yml index 56de3d06f22..0df98ea5b5d 100644 --- a/playbooks/roles/ad_hoc_reporting/defaults/main.yml +++ b/playbooks/roles/ad_hoc_reporting/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # diff --git a/playbooks/roles/ad_hoc_reporting/meta/main.yml b/playbooks/roles/ad_hoc_reporting/meta/main.yml index 5a126aa92ab..c54dfb87cce 100644 --- a/playbooks/roles/ad_hoc_reporting/meta/main.yml +++ b/playbooks/roles/ad_hoc_reporting/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role bastion diff --git a/playbooks/roles/ad_hoc_reporting/tasks/main.yml b/playbooks/roles/ad_hoc_reporting/tasks/main.yml index 2b325ee4335..4274661bff3 100644 --- a/playbooks/roles/ad_hoc_reporting/tasks/main.yml +++ b/playbooks/roles/ad_hoc_reporting/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Creates scripts and users to enable ad-hoc reporting using MySQL # read replicas and MongoDB secondary nodes. diff --git a/playbooks/roles/add_user/defaults/main.yml b/playbooks/roles/add_user/defaults/main.yml index 8126c82b0c6..c8f1ec72758 100644 --- a/playbooks/roles/add_user/defaults/main.yml +++ b/playbooks/roles/add_user/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Example play: # diff --git a/playbooks/roles/add_user/meta/main.yml b/playbooks/roles/add_user/meta/main.yml index 68bef5a20a4..8a472fb05c4 100644 --- a/playbooks/roles/add_user/meta/main.yml +++ b/playbooks/roles/add_user/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role add_user diff --git a/playbooks/roles/add_user/tasks/main.yml b/playbooks/roles/add_user/tasks/main.yml index 885ecb7be3a..160b149cab6 100644 --- a/playbooks/roles/add_user/tasks/main.yml +++ b/playbooks/roles/add_user/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index 8bee6c37942..42a200e7d53 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role analytics_api diff --git a/playbooks/roles/analytics_api/meta/main.yml b/playbooks/roles/analytics_api/meta/main.yml index 2b098c31c83..7ab6f3f95ae 100644 --- a/playbooks/roles/analytics_api/meta/main.yml +++ b/playbooks/roles/analytics_api/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Role includes for role analytics_api diff --git a/playbooks/roles/analytics_api/tasks/main.yml b/playbooks/roles/analytics_api/tasks/main.yml index 38632ec4297..4f460d56d8f 100644 --- a/playbooks/roles/analytics_api/tasks/main.yml +++ b/playbooks/roles/analytics_api/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for role analytics_api # diff --git a/playbooks/roles/analytics_pipeline/defaults/main.yml b/playbooks/roles/analytics_pipeline/defaults/main.yml index 0960485f34d..c36fed2feb9 100644 --- a/playbooks/roles/analytics_pipeline/defaults/main.yml +++ b/playbooks/roles/analytics_pipeline/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role analytics_pipeline diff --git a/playbooks/roles/analytics_pipeline/meta/main.yml b/playbooks/roles/analytics_pipeline/meta/main.yml index 75c22b92ba8..98bac28b2b7 100644 --- a/playbooks/roles/analytics_pipeline/meta/main.yml +++ b/playbooks/roles/analytics_pipeline/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role analytics_pipeline diff --git a/playbooks/roles/analytics_pipeline/tasks/main.yml b/playbooks/roles/analytics_pipeline/tasks/main.yml index 3775626cb30..65b42d70279 100644 --- a/playbooks/roles/analytics_pipeline/tasks/main.yml +++ b/playbooks/roles/analytics_pipeline/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/ansible-role-django-ida/templates/header.j2 b/playbooks/roles/ansible-role-django-ida/templates/header.j2 index dbb05d32689..de44282a38b 100644 --- a/playbooks/roles/ansible-role-django-ida/templates/header.j2 +++ b/playbooks/roles/ansible-role-django-ida/templates/header.j2 @@ -1,9 +1,9 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # diff --git a/playbooks/roles/ansible-role/templates/header.j2 b/playbooks/roles/ansible-role/templates/header.j2 index dbb05d32689..de44282a38b 100644 --- a/playbooks/roles/ansible-role/templates/header.j2 +++ b/playbooks/roles/ansible-role/templates/header.j2 @@ -1,9 +1,9 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # diff --git a/playbooks/roles/antivirus/defaults/main.yml b/playbooks/roles/antivirus/defaults/main.yml index 0cf39ccc90b..a5e7f71f583 100644 --- a/playbooks/roles/antivirus/defaults/main.yml +++ b/playbooks/roles/antivirus/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role antivirus diff --git a/playbooks/roles/antivirus/handlers/main.yml b/playbooks/roles/antivirus/handlers/main.yml index 138d971f9b2..978a96d5037 100644 --- a/playbooks/roles/antivirus/handlers/main.yml +++ b/playbooks/roles/antivirus/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/antivirus/meta/main.yml b/playbooks/roles/antivirus/meta/main.yml index 7b30cde9af4..72c3ee8e7d7 100644 --- a/playbooks/roles/antivirus/meta/main.yml +++ b/playbooks/roles/antivirus/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role antivirus diff --git a/playbooks/roles/antivirus/tasks/main.yml b/playbooks/roles/antivirus/tasks/main.yml index 297714ba13f..f44bc10231f 100644 --- a/playbooks/roles/antivirus/tasks/main.yml +++ b/playbooks/roles/antivirus/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/automated/defaults/main.yml b/playbooks/roles/automated/defaults/main.yml index 3d764e347eb..ee7a08b59cc 100644 --- a/playbooks/roles/automated/defaults/main.yml +++ b/playbooks/roles/automated/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Vars for role automated diff --git a/playbooks/roles/automated/meta/main.yml b/playbooks/roles/automated/meta/main.yml index 4f5e0cdfe43..9cbd3d8ae55 100644 --- a/playbooks/roles/automated/meta/main.yml +++ b/playbooks/roles/automated/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Allow this role to be duplicated in dependencies. diff --git a/playbooks/roles/automated/tasks/main.yml b/playbooks/roles/automated/tasks/main.yml index 3e8d3dfc5d1..e92155b9eb9 100644 --- a/playbooks/roles/automated/tasks/main.yml +++ b/playbooks/roles/automated/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for role automated # diff --git a/playbooks/roles/aws/defaults/main.yml b/playbooks/roles/aws/defaults/main.yml index 38781b83f55..f433e3c5a68 100644 --- a/playbooks/roles/aws/defaults/main.yml +++ b/playbooks/roles/aws/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role aws diff --git a/playbooks/roles/aws/meta/main.yml b/playbooks/roles/aws/meta/main.yml index 65896c51ee4..a83bb5c5fc3 100644 --- a/playbooks/roles/aws/meta/main.yml +++ b/playbooks/roles/aws/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role aws diff --git a/playbooks/roles/aws/tasks/main.yml b/playbooks/roles/aws/tasks/main.yml index 8ad20155e64..988a4ae65e0 100644 --- a/playbooks/roles/aws/tasks/main.yml +++ b/playbooks/roles/aws/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/blockstore/defaults/main.yml b/playbooks/roles/blockstore/defaults/main.yml index 6027f3c97f0..0f1713f7863 100644 --- a/playbooks/roles/blockstore/defaults/main.yml +++ b/playbooks/roles/blockstore/defaults/main.yml @@ -1,7 +1,7 @@ --- # Role to deploy Blockstore, the next-generation Open edX Learning Object Repository # -# github: https://github.com/edx/blockstore +# github: https://github.com/openedx/blockstore # blockstore_service_name: 'blockstore' diff --git a/playbooks/roles/blockstore/meta/main.yml b/playbooks/roles/blockstore/meta/main.yml index b2058f15c50..4ba3319b7f4 100644 --- a/playbooks/roles/blockstore/meta/main.yml +++ b/playbooks/roles/blockstore/meta/main.yml @@ -1,7 +1,7 @@ --- # Role to deploy Blockstore, the next-generation Open edX Learning Object Repository # -# github: https://github.com/edx/blockstore +# github: https://github.com/openedx/blockstore # ## # Role includes for role blockstore diff --git a/playbooks/roles/blockstore/tasks/main.yml b/playbooks/roles/blockstore/tasks/main.yml index db8ffcb09db..f1f43e9197d 100644 --- a/playbooks/roles/blockstore/tasks/main.yml +++ b/playbooks/roles/blockstore/tasks/main.yml @@ -1,7 +1,7 @@ --- # Role to deploy Blockstore, the next-generation Open edX Learning Object Repository # -# github: https://github.com/edx/blockstore +# github: https://github.com/openedx/blockstore # # # Tasks for role blockstore diff --git a/playbooks/roles/cassandra/tasks/main.yml b/playbooks/roles/cassandra/tasks/main.yml index e54f4fe82f4..6612c72d147 100644 --- a/playbooks/roles/cassandra/tasks/main.yml +++ b/playbooks/roles/cassandra/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/codejail/tasks/main.yml b/playbooks/roles/codejail/tasks/main.yml index 7894aa38a0e..87ac196986c 100644 --- a/playbooks/roles/codejail/tasks/main.yml +++ b/playbooks/roles/codejail/tasks/main.yml @@ -21,7 +21,7 @@ with_items: "{{ CODEJAIL_PYTHON_VERSIONS }}" - name: Clone codejail repo git: - repo: 'https://github.com/edx/codejail.git' + repo: 'https://github.com/openedx/codejail.git' dest: '/tmp/codejail' version: 'master' - name: Install codejail sandbox dependencies diff --git a/playbooks/roles/commerce_coordinator/defaults/main.yml b/playbooks/roles/commerce_coordinator/defaults/main.yml index 9069ec1c11d..2324e1af4e2 100644 --- a/playbooks/roles/commerce_coordinator/defaults/main.yml +++ b/playbooks/roles/commerce_coordinator/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role commerce_coordinator diff --git a/playbooks/roles/commerce_coordinator/meta/main.yml b/playbooks/roles/commerce_coordinator/meta/main.yml index fd5e39f102b..9f702ec6726 100644 --- a/playbooks/roles/commerce_coordinator/meta/main.yml +++ b/playbooks/roles/commerce_coordinator/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role commerce_coordinator diff --git a/playbooks/roles/commerce_coordinator/tasks/main.yml b/playbooks/roles/commerce_coordinator/tasks/main.yml index 8ebcd1a0080..e146e1d576a 100644 --- a/playbooks/roles/commerce_coordinator/tasks/main.yml +++ b/playbooks/roles/commerce_coordinator/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index d0757ed491e..7ed87040a50 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -260,13 +260,13 @@ COMMON_FAVICON_URL: "https://edx-cdn.org/v3/default/favicon.ico" ############ # Settings related to JSON Web Tokens (JWTs). -# See https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0003-use-jwt-as-oauth-tokens-remove-openid-connect.rst +# See https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0003-use-jwt-as-oauth-tokens-remove-openid-connect.rst COMMON_JWT_AUDIENCE: 'SET-ME-PLEASE' COMMON_JWT_ISSUER: '{{ COMMON_OAUTH_URL_ROOT }}' # The following should be the string representation of a JSON Web Key Set (JWK set) # containing active public keys for signing JWTs. -# See https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst +# See https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst COMMON_JWT_PUBLIC_SIGNING_JWK_SET: '' COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD: 'edx-jwt-cookie-header-payload' diff --git a/playbooks/roles/conductor/defaults/main.yml b/playbooks/roles/conductor/defaults/main.yml index cc2263be26c..3e2f1f595e8 100644 --- a/playbooks/roles/conductor/defaults/main.yml +++ b/playbooks/roles/conductor/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role conductor diff --git a/playbooks/roles/credentials/defaults/main.yml b/playbooks/roles/credentials/defaults/main.yml index 4609382c5b4..d5b3a6af2b7 100644 --- a/playbooks/roles/credentials/defaults/main.yml +++ b/playbooks/roles/credentials/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role credentials diff --git a/playbooks/roles/credentials/meta/main.yml b/playbooks/roles/credentials/meta/main.yml index f95f098be69..3910c275ecd 100644 --- a/playbooks/roles/credentials/meta/main.yml +++ b/playbooks/roles/credentials/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role credentials diff --git a/playbooks/roles/credentials/tasks/main.yml b/playbooks/roles/credentials/tasks/main.yml index bd0630d76f5..5816e28f256 100644 --- a/playbooks/roles/credentials/tasks/main.yml +++ b/playbooks/roles/credentials/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/demo/defaults/main.yml b/playbooks/roles/demo/defaults/main.yml index bcb545c13b5..8fa360a39e0 100644 --- a/playbooks/roles/demo/defaults/main.yml +++ b/playbooks/roles/demo/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role demo diff --git a/playbooks/roles/demo/tasks/main.yml b/playbooks/roles/demo/tasks/main.yml index 150f762c4c7..992099d15f9 100644 --- a/playbooks/roles/demo/tasks/main.yml +++ b/playbooks/roles/demo/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # @@ -14,7 +14,7 @@ # Overview: # # Imports the demo course into studio -# https://github.com/edx/edx-demo-course +# https://github.com/openedx/edx-demo-course # # Once imported this role will only re-import the course # if the edx-demo-course repo has been updated diff --git a/playbooks/roles/designer/defaults/main.yml b/playbooks/roles/designer/defaults/main.yml index 36dc0240515..f991bed1172 100644 --- a/playbooks/roles/designer/defaults/main.yml +++ b/playbooks/roles/designer/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role designer diff --git a/playbooks/roles/designer/meta/main.yml b/playbooks/roles/designer/meta/main.yml index 09d6106ae24..b0fb6fc1711 100644 --- a/playbooks/roles/designer/meta/main.yml +++ b/playbooks/roles/designer/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role ansible-role-django-ida diff --git a/playbooks/roles/designer/tasks/main.yml b/playbooks/roles/designer/tasks/main.yml index 89fe4319c07..21299e9c842 100644 --- a/playbooks/roles/designer/tasks/main.yml +++ b/playbooks/roles/designer/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/devpi/defaults/main.yml b/playbooks/roles/devpi/defaults/main.yml index 4bfc6e88806..367d59d3f85 100644 --- a/playbooks/roles/devpi/defaults/main.yml +++ b/playbooks/roles/devpi/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role devpi diff --git a/playbooks/roles/devpi/tasks/main.yml b/playbooks/roles/devpi/tasks/main.yml index 07639a540e2..2396cd95927 100644 --- a/playbooks/roles/devpi/tasks/main.yml +++ b/playbooks/roles/devpi/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index efd7158a2c0..441b751463f 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role discovery diff --git a/playbooks/roles/discovery/meta/main.yml b/playbooks/roles/discovery/meta/main.yml index 7bb3b3d4867..2fb288fe078 100644 --- a/playbooks/roles/discovery/meta/main.yml +++ b/playbooks/roles/discovery/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role discovery diff --git a/playbooks/roles/discovery/tasks/main.yml b/playbooks/roles/discovery/tasks/main.yml index e9e4dc83519..2f950be92e3 100644 --- a/playbooks/roles/discovery/tasks/main.yml +++ b/playbooks/roles/discovery/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 78a569c31cd..0e3decffe41 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role ecommerce diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index c80ca6ca149..504deab4ce5 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role ecommerce diff --git a/playbooks/roles/ecomworker/defaults/main.yml b/playbooks/roles/ecomworker/defaults/main.yml index 0762227a61b..575df70021c 100644 --- a/playbooks/roles/ecomworker/defaults/main.yml +++ b/playbooks/roles/ecomworker/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Defaults for role ecomworker. # diff --git a/playbooks/roles/ecomworker/meta/main.yml b/playbooks/roles/ecomworker/meta/main.yml index d9922ce387e..851b97f7976 100644 --- a/playbooks/roles/ecomworker/meta/main.yml +++ b/playbooks/roles/ecomworker/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Includes for role ecommerce_worker. # diff --git a/playbooks/roles/ecomworker/tasks/main.yml b/playbooks/roles/ecomworker/tasks/main.yml index 55ac0298b04..c99eb68e6ec 100644 --- a/playbooks/roles/ecomworker/tasks/main.yml +++ b/playbooks/roles/ecomworker/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for role ecommerce_worker. # diff --git a/playbooks/roles/edx-sandbox/tasks/main.yml b/playbooks/roles/edx-sandbox/tasks/main.yml index c83ac00f808..8e13f70e070 100644 --- a/playbooks/roles/edx-sandbox/tasks/main.yml +++ b/playbooks/roles/edx-sandbox/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_ansible/defaults/main.yml b/playbooks/roles/edx_ansible/defaults/main.yml index 2f76a7c46c1..db0c31aef7f 100644 --- a/playbooks/roles/edx_ansible/defaults/main.yml +++ b/playbooks/roles/edx_ansible/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role edx_ansible, an edx_ansible role to install edx_ansible @@ -53,7 +53,7 @@ edx_ansible_venvs_dir: "{{ edx_ansible_app_dir }}/venvs" edx_ansible_venv_dir: "{{ edx_ansible_venvs_dir }}/edx_ansible" edx_ansible_venv_bin: "{{ edx_ansible_venv_dir }}/bin" edx_ansible_user: "edx-ansible" -edx_ansible_source_repo: https://github.com/edx/configuration.git +edx_ansible_source_repo: https://github.com/openedx/configuration.git edx_ansible_requirements_file: "{{ edx_ansible_code_dir }}/requirements.txt" edx_ansible_requirements_files: - "{{ edx_ansible_code_dir }}/pre-requirements.txt" diff --git a/playbooks/roles/edx_ansible/meta/main.yml b/playbooks/roles/edx_ansible/meta/main.yml index 831a159e7f7..5fd7311d2a1 100644 --- a/playbooks/roles/edx_ansible/meta/main.yml +++ b/playbooks/roles/edx_ansible/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx_ansible diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml index b58197b9dfc..ad8265dddbf 100644 --- a/playbooks/roles/edx_ansible/tasks/main.yml +++ b/playbooks/roles/edx_ansible/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Tasks for role edx_ansible diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 00fe4e7e696..62616bbf9ae 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml index 189c6e6bfd8..2ce3960ed56 100644 --- a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml +++ b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_maintenance/defaults/main.yml b/playbooks/roles/edx_maintenance/defaults/main.yml index 9a8e81a1a4e..6eec1fbae8b 100644 --- a/playbooks/roles/edx_maintenance/defaults/main.yml +++ b/playbooks/roles/edx_maintenance/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role edx_maintenance diff --git a/playbooks/roles/edx_maintenance/meta/main.yml b/playbooks/roles/edx_maintenance/meta/main.yml index 761b9524e56..59d8bde041c 100644 --- a/playbooks/roles/edx_maintenance/meta/main.yml +++ b/playbooks/roles/edx_maintenance/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx_maintenance diff --git a/playbooks/roles/edx_maintenance/tasks/main.yml b/playbooks/roles/edx_maintenance/tasks/main.yml index 15ad5eb8372..d23f3589e9e 100644 --- a/playbooks/roles/edx_maintenance/tasks/main.yml +++ b/playbooks/roles/edx_maintenance/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_notes_api/defaults/main.yml b/playbooks/roles/edx_notes_api/defaults/main.yml index a0c045c80fb..bf81f11b35e 100644 --- a/playbooks/roles/edx_notes_api/defaults/main.yml +++ b/playbooks/roles/edx_notes_api/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role edx-notes-api diff --git a/playbooks/roles/edx_notes_api/meta/main.yml b/playbooks/roles/edx_notes_api/meta/main.yml index 2a73bba84b7..5b268a95d51 100644 --- a/playbooks/roles/edx_notes_api/meta/main.yml +++ b/playbooks/roles/edx_notes_api/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx-notes-api diff --git a/playbooks/roles/edx_notes_api/tasks/main.yml b/playbooks/roles/edx_notes_api/tasks/main.yml index 61a8b886274..a412fd81dcc 100644 --- a/playbooks/roles/edx_notes_api/tasks/main.yml +++ b/playbooks/roles/edx_notes_api/tasks/main.yml @@ -2,17 +2,17 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Tasks for role edx-notes-api # # Overview: # -# Role for installing the edx-notes-api Django application, https://github.com/edx/edx-notes-api. +# Role for installing the edx-notes-api Django application, https://github.com/openedx/edx-notes-api. # # Dependencies: # diff --git a/playbooks/roles/edx_service/defaults/main.yml b/playbooks/roles/edx_service/defaults/main.yml index e6e30997725..99eef1aaa7a 100644 --- a/playbooks/roles/edx_service/defaults/main.yml +++ b/playbooks/roles/edx_service/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role edx_service diff --git a/playbooks/roles/edx_service/meta/main.yml b/playbooks/roles/edx_service/meta/main.yml index 69d3d80d087..bacc9826fe3 100644 --- a/playbooks/roles/edx_service/meta/main.yml +++ b/playbooks/roles/edx_service/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx_service diff --git a/playbooks/roles/edx_service/tasks/main.yml b/playbooks/roles/edx_service/tasks/main.yml index 159ad57e93b..54a9d3ff7b4 100644 --- a/playbooks/roles/edx_service/tasks/main.yml +++ b/playbooks/roles/edx_service/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_service_with_rendered_config/defaults/main.yml b/playbooks/roles/edx_service_with_rendered_config/defaults/main.yml index 3ba7d4a3626..a06f7d3f4d3 100644 --- a/playbooks/roles/edx_service_with_rendered_config/defaults/main.yml +++ b/playbooks/roles/edx_service_with_rendered_config/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role edx_service_with_rendered_config diff --git a/playbooks/roles/edx_service_with_rendered_config/meta/main.yml b/playbooks/roles/edx_service_with_rendered_config/meta/main.yml index 1b8767683e5..6ba0b115b12 100644 --- a/playbooks/roles/edx_service_with_rendered_config/meta/main.yml +++ b/playbooks/roles/edx_service_with_rendered_config/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx_service_with_rendered_config diff --git a/playbooks/roles/edx_service_with_rendered_config/tasks/main.yml b/playbooks/roles/edx_service_with_rendered_config/tasks/main.yml index 75ee2a74af8..92bf7342075 100644 --- a/playbooks/roles/edx_service_with_rendered_config/tasks/main.yml +++ b/playbooks/roles/edx_service_with_rendered_config/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edx_themes/defaults/main.yml b/playbooks/roles/edx_themes/defaults/main.yml index f5ac2559005..d7849a0e8dd 100644 --- a/playbooks/roles/edx_themes/defaults/main.yml +++ b/playbooks/roles/edx_themes/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role themes diff --git a/playbooks/roles/edx_themes/meta/main.yml b/playbooks/roles/edx_themes/meta/main.yml index 290037738cb..a5486bb33f6 100644 --- a/playbooks/roles/edx_themes/meta/main.yml +++ b/playbooks/roles/edx_themes/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role edx_themes diff --git a/playbooks/roles/edx_themes/tasks/main.yml b/playbooks/roles/edx_themes/tasks/main.yml index e78f6c29c86..fa2f35b681e 100644 --- a/playbooks/roles/edx_themes/tasks/main.yml +++ b/playbooks/roles/edx_themes/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 9fd0c644af0..ee4bafb4c7e 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -445,7 +445,7 @@ EDXAPP_JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD EDXAPP_JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' EDXAPP_JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' -# See https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst +# See https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst EDXAPP_JWT_SIGNING_ALGORITHM: !!null EDXAPP_JWT_PRIVATE_SIGNING_JWK: !!null @@ -557,17 +557,17 @@ EDXAPP_PRIVATE_REQUIREMENTS: extra_args: -e # This repository contains schoolyourself-xblock, which is used in # edX's "AlgebraX" and "GeometryX" courses. - - name: git+https://github.com/edx/schoolyourself-xblock.git@2093048720cfb36cc05b3143cd6f2585c7c64d85#egg=schoolyourself-xblock + - name: git+https://github.com/openedx/schoolyourself-xblock.git@2093048720cfb36cc05b3143cd6f2585c7c64d85#egg=schoolyourself-xblock extra_args: -e # Prototype XBlocks from edX learning sciences limited roll-outs and user testing. # Concept XBlock, in particular, is nowhere near finished and an early prototype. # Profile XBlock is there so we can play with XBlock arguments in the platform, but isn't ready for use outside of # edX. - - name: git+https://github.com/edx/ConceptXBlock.git@a45a6560c92b6d8b62be1f939ff1d00dfff84e70#egg=concept-xblock + - name: git+https://github.com/openedx/ConceptXBlock.git@a45a6560c92b6d8b62be1f939ff1d00dfff84e70#egg=concept-xblock extra_args: -e - - name: git+https://github.com/edx/AudioXBlock.git@20538c6e9bb704801a71ecbb6981f794556dfc45#egg=audio-xblock + - name: git+https://github.com/openedx/AudioXBlock.git@20538c6e9bb704801a71ecbb6981f794556dfc45#egg=audio-xblock extra_args: -e - - name: git+https://github.com/edx/AnimationXBlock.git@c950ffdda2f69effda93bf03df8646f61d3ffada#egg=animation-xblock + - name: git+https://github.com/openedx/AnimationXBlock.git@c950ffdda2f69effda93bf03df8646f61d3ffada#egg=animation-xblock extra_args: -e # Peer instruction XBlock - name: git+https://github.com/ubc/ubcpi.git@1.0.0#egg=ubcpi-xblock @@ -578,17 +578,17 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/open-craft/xblock-activetable.git@d3fb772435c382b59293e4e688a6a3096c4f6fd7#egg=activetable-xblock extra_args: -e # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - - name: git+https://github.com/edx/xblock-qualtrics-survey.git@02d87f567ec2af4579642cf795bb851507d6edf9#egg=xblock_qualtrics_survey + - name: git+https://github.com/openedx/xblock-qualtrics-survey.git@02d87f567ec2af4579642cf795bb851507d6edf9#egg=xblock_qualtrics_survey extra_args: -e - - name: git+https://github.com/edx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock + - name: git+https://github.com/openedx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock extra_args: -e - - name: git+https://github.com/edx/xblock-submit-and-compare@8f0b3279f36e25aff8cce3f0b1793b9f19dc8729#egg=xblock-submit-and-compare + - name: git+https://github.com/openedx/xblock-submit-and-compare@8f0b3279f36e25aff8cce3f0b1793b9f19dc8729#egg=xblock-submit-and-compare extra_args: -e - - name: git+https://github.com/edx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response + - name: git+https://github.com/openedx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response extra_args: -e - - name: git+https://github.com/edx/xblock-sql-grader@5257b4a1aa07aa572fd3865647d91d0628f0cbd3#egg=xblock-sql-grader + - name: git+https://github.com/openedx/xblock-sql-grader@5257b4a1aa07aa572fd3865647d91d0628f0cbd3#egg=xblock-sql-grader extra_args: -e - - name: git+https://github.com/edx/xblock-image-modal@bef91413447e15570863ab08316a04c0b546b268#egg=xblock-image-modal + - name: git+https://github.com/openedx/xblock-image-modal@bef91413447e15570863ab08316a04c0b546b268#egg=xblock-image-modal extra_args: -e # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks @@ -1541,7 +1541,7 @@ generic_env_config: &edxapp_generic_env SOCIAL_SHARING_SETTINGS: "{{ EDXAPP_SOCIAL_SHARING_SETTINGS }}" SESSION_COOKIE_SECURE: "{{ EDXAPP_SESSION_COOKIE_SECURE }}" CODE_JAIL: - # from https://github.com/edx/codejail/blob/master/codejail/django_integration.py#L24, '' should be same as None + # from https://github.com/openedx/codejail/blob/master/codejail/django_integration.py#L24, '' should be same as None python_bin: '{% if EDXAPP_PYTHON_SANDBOX %}{{ edxapp_sandbox_venv_dir }}/bin/python{% endif %}' limits: "{{ EDXAPP_CODE_JAIL_LIMITS }}" user: '{{ edxapp_sandbox_user }}' diff --git a/playbooks/roles/enhanced_networking/tasks/main.yml b/playbooks/roles/enhanced_networking/tasks/main.yml index 0834f404fac..ceee6f54ce4 100644 --- a/playbooks/roles/enhanced_networking/tasks/main.yml +++ b/playbooks/roles/enhanced_networking/tasks/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/enterprise_catalog/defaults/main.yml b/playbooks/roles/enterprise_catalog/defaults/main.yml index 248f9e20ea9..eb595040554 100644 --- a/playbooks/roles/enterprise_catalog/defaults/main.yml +++ b/playbooks/roles/enterprise_catalog/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role enterprise_catalog diff --git a/playbooks/roles/enterprise_catalog/meta/main.yml b/playbooks/roles/enterprise_catalog/meta/main.yml index af48090ee00..57255fe4748 100644 --- a/playbooks/roles/enterprise_catalog/meta/main.yml +++ b/playbooks/roles/enterprise_catalog/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role enterprise_catalog diff --git a/playbooks/roles/enterprise_catalog/tasks/main.yml b/playbooks/roles/enterprise_catalog/tasks/main.yml index 44f4dcdd5c0..3ffc9be1683 100644 --- a/playbooks/roles/enterprise_catalog/tasks/main.yml +++ b/playbooks/roles/enterprise_catalog/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 8df0032c3a7..d9ad4c19ac7 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -101,7 +101,7 @@ devstack_forum_env: forum_user: "forum" FORUM_RUBY_VERSION: "2.5.7" -forum_source_repo: "https://github.com/edx/cs_comments_service.git" +forum_source_repo: "https://github.com/openedx/cs_comments_service.git" FORUM_VERSION: "master" # diff --git a/playbooks/roles/gh_mirror/defaults/main.yml b/playbooks/roles/gh_mirror/defaults/main.yml index 4b8a83b5799..900a6d5ec0d 100644 --- a/playbooks/roles/gh_mirror/defaults/main.yml +++ b/playbooks/roles/gh_mirror/defaults/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/gh_mirror/tasks/main.yml b/playbooks/roles/gh_mirror/tasks/main.yml index a867024fc39..e0802f98b0a 100644 --- a/playbooks/roles/gh_mirror/tasks/main.yml +++ b/playbooks/roles/gh_mirror/tasks/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/git_clone/defaults/main.yml b/playbooks/roles/git_clone/defaults/main.yml index 892f4997767..8b727eca79e 100644 --- a/playbooks/roles/git_clone/defaults/main.yml +++ b/playbooks/roles/git_clone/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Example play: # diff --git a/playbooks/roles/git_clone/meta/main.yml b/playbooks/roles/git_clone/meta/main.yml index b505d397964..98c26a7a0ee 100644 --- a/playbooks/roles/git_clone/meta/main.yml +++ b/playbooks/roles/git_clone/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role git_clone diff --git a/playbooks/roles/git_clone/tasks/main.yml b/playbooks/roles/git_clone/tasks/main.yml index b3dd6e0cc68..e05ab4cdd05 100644 --- a/playbooks/roles/git_clone/tasks/main.yml +++ b/playbooks/roles/git_clone/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/gitreload/defaults/main.yml b/playbooks/roles/gitreload/defaults/main.yml index 5d45472fb86..4205b7e99a4 100644 --- a/playbooks/roles/gitreload/defaults/main.yml +++ b/playbooks/roles/gitreload/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role gitreload @@ -15,7 +15,7 @@ GITRELOAD_COURSE_CHECKOUT: false GITRELOAD_GIT_IDENTITY: !!null GITRELOAD_REPOS: - name: "edx-demo-course" - url: "https://github.com/edx/edx-demo-course.git" + url: "https://github.com/openedx/edx-demo-course.git" commit: "master" GITRELOAD_REPODIR: "{{ EDXAPP_GIT_REPO_DIR }}" GITRELOAD_LOG_LEVEL: "debug" diff --git a/playbooks/roles/gitreload/handlers/main.yml b/playbooks/roles/gitreload/handlers/main.yml index 6d0869d07b9..0737e499e88 100644 --- a/playbooks/roles/gitreload/handlers/main.yml +++ b/playbooks/roles/gitreload/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/gitreload/meta/main.yml b/playbooks/roles/gitreload/meta/main.yml index 3cacc9e9c61..5ce35da7024 100644 --- a/playbooks/roles/gitreload/meta/main.yml +++ b/playbooks/roles/gitreload/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role gitreload diff --git a/playbooks/roles/gitreload/tasks/main.yml b/playbooks/roles/gitreload/tasks/main.yml index aa6a21a7fef..00c67abadd9 100644 --- a/playbooks/roles/gitreload/tasks/main.yml +++ b/playbooks/roles/gitreload/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/go-agent-docker-server/defaults/main.yml b/playbooks/roles/go-agent-docker-server/defaults/main.yml index 474f58d6b46..b1a3213d3a1 100644 --- a/playbooks/roles/go-agent-docker-server/defaults/main.yml +++ b/playbooks/roles/go-agent-docker-server/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Defaults for role go-agent-docker-server diff --git a/playbooks/roles/go-agent-docker-server/tasks/main.yml b/playbooks/roles/go-agent-docker-server/tasks/main.yml index e381793e5ef..93e507bb50a 100644 --- a/playbooks/roles/go-agent-docker-server/tasks/main.yml +++ b/playbooks/roles/go-agent-docker-server/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/grafana/tasks/main.yml b/playbooks/roles/grafana/tasks/main.yml index 5dfe4ef3476..e45f2c6253a 100644 --- a/playbooks/roles/grafana/tasks/main.yml +++ b/playbooks/roles/grafana/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Tasks for role graphite diff --git a/playbooks/roles/graphite/tasks/main.yml b/playbooks/roles/graphite/tasks/main.yml index e0afc4acb9e..2635768317b 100644 --- a/playbooks/roles/graphite/tasks/main.yml +++ b/playbooks/roles/graphite/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Tasks for role graphite diff --git a/playbooks/roles/hadoop_common/defaults/main.yml b/playbooks/roles/hadoop_common/defaults/main.yml index fed4c5a8106..45d332f7a1f 100644 --- a/playbooks/roles/hadoop_common/defaults/main.yml +++ b/playbooks/roles/hadoop_common/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role hadoop_common diff --git a/playbooks/roles/hadoop_common/meta/main.yml b/playbooks/roles/hadoop_common/meta/main.yml index 747b6871d76..b634f7de7fd 100644 --- a/playbooks/roles/hadoop_common/meta/main.yml +++ b/playbooks/roles/hadoop_common/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role hadoop_common diff --git a/playbooks/roles/hadoop_common/tasks/main.yml b/playbooks/roles/hadoop_common/tasks/main.yml index 520411b45ff..c0ba9733f50 100644 --- a/playbooks/roles/hadoop_common/tasks/main.yml +++ b/playbooks/roles/hadoop_common/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/hadoop_master/defaults/main.yml b/playbooks/roles/hadoop_master/defaults/main.yml index 4e3193d71f6..e639085bff0 100644 --- a/playbooks/roles/hadoop_master/defaults/main.yml +++ b/playbooks/roles/hadoop_master/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role hadoop_master diff --git a/playbooks/roles/hadoop_master/meta/main.yml b/playbooks/roles/hadoop_master/meta/main.yml index 735ae2c7b77..82ce2f9461e 100644 --- a/playbooks/roles/hadoop_master/meta/main.yml +++ b/playbooks/roles/hadoop_master/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role hadoop_master diff --git a/playbooks/roles/hadoop_master/tasks/main.yml b/playbooks/roles/hadoop_master/tasks/main.yml index 0ca0569dc83..122e862f9c4 100644 --- a/playbooks/roles/hadoop_master/tasks/main.yml +++ b/playbooks/roles/hadoop_master/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/haproxy/defaults/main.yml b/playbooks/roles/haproxy/defaults/main.yml index 76d6c74f80c..303bb5f9028 100644 --- a/playbooks/roles/haproxy/defaults/main.yml +++ b/playbooks/roles/haproxy/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role haproxy diff --git a/playbooks/roles/haproxy/handlers/main.yml b/playbooks/roles/haproxy/handlers/main.yml index 9b463490ac3..5c2dcbfaff7 100644 --- a/playbooks/roles/haproxy/handlers/main.yml +++ b/playbooks/roles/haproxy/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/haproxy/meta/main.yml b/playbooks/roles/haproxy/meta/main.yml index 78c5187f5cb..591035273fe 100644 --- a/playbooks/roles/haproxy/meta/main.yml +++ b/playbooks/roles/haproxy/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role haproxy diff --git a/playbooks/roles/haproxy/tasks/main.yml b/playbooks/roles/haproxy/tasks/main.yml index 27f16008ec9..b0c3d649a8f 100644 --- a/playbooks/roles/haproxy/tasks/main.yml +++ b/playbooks/roles/haproxy/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/hermes/defaults/main.yml b/playbooks/roles/hermes/defaults/main.yml index de8dec1899a..03859ab3f91 100644 --- a/playbooks/roles/hermes/defaults/main.yml +++ b/playbooks/roles/hermes/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role hermes diff --git a/playbooks/roles/hermes/meta/main.yml b/playbooks/roles/hermes/meta/main.yml index 78128d26e16..4506eeb7b74 100644 --- a/playbooks/roles/hermes/meta/main.yml +++ b/playbooks/roles/hermes/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role hermes diff --git a/playbooks/roles/hermes/tasks/main.yml b/playbooks/roles/hermes/tasks/main.yml index b9bf1ff2a97..8d5f0222def 100644 --- a/playbooks/roles/hermes/tasks/main.yml +++ b/playbooks/roles/hermes/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/hive/defaults/main.yml b/playbooks/roles/hive/defaults/main.yml index 61b48b05e21..d6d26115c4a 100644 --- a/playbooks/roles/hive/defaults/main.yml +++ b/playbooks/roles/hive/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role hive diff --git a/playbooks/roles/hive/meta/main.yml b/playbooks/roles/hive/meta/main.yml index 206635ebbef..3c267c31f85 100644 --- a/playbooks/roles/hive/meta/main.yml +++ b/playbooks/roles/hive/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role hive diff --git a/playbooks/roles/hive/tasks/main.yml b/playbooks/roles/hive/tasks/main.yml index 88b677f2932..97416fbc25e 100644 --- a/playbooks/roles/hive/tasks/main.yml +++ b/playbooks/roles/hive/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/hotg/defaults/main.yml b/playbooks/roles/hotg/defaults/main.yml index 56ecbbc5c6b..d821e7a41d7 100644 --- a/playbooks/roles/hotg/defaults/main.yml +++ b/playbooks/roles/hotg/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Defaults for role hotg diff --git a/playbooks/roles/hotg/meta/main.yml b/playbooks/roles/hotg/meta/main.yml index 8960d457f41..93185a73030 100644 --- a/playbooks/roles/hotg/meta/main.yml +++ b/playbooks/roles/hotg/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role hotg diff --git a/playbooks/roles/hotg/tasks/main.yml b/playbooks/roles/hotg/tasks/main.yml index cdc8864947a..c048aaa0be0 100644 --- a/playbooks/roles/hotg/tasks/main.yml +++ b/playbooks/roles/hotg/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/insights/defaults/main.yml b/playbooks/roles/insights/defaults/main.yml index 05fa0574588..01021c59fc2 100644 --- a/playbooks/roles/insights/defaults/main.yml +++ b/playbooks/roles/insights/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Defaults for role insights diff --git a/playbooks/roles/insights/meta/main.yml b/playbooks/roles/insights/meta/main.yml index 923f61ba84b..2d2c474f2f7 100644 --- a/playbooks/roles/insights/meta/main.yml +++ b/playbooks/roles/insights/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role insights diff --git a/playbooks/roles/insights/tasks/main.yml b/playbooks/roles/insights/tasks/main.yml index 24c18627a54..e88891095c1 100644 --- a/playbooks/roles/insights/tasks/main.yml +++ b/playbooks/roles/insights/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/jenkins_admin/defaults/main.yml b/playbooks/roles/jenkins_admin/defaults/main.yml index fff3e896c35..218d81edcf4 100644 --- a/playbooks/roles/jenkins_admin/defaults/main.yml +++ b/playbooks/roles/jenkins_admin/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role jenkins_admin diff --git a/playbooks/roles/jenkins_admin/handlers/main.yml b/playbooks/roles/jenkins_admin/handlers/main.yml index 8ff0548e53f..a64427a5122 100644 --- a/playbooks/roles/jenkins_admin/handlers/main.yml +++ b/playbooks/roles/jenkins_admin/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/jenkins_admin/meta/main.yml b/playbooks/roles/jenkins_admin/meta/main.yml index 112c3d3100e..142b900ebd6 100644 --- a/playbooks/roles/jenkins_admin/meta/main.yml +++ b/playbooks/roles/jenkins_admin/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role jenkins_admin diff --git a/playbooks/roles/jenkins_admin/tasks/main.yml b/playbooks/roles/jenkins_admin/tasks/main.yml index 50a600eed50..777cd731284 100644 --- a/playbooks/roles/jenkins_admin/tasks/main.yml +++ b/playbooks/roles/jenkins_admin/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml b/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml index 93f2c5ca130..c5f8ba0da6a 100644 --- a/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml +++ b/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml @@ -4,7 +4,7 @@ - name: Create shallow clone of edx-load-tests git: - repo: https://github.com/edx/edx-load-tests.git + repo: https://github.com/edx-unsupported/edx-load-tests.git dest: "{{ jenkins_home }}/shallow-clone" version: "master" depth: 1 diff --git a/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml b/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml index 4b5cf6acb3a..35c03fd1047 100644 --- a/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml +++ b/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml @@ -7,7 +7,7 @@ # will truncate all history prior to the last revision. - name: Create shallow clone of edx-platform git: - repo: https://github.com/edx/edx-platform.git + repo: https://github.com/openedx/edx-platform.git dest: "{{ jenkins_home }}/shallow-clone" version: "{{ JENKINS_EDX_PLATFORM_VERSION }}" depth: 1 diff --git a/playbooks/roles/learner_portal/defaults/main.yml b/playbooks/roles/learner_portal/defaults/main.yml index 98e263f41d9..3b077848fa1 100644 --- a/playbooks/roles/learner_portal/defaults/main.yml +++ b/playbooks/roles/learner_portal/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # @@ -14,7 +14,7 @@ NVM_DIR: '{{ learner_portal_home }}' learner_portal_user: 'root' learner_portal_git_identity: 'none' edx_django_service_use_python3: false -learner_portal_repo: 'https://github.com/edx/frontend-app-learner-portal.git' +learner_portal_repo: 'https://github.com/openedx/frontend-app-learner-portal-programs.git' LEARNER_PORTAL_VERSION: 'master' learner_portal_service_name: 'learner_portal' LEARNER_PORTAL_NODE_VERSION: '16.14.2' diff --git a/playbooks/roles/license_manager/defaults/main.yml b/playbooks/roles/license_manager/defaults/main.yml index 59986cb8dc4..1b356f7c25a 100644 --- a/playbooks/roles/license_manager/defaults/main.yml +++ b/playbooks/roles/license_manager/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role license_manager diff --git a/playbooks/roles/license_manager/meta/main.yml b/playbooks/roles/license_manager/meta/main.yml index 375e37c93c3..dd95652ffa1 100644 --- a/playbooks/roles/license_manager/meta/main.yml +++ b/playbooks/roles/license_manager/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role license_manager diff --git a/playbooks/roles/license_manager/tasks/main.yml b/playbooks/roles/license_manager/tasks/main.yml index c37f37ef6e7..81c41249564 100644 --- a/playbooks/roles/license_manager/tasks/main.yml +++ b/playbooks/roles/license_manager/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/loadtest_driver/meta/main.yml b/playbooks/roles/loadtest_driver/meta/main.yml index d59f749ea79..d11985cdb9a 100644 --- a/playbooks/roles/loadtest_driver/meta/main.yml +++ b/playbooks/roles/loadtest_driver/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role loadtest_driver diff --git a/playbooks/roles/locust/defaults/main.yml b/playbooks/roles/locust/defaults/main.yml index cee0778bb4b..a2c16efe7bc 100644 --- a/playbooks/roles/locust/defaults/main.yml +++ b/playbooks/roles/locust/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role locust diff --git a/playbooks/roles/locust/meta/main.yml b/playbooks/roles/locust/meta/main.yml index 2807f88c073..499eda3a3e1 100644 --- a/playbooks/roles/locust/meta/main.yml +++ b/playbooks/roles/locust/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role locust diff --git a/playbooks/roles/locust/tasks/main.yml b/playbooks/roles/locust/tasks/main.yml index f75b102cf7f..1517f65014d 100644 --- a/playbooks/roles/locust/tasks/main.yml +++ b/playbooks/roles/locust/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/mariadb/defaults/main.yml b/playbooks/roles/mariadb/defaults/main.yml index 722c824edac..4c002a0075d 100644 --- a/playbooks/roles/mariadb/defaults/main.yml +++ b/playbooks/roles/mariadb/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role mariadb diff --git a/playbooks/roles/mariadb/meta/main.yml b/playbooks/roles/mariadb/meta/main.yml index 2b8b8ba0bf5..a1e245e7f36 100644 --- a/playbooks/roles/mariadb/meta/main.yml +++ b/playbooks/roles/mariadb/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role mariadb diff --git a/playbooks/roles/mariadb/tasks/main.yml b/playbooks/roles/mariadb/tasks/main.yml index 74fdd69063c..75b6663412e 100644 --- a/playbooks/roles/mariadb/tasks/main.yml +++ b/playbooks/roles/mariadb/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/mfe/tasks/main.yml b/playbooks/roles/mfe/tasks/main.yml index fbea317ffcc..cb4aae50257 100644 --- a/playbooks/roles/mfe/tasks/main.yml +++ b/playbooks/roles/mfe/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/mfe_deployer/README.rst b/playbooks/roles/mfe_deployer/README.rst index 39fbf49e56b..8798cb11ec0 100644 --- a/playbooks/roles/mfe_deployer/README.rst +++ b/playbooks/roles/mfe_deployer/README.rst @@ -199,6 +199,6 @@ __________________________________________________________ ECOMMERCE_CORS_ALLOW_CREDENTIALS: true ECOMMERCE_ENABLE_PAYMENT_MFE: true -.. _decision record about asymmetric JWT: https://github.com/edx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst +.. _decision record about asymmetric JWT: https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst .. _Developer Documentation: https://edx.readthedocs.io/projects/edx-developer-docs/en/latest/developers_guide/micro_frontends_in_open_edx.html#overriding-brand-specific-elements -.. _MFE_ENVIRONMENT_DEFAULT: https://github.com/edx/configuration/blob/master/playbooks/roles/mfe/defaults/main.yml#L95 +.. _MFE_ENVIRONMENT_DEFAULT: https://github.com/openedx/configuration/blob/master/playbooks/roles/mfe/defaults/main.yml#L95 diff --git a/playbooks/roles/mfe_flags_setup/tasks/main.yml b/playbooks/roles/mfe_flags_setup/tasks/main.yml index 450d74d0652..d3573060912 100644 --- a/playbooks/roles/mfe_flags_setup/tasks/main.yml +++ b/playbooks/roles/mfe_flags_setup/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/minos/defaults/main.yml b/playbooks/roles/minos/defaults/main.yml index d9c026c9d3c..813d4d7b0a8 100644 --- a/playbooks/roles/minos/defaults/main.yml +++ b/playbooks/roles/minos/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role minos diff --git a/playbooks/roles/minos/meta/main.yml b/playbooks/roles/minos/meta/main.yml index e73b1272d1f..ddb7a456ce9 100644 --- a/playbooks/roles/minos/meta/main.yml +++ b/playbooks/roles/minos/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role minos diff --git a/playbooks/roles/minos/tasks/main.yml b/playbooks/roles/minos/tasks/main.yml index 35b6e043df7..058dd968f8e 100644 --- a/playbooks/roles/minos/tasks/main.yml +++ b/playbooks/roles/minos/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/mount_ebs/tasks/main.yml b/playbooks/roles/mount_ebs/tasks/main.yml index 6b369b784a4..52fe8291ee5 100644 --- a/playbooks/roles/mount_ebs/tasks/main.yml +++ b/playbooks/roles/mount_ebs/tasks/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/munin_node/defaults/main.yml b/playbooks/roles/munin_node/defaults/main.yml index 5fd2c99f826..294825d2c86 100644 --- a/playbooks/roles/munin_node/defaults/main.yml +++ b/playbooks/roles/munin_node/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Defaults for role munin-node diff --git a/playbooks/roles/munin_node/tasks/main.yml b/playbooks/roles/munin_node/tasks/main.yml index dd95b8dbb51..cef144f577b 100644 --- a/playbooks/roles/munin_node/tasks/main.yml +++ b/playbooks/roles/munin_node/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/neo4j/defaults/main.yml b/playbooks/roles/neo4j/defaults/main.yml index 3c0924f38b2..12d7dd74a72 100644 --- a/playbooks/roles/neo4j/defaults/main.yml +++ b/playbooks/roles/neo4j/defaults/main.yml @@ -4,10 +4,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role neo4j @@ -21,7 +21,7 @@ NEO4J_AUTH_ENABLED: "true" # When updating this version, please update the corresponding # neo4j Docker image tag used by the Devstack coursegraph service -# (see github.com/edx/devstack/tree/master/docker-compose.yml). +# (see github.com/openedx/devstack/tree/master/docker-compose.yml). # Note that the corresponding docker image tag does not include the # epoch prefix ('1:') -- it's just 'Major.Minor.Patch'. NEO4J_VERSION: "1:3.5.28" diff --git a/playbooks/roles/neo4j/tasks/main.yml b/playbooks/roles/neo4j/tasks/main.yml index c6e26e7f8a8..1ccd4ac0ea1 100644 --- a/playbooks/roles/neo4j/tasks/main.yml +++ b/playbooks/roles/neo4j/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/newrelic_infrastructure/defaults/main.yml b/playbooks/roles/newrelic_infrastructure/defaults/main.yml index e14a8b6edae..4833da32155 100644 --- a/playbooks/roles/newrelic_infrastructure/defaults/main.yml +++ b/playbooks/roles/newrelic_infrastructure/defaults/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role newrelic_infrastructure diff --git a/playbooks/roles/newrelic_infrastructure/tasks/main.yml b/playbooks/roles/newrelic_infrastructure/tasks/main.yml index fed66ad3f53..49b10ba44c7 100644 --- a/playbooks/roles/newrelic_infrastructure/tasks/main.yml +++ b/playbooks/roles/newrelic_infrastructure/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # newrelic_infrastructure # diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/maps.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/maps.j2 index c4639dda84e..23c371bed02 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/maps.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/maps.j2 @@ -1,7 +1,7 @@ # nginx maps are defined at the top level and are global # # THESE ARE GLOBAL TO ALL IDAs, USE CAUTION WHEN DEFINING HERE -# SEE https://github.com/edx/configuration/pull/5056 FOR A +# SEE https://github.com/openedx/configuration/pull/5056 FOR A # CAUTIONARY TALE # cache header for static files diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 3906b24b31f..6823542f706 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role oauth_client_setup diff --git a/playbooks/roles/oauth_client_setup/tasks/main.yml b/playbooks/roles/oauth_client_setup/tasks/main.yml index 467cdc1c394..a70a96f0a2d 100644 --- a/playbooks/roles/oauth_client_setup/tasks/main.yml +++ b/playbooks/roles/oauth_client_setup/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/openstack/defaults/main.yml b/playbooks/roles/openstack/defaults/main.yml index 06db4e814ab..0b192e217a1 100644 --- a/playbooks/roles/openstack/defaults/main.yml +++ b/playbooks/roles/openstack/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role openstack diff --git a/playbooks/roles/openstack/meta/main.yml b/playbooks/roles/openstack/meta/main.yml index f579538d160..35482345055 100644 --- a/playbooks/roles/openstack/meta/main.yml +++ b/playbooks/roles/openstack/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role openstack diff --git a/playbooks/roles/openstack/tasks/main.yml b/playbooks/roles/openstack/tasks/main.yml index 8a9a7df417f..f0c5b878120 100644 --- a/playbooks/roles/openstack/tasks/main.yml +++ b/playbooks/roles/openstack/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/program_console/defaults/main.yml b/playbooks/roles/program_console/defaults/main.yml index 3c9c932994b..4e84536aaa3 100644 --- a/playbooks/roles/program_console/defaults/main.yml +++ b/playbooks/roles/program_console/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # @@ -21,7 +21,7 @@ program_console_home: '{{ COMMON_APP_DIR }}/{{ program_console_service_name }}' NVM_DIR: '{{ program_console_home }}' program_console_user: 'root' program_console_git_identity: 'none' -program_console_repo: 'https://github.com/edx/frontend-app-program-console.git' +program_console_repo: 'https://github.com/openedx/frontend-app-program-console.git' PROGRAM_CONSOLE_VERSION: 'master' program_console_service_name: 'program-console' PROGRAM_CONSOLE_NODE_VERSION: '12.11.1' diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index b7a9183e19e..0a245116f43 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role prospectus diff --git a/playbooks/roles/rabbitmq/defaults/main.yml b/playbooks/roles/rabbitmq/defaults/main.yml index 7785fc35472..630572de7ae 100644 --- a/playbooks/roles/rabbitmq/defaults/main.yml +++ b/playbooks/roles/rabbitmq/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration -# wiki: https://github.com/edx/configuration/wiki -# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# github: https://github.com/openedx/configuration +# wiki: https://github.com/openedx/configuration/wiki +# code style: https://github.com/openedx/configuration/wiki/Ansible-Coding-Conventions +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Defaults for role rabbitmq diff --git a/playbooks/roles/redis/defaults/main.yml b/playbooks/roles/redis/defaults/main.yml index a8c061f8724..337a31174f3 100644 --- a/playbooks/roles/redis/defaults/main.yml +++ b/playbooks/roles/redis/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role redis diff --git a/playbooks/roles/redis/handlers/main.yml b/playbooks/roles/redis/handlers/main.yml index ad31d60afbf..dbdbf38640c 100644 --- a/playbooks/roles/redis/handlers/main.yml +++ b/playbooks/roles/redis/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/redis/meta/main.yml b/playbooks/roles/redis/meta/main.yml index 7e0f7e4b0c3..4c1995126fa 100644 --- a/playbooks/roles/redis/meta/main.yml +++ b/playbooks/roles/redis/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role redis diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml index f799cdd8113..15a8d74e1ca 100644 --- a/playbooks/roles/redis/tasks/main.yml +++ b/playbooks/roles/redis/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/registrar/defaults/main.yml b/playbooks/roles/registrar/defaults/main.yml index 35674942262..342c93ea9d8 100644 --- a/playbooks/roles/registrar/defaults/main.yml +++ b/playbooks/roles/registrar/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role registrar diff --git a/playbooks/roles/registrar/meta/main.yml b/playbooks/roles/registrar/meta/main.yml index 232279c4923..362772f2627 100644 --- a/playbooks/roles/registrar/meta/main.yml +++ b/playbooks/roles/registrar/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role registrar diff --git a/playbooks/roles/registrar/tasks/main.yml b/playbooks/roles/registrar/tasks/main.yml index 6ae3b656217..bc07f5d8715 100644 --- a/playbooks/roles/registrar/tasks/main.yml +++ b/playbooks/roles/registrar/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/s3fs/defaults/main.yml b/playbooks/roles/s3fs/defaults/main.yml index 55d6193a50a..a38f98e5d79 100644 --- a/playbooks/roles/s3fs/defaults/main.yml +++ b/playbooks/roles/s3fs/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Vars for role s3fs diff --git a/playbooks/roles/s3fs/tasks/main.yml b/playbooks/roles/s3fs/tasks/main.yml index 8c9a351c9d6..325d4947a78 100644 --- a/playbooks/roles/s3fs/tasks/main.yml +++ b/playbooks/roles/s3fs/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for role s3fs # diff --git a/playbooks/roles/security/defaults/main.yml b/playbooks/roles/security/defaults/main.yml index 4699b792cec..03dba03a365 100644 --- a/playbooks/roles/security/defaults/main.yml +++ b/playbooks/roles/security/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role security diff --git a/playbooks/roles/security/tasks/main.yml b/playbooks/roles/security/tasks/main.yml index e3733251ea6..eefa8213ea1 100644 --- a/playbooks/roles/security/tasks/main.yml +++ b/playbooks/roles/security/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/server_utils/defaults/main.yml b/playbooks/roles/server_utils/defaults/main.yml index 8e5d7550aa7..6e0011059d3 100644 --- a/playbooks/roles/server_utils/defaults/main.yml +++ b/playbooks/roles/server_utils/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role server_utils diff --git a/playbooks/roles/server_utils/meta/main.yml b/playbooks/roles/server_utils/meta/main.yml index 45a7881b2bb..68067364fc2 100644 --- a/playbooks/roles/server_utils/meta/main.yml +++ b/playbooks/roles/server_utils/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role insights diff --git a/playbooks/roles/server_utils/tasks/main.yml b/playbooks/roles/server_utils/tasks/main.yml index 5b8d81bf848..0f9e3d38b6c 100644 --- a/playbooks/roles/server_utils/tasks/main.yml +++ b/playbooks/roles/server_utils/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/simple_theme/defaults/main.yml b/playbooks/roles/simple_theme/defaults/main.yml index d04de48f691..ac152d8ea30 100644 --- a/playbooks/roles/simple_theme/defaults/main.yml +++ b/playbooks/roles/simple_theme/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Simple theme. Creates a basic theme at deploy time. diff --git a/playbooks/roles/simple_theme/meta/main.yml b/playbooks/roles/simple_theme/meta/main.yml index 05ec022c02b..8ee711cbfcb 100644 --- a/playbooks/roles/simple_theme/meta/main.yml +++ b/playbooks/roles/simple_theme/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # dependencies: [] diff --git a/playbooks/roles/simple_theme/tasks/deploy.yml b/playbooks/roles/simple_theme/tasks/deploy.yml index a541fc0ccaf..759f3016b6f 100644 --- a/playbooks/roles/simple_theme/tasks/deploy.yml +++ b/playbooks/roles/simple_theme/tasks/deploy.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/splunk-server/defaults/main.yml b/playbooks/roles/splunk-server/defaults/main.yml index af2ef2acd48..2c8f46190c3 100644 --- a/playbooks/roles/splunk-server/defaults/main.yml +++ b/playbooks/roles/splunk-server/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role splunk-server diff --git a/playbooks/roles/splunk-server/meta/main.yml b/playbooks/roles/splunk-server/meta/main.yml index 86943d37869..9d50d334212 100644 --- a/playbooks/roles/splunk-server/meta/main.yml +++ b/playbooks/roles/splunk-server/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role splunk-server diff --git a/playbooks/roles/splunk-server/tasks/main.yml b/playbooks/roles/splunk-server/tasks/main.yml index 1656aa72517..dcfee03d575 100644 --- a/playbooks/roles/splunk-server/tasks/main.yml +++ b/playbooks/roles/splunk-server/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/splunkforwarder/defaults/main.yml b/playbooks/roles/splunkforwarder/defaults/main.yml index a6ce3707bd9..7cba8d8b96d 100644 --- a/playbooks/roles/splunkforwarder/defaults/main.yml +++ b/playbooks/roles/splunkforwarder/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Vars for role splunk diff --git a/playbooks/roles/splunkforwarder/handlers/main.yml b/playbooks/roles/splunkforwarder/handlers/main.yml index 07c62e9634d..a342f1fc209 100644 --- a/playbooks/roles/splunkforwarder/handlers/main.yml +++ b/playbooks/roles/splunkforwarder/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/splunkforwarder/tasks/main.yml b/playbooks/roles/splunkforwarder/tasks/main.yml index 668e72faf78..e55c4e0a54f 100644 --- a/playbooks/roles/splunkforwarder/tasks/main.yml +++ b/playbooks/roles/splunkforwarder/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/sqoop/defaults/main.yml b/playbooks/roles/sqoop/defaults/main.yml index a655924b338..0ffe6e8f355 100644 --- a/playbooks/roles/sqoop/defaults/main.yml +++ b/playbooks/roles/sqoop/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role sqoop diff --git a/playbooks/roles/sqoop/meta/main.yml b/playbooks/roles/sqoop/meta/main.yml index 69e87d311b9..214928bec94 100644 --- a/playbooks/roles/sqoop/meta/main.yml +++ b/playbooks/roles/sqoop/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role sqoop diff --git a/playbooks/roles/sqoop/tasks/main.yml b/playbooks/roles/sqoop/tasks/main.yml index 9810ccd14f2..cb09b5405f1 100644 --- a/playbooks/roles/sqoop/tasks/main.yml +++ b/playbooks/roles/sqoop/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/stop_all_edx_services/defaults/main.yml b/playbooks/roles/stop_all_edx_services/defaults/main.yml index 0bf549f7672..792056a197f 100644 --- a/playbooks/roles/stop_all_edx_services/defaults/main.yml +++ b/playbooks/roles/stop_all_edx_services/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role stop_all_edx_services diff --git a/playbooks/roles/stop_all_edx_services/handlers/main.yml b/playbooks/roles/stop_all_edx_services/handlers/main.yml index 61e781e2b5c..977bdb89b44 100644 --- a/playbooks/roles/stop_all_edx_services/handlers/main.yml +++ b/playbooks/roles/stop_all_edx_services/handlers/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/stop_all_edx_services/tasks/main.yml b/playbooks/roles/stop_all_edx_services/tasks/main.yml index 38520a69ebe..e6934b1f9cf 100644 --- a/playbooks/roles/stop_all_edx_services/tasks/main.yml +++ b/playbooks/roles/stop_all_edx_services/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/supervisor/defaults/main.yml b/playbooks/roles/supervisor/defaults/main.yml index eb669098757..75c00e79943 100644 --- a/playbooks/roles/supervisor/defaults/main.yml +++ b/playbooks/roles/supervisor/defaults/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/supervisor/tasks/main.yml b/playbooks/roles/supervisor/tasks/main.yml index b4cd1dd939c..ee9c1158f4d 100644 --- a/playbooks/roles/supervisor/tasks/main.yml +++ b/playbooks/roles/supervisor/tasks/main.yml @@ -1,10 +1,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/test_build_server/defaults/main.yml b/playbooks/roles/test_build_server/defaults/main.yml index 6b68b5aa26b..300843ef990 100644 --- a/playbooks/roles/test_build_server/defaults/main.yml +++ b/playbooks/roles/test_build_server/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role test_build_server diff --git a/playbooks/roles/test_build_server/meta/main.yml b/playbooks/roles/test_build_server/meta/main.yml index b0c4bc07b59..39c023632c8 100644 --- a/playbooks/roles/test_build_server/meta/main.yml +++ b/playbooks/roles/test_build_server/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role test_build_server diff --git a/playbooks/roles/test_build_server/tasks/main.yml b/playbooks/roles/test_build_server/tasks/main.yml index 0ecc3912c12..d2a93b8aeea 100644 --- a/playbooks/roles/test_build_server/tasks/main.yml +++ b/playbooks/roles/test_build_server/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # @@ -22,7 +22,7 @@ - name: Create clone of edx-platform git: - repo: "https://github.com/edx/edx-platform.git" + repo: "https://github.com/openedx/edx-platform.git" dest: "{{ test_build_server_repo_path }}/edx-platform-clone" version: "{{ TEST_EDX_PLATFORM_VERSION }}" become_user: "{{ test_build_server_user }}" diff --git a/playbooks/roles/testcourses/defaults/main.yml b/playbooks/roles/testcourses/defaults/main.yml index 355b4838a43..6ef7ed67a12 100644 --- a/playbooks/roles/testcourses/defaults/main.yml +++ b/playbooks/roles/testcourses/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role testcourses diff --git a/playbooks/roles/testcourses/tasks/main.yml b/playbooks/roles/testcourses/tasks/main.yml index f6c869d1ea3..b7054c67db2 100644 --- a/playbooks/roles/testcourses/tasks/main.yml +++ b/playbooks/roles/testcourses/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/tinymce_plugins/README.rst b/playbooks/roles/tinymce_plugins/README.rst index 35210fa4de7..ae9660704e5 100644 --- a/playbooks/roles/tinymce_plugins/README.rst +++ b/playbooks/roles/tinymce_plugins/README.rst @@ -44,4 +44,4 @@ Enabling Plugins There's a decent `guide on enabling the plugins through the edX platform`_, specifically using the ``TINYMCE_ADDITIONAL_PLUGINS`` extra JavaScript configuration. -.. _guide on enabling the plugins through the edX platform: https://github.com/edx/edx-platform/blob/master/docs/guides/extensions/tinymce_plugins.rst +.. _guide on enabling the plugins through the edX platform: https://github.com/openedx/edx-platform/blob/master/docs/guides/extensions/tinymce_plugins.rst diff --git a/playbooks/roles/user/defaults/main.yml b/playbooks/roles/user/defaults/main.yml index 6371dc0d73a..0378c70ffda 100644 --- a/playbooks/roles/user/defaults/main.yml +++ b/playbooks/roles/user/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Vars for role user diff --git a/playbooks/roles/user/tasks/main.yml b/playbooks/roles/user/tasks/main.yml index 22cfb259015..6c316d67ff1 100644 --- a/playbooks/roles/user/tasks/main.yml +++ b/playbooks/roles/user/tasks/main.yml @@ -2,10 +2,10 @@ # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for role user # diff --git a/playbooks/roles/user_retirement_pipeline/defaults/main.yml b/playbooks/roles/user_retirement_pipeline/defaults/main.yml index d648d69d569..aaf7fffedb1 100644 --- a/playbooks/roles/user_retirement_pipeline/defaults/main.yml +++ b/playbooks/roles/user_retirement_pipeline/defaults/main.yml @@ -2,10 +2,10 @@ # # Open edX Retirement Pipeline Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # Deploy the User Retirement Pipeline diff --git a/playbooks/roles/veda_delivery_worker/defaults/main.yml b/playbooks/roles/veda_delivery_worker/defaults/main.yml index ff46d527a53..263b7e68544 100644 --- a/playbooks/roles/veda_delivery_worker/defaults/main.yml +++ b/playbooks/roles/veda_delivery_worker/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for veda_delivery_worker role diff --git a/playbooks/roles/veda_delivery_worker/meta/main.yml b/playbooks/roles/veda_delivery_worker/meta/main.yml index 56950d0f6b4..bceab068736 100644 --- a/playbooks/roles/veda_delivery_worker/meta/main.yml +++ b/playbooks/roles/veda_delivery_worker/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for veda_delivery_worker role diff --git a/playbooks/roles/veda_delivery_worker/tasks/main.yml b/playbooks/roles/veda_delivery_worker/tasks/main.yml index e5a3ac4bd3a..5914c6cb8c5 100644 --- a/playbooks/roles/veda_delivery_worker/tasks/main.yml +++ b/playbooks/roles/veda_delivery_worker/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for veda_delivery_worker role # diff --git a/playbooks/roles/veda_encode_worker/defaults/main.yml b/playbooks/roles/veda_encode_worker/defaults/main.yml index 92e591d1a62..a932a510909 100644 --- a/playbooks/roles/veda_encode_worker/defaults/main.yml +++ b/playbooks/roles/veda_encode_worker/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for veda_encode_worker role diff --git a/playbooks/roles/veda_encode_worker/meta/main.yml b/playbooks/roles/veda_encode_worker/meta/main.yml index 74ecad7ab26..48d582d5750 100644 --- a/playbooks/roles/veda_encode_worker/meta/main.yml +++ b/playbooks/roles/veda_encode_worker/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for veda_encode_worker role diff --git a/playbooks/roles/veda_encode_worker/tasks/main.yml b/playbooks/roles/veda_encode_worker/tasks/main.yml index 192c9dee141..6dc6bb9b27d 100644 --- a/playbooks/roles/veda_encode_worker/tasks/main.yml +++ b/playbooks/roles/veda_encode_worker/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for veda_encode_worker role # diff --git a/playbooks/roles/veda_ffmpeg/defaults/main.yml b/playbooks/roles/veda_ffmpeg/defaults/main.yml index 54b4e6a7ef2..d730fc02bd0 100644 --- a/playbooks/roles/veda_ffmpeg/defaults/main.yml +++ b/playbooks/roles/veda_ffmpeg/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for veda_ffmpeg role diff --git a/playbooks/roles/veda_ffmpeg/meta/main.yml b/playbooks/roles/veda_ffmpeg/meta/main.yml index 82d09cf8747..6236747e3d2 100644 --- a/playbooks/roles/veda_ffmpeg/meta/main.yml +++ b/playbooks/roles/veda_ffmpeg/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Dependencies for veda_ffmpeg role # diff --git a/playbooks/roles/veda_ffmpeg/tasks/main.yml b/playbooks/roles/veda_ffmpeg/tasks/main.yml index ceb2b07aec5..d5f2b9833eb 100644 --- a/playbooks/roles/veda_ffmpeg/tasks/main.yml +++ b/playbooks/roles/veda_ffmpeg/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for veda_ffmpeg role # diff --git a/playbooks/roles/veda_pipeline_worker/defaults/main.yml b/playbooks/roles/veda_pipeline_worker/defaults/main.yml index 018d4c2c2b1..c8ad796fd42 100644 --- a/playbooks/roles/veda_pipeline_worker/defaults/main.yml +++ b/playbooks/roles/veda_pipeline_worker/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for veda_pipeline_worker role diff --git a/playbooks/roles/veda_pipeline_worker/meta/main.yml b/playbooks/roles/veda_pipeline_worker/meta/main.yml index 3fa6c13fa92..13d2fd431b0 100644 --- a/playbooks/roles/veda_pipeline_worker/meta/main.yml +++ b/playbooks/roles/veda_pipeline_worker/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for veda_pipeline_worker role diff --git a/playbooks/roles/veda_pipeline_worker/tasks/main.yml b/playbooks/roles/veda_pipeline_worker/tasks/main.yml index c17ebbb2520..ae5de3a6b36 100644 --- a/playbooks/roles/veda_pipeline_worker/tasks/main.yml +++ b/playbooks/roles/veda_pipeline_worker/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # Tasks for veda_pipeline_worker role # diff --git a/playbooks/roles/veda_web_frontend/defaults/main.yml b/playbooks/roles/veda_web_frontend/defaults/main.yml index b00890664d0..87337dd18de 100644 --- a/playbooks/roles/veda_web_frontend/defaults/main.yml +++ b/playbooks/roles/veda_web_frontend/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role veda_web_frontend diff --git a/playbooks/roles/veda_web_frontend/meta/main.yml b/playbooks/roles/veda_web_frontend/meta/main.yml index cb2f17598c8..0cb8a8ae558 100644 --- a/playbooks/roles/veda_web_frontend/meta/main.yml +++ b/playbooks/roles/veda_web_frontend/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role veda_web_frontend diff --git a/playbooks/roles/veda_web_frontend/tasks/main.yml b/playbooks/roles/veda_web_frontend/tasks/main.yml index 9240e2e37b8..0d67dbd7676 100644 --- a/playbooks/roles/veda_web_frontend/tasks/main.yml +++ b/playbooks/roles/veda_web_frontend/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Tasks for role veda_web_frontend diff --git a/playbooks/roles/vhost/defaults/main.yml b/playbooks/roles/vhost/defaults/main.yml index 51ebe49de62..8c8fa266d6f 100644 --- a/playbooks/roles/vhost/defaults/main.yml +++ b/playbooks/roles/vhost/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role vhost diff --git a/playbooks/roles/vhost/meta/main.yml b/playbooks/roles/vhost/meta/main.yml index 25883a06a8b..932980a4d7e 100644 --- a/playbooks/roles/vhost/meta/main.yml +++ b/playbooks/roles/vhost/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role vhost diff --git a/playbooks/roles/vhost/tasks/main.yml b/playbooks/roles/vhost/tasks/main.yml index 5eec3ce06e6..1374ca892a2 100644 --- a/playbooks/roles/vhost/tasks/main.yml +++ b/playbooks/roles/vhost/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/video_pipeline_base/defaults/main.yml b/playbooks/roles/video_pipeline_base/defaults/main.yml index 52fead31178..6ded12f6692 100644 --- a/playbooks/roles/video_pipeline_base/defaults/main.yml +++ b/playbooks/roles/video_pipeline_base/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for common variables for video_pipeline. This is the base role for all video pipeline roles. diff --git a/playbooks/roles/video_pipeline_integration/defaults/main.yml b/playbooks/roles/video_pipeline_integration/defaults/main.yml index 53f0ea700d0..96b46051ce5 100644 --- a/playbooks/roles/video_pipeline_integration/defaults/main.yml +++ b/playbooks/roles/video_pipeline_integration/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role video_pipeline_integration diff --git a/playbooks/roles/video_pipeline_integration/tasks/main.yml b/playbooks/roles/video_pipeline_integration/tasks/main.yml index 62436004252..29ab9ea0cad 100644 --- a/playbooks/roles/video_pipeline_integration/tasks/main.yml +++ b/playbooks/roles/video_pipeline_integration/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # diff --git a/playbooks/roles/xqwatcher/defaults/main.yml b/playbooks/roles/xqwatcher/defaults/main.yml index dc47bbe229a..d79b0551ce6 100644 --- a/playbooks/roles/xqwatcher/defaults/main.yml +++ b/playbooks/roles/xqwatcher/defaults/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Defaults for role xqwatcher diff --git a/playbooks/roles/xqwatcher/meta/main.yml b/playbooks/roles/xqwatcher/meta/main.yml index 1c8f0b9d238..90773791dc1 100644 --- a/playbooks/roles/xqwatcher/meta/main.yml +++ b/playbooks/roles/xqwatcher/meta/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # ## # Role includes for role xqwatcher diff --git a/playbooks/roles/xqwatcher/tasks/code_jail.yml b/playbooks/roles/xqwatcher/tasks/code_jail.yml index ce7186e9eae..9ef714f5ef3 100644 --- a/playbooks/roles/xqwatcher/tasks/code_jail.yml +++ b/playbooks/roles/xqwatcher/tasks/code_jail.yml @@ -122,7 +122,7 @@ when: CODE_JAIL_COMPLAIN|bool with_items: "{{ XQWATCHER_COURSES }}" # AA having issues on 14.04 - # https://github.com/edx/codejail/issues/38 + # https://github.com/openedx/codejail/issues/38 tags: - manage - manage:sandbox diff --git a/playbooks/roles/xqwatcher/tasks/main.yml b/playbooks/roles/xqwatcher/tasks/main.yml index 279f5eb711c..a755b2b66b5 100644 --- a/playbooks/roles/xqwatcher/tasks/main.yml +++ b/playbooks/roles/xqwatcher/tasks/main.yml @@ -2,10 +2,10 @@ # # edX Configuration # -# github: https://github.com/edx/configuration +# github: https://github.com/openedx/configuration # wiki: https://openedx.atlassian.net/wiki/display/OpenOPS # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/edx/configuration/blob/master/LICENSE.TXT +# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT # # # @@ -14,7 +14,7 @@ # Overview: # # This play installs a sandboxed, pull grader that integrates with XQueue. The -# code for the XQWatcher lives here: https://github.com/edx/xqueue-watcher +# code for the XQWatcher lives here: https://github.com/openedx/xqueue-watcher # # Multiple courses can be installed on a single server with distinct sandboxes. # diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index 291af0ecb77..99d9e125aab 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -158,7 +158,7 @@ #ANALYTICS_API_GIT_IDENTITY: "{{ _local_git_identity }}" # #TESTCOURSES_EXPORTS: -# - github_url: "https://github.com/edx/edx-demo-course.git" +# - github_url: "https://github.com/openedx/edx-demo-course.git" # install: "{{ ENABLE_EDX_DEMO_COURSE }}" # course_id: "course-v1:edX+DemoX+Demo_Course" # diff --git a/util/create_data_czar/assign_czar_org_groups.py b/util/create_data_czar/assign_czar_org_groups.py index f5a30b340c6..a3336fa570b 100755 --- a/util/create_data_czar/assign_czar_org_groups.py +++ b/util/create_data_czar/assign_czar_org_groups.py @@ -2,9 +2,7 @@ assign_czar_org_groups.py Assigns data czars to the iam groups for their org based on the configuration specificed in the analytics-exporter -repository, https://github.com/edx/analytics-exporter/blob/master/config/default.yaml. - -The file needs to be available locally as the repo is private. +repository, https://github.com/openedx/edx-analytics-exporter/blob/master/sample-config.yaml Assumes that a group for the org has already been created using the create_org_data_czar_polcy.py script. diff --git a/util/install/ansible-bootstrap.sh b/util/install/ansible-bootstrap.sh index fafc80b108e..c1e17464acc 100755 --- a/util/install/ansible-bootstrap.sh +++ b/util/install/ansible-bootstrap.sh @@ -14,7 +14,7 @@ set -xe if [[ -z "${CONFIGURATION_REPO}" ]]; then - CONFIGURATION_REPO="https://github.com/edx/configuration.git" + CONFIGURATION_REPO="https://github.com/openedx/configuration.git" fi if [[ -z "${CONFIGURATION_VERSION}" ]]; then diff --git a/util/install/native.sh b/util/install/native.sh index ec37a281618..d8432b062d9 100644 --- a/util/install/native.sh +++ b/util/install/native.sh @@ -148,7 +148,7 @@ CONFIGURATION_VERSION=${CONFIGURATION_VERSION-$OPENEDX_RELEASE} ## Clone the configuration repository and run Ansible ## cd /var/tmp -git clone https://github.com/edx/configuration +git clone https://github.com/openedx/configuration cd configuration git checkout $CONFIGURATION_VERSION git pull diff --git a/util/jenkins/get-rc-branches.sh b/util/jenkins/get-rc-branches.sh index 8a0de7fc8ca..d92c2795ece 100755 --- a/util/jenkins/get-rc-branches.sh +++ b/util/jenkins/get-rc-branches.sh @@ -17,7 +17,7 @@ usage() { -r repo to look in -f filter string for branch list - Example: $prog -r https://github.com/edx/edx-platform -f "rc/" + Example: $prog -r https://github.com/openedx/edx-platform -f "rc/" EOF } From 975e0d5c31e4a8eced04d171734f4af20db3a70c Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Sat, 10 Sep 2022 17:40:12 -0400 Subject: [PATCH 344/664] fix: update path to .github workflows to read from openedx org --- .github/workflows/commitlint.yml | 2 +- .github/workflows/upgrade-python-requirements.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index e2b066153f2..fec11d6c259 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -7,4 +7,4 @@ on: jobs: commitlint: - uses: edx/.github/.github/workflows/commitlint.yml@master + uses: openedx/.github/.github/workflows/commitlint.yml@master diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index 92fa30981d5..fcdad5c826e 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -21,4 +21,4 @@ jobs: requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} - uses: edx/.github/.github/workflows/upgrade-python-requirements.yml@master + uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master From 2a454408993faf7704f7e3ab2afa1cd9e2c66812 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 3 Oct 2022 11:15:40 +0500 Subject: [PATCH 345/664] feat: install edx-exams on sandbox using docker --- playbooks/edx_continuous_integration.yml | 1 + playbooks/roles/launch_ec2/tasks/main.yml | 2 +- util/jenkins/ansible-provision.sh | 109 ++++++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index abc1ded71d1..97f6f24c0fa 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -18,6 +18,7 @@ - prospectus nginx_default_sites: - lms + - docker-tools - mysql - role: edxlocal tags: edxlocal diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index d4975b76308..63c0d3d2814 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,7 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator', 'edx-exams'] - name: Add new instance to host group local_action: diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 2ac24205b99..b0ba1db08b3 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -655,5 +655,114 @@ if [[ $enable_newrelic == "true" ]]; then run_ansible run_role.yml -i "${deploy_host}," -e role=newrelic_infrastructure $extra_var_arg --user ubuntu fi +function provision_containerized_app() { + echo "#!/usr/bin/env bash" + echo "set -ex" + + # Create app staticfiles dir + echo "mkdir /edx/var/${app_service_name}/staticfiles/ -p && chmod 777 /edx/var/${app_service_name} -R" + + # Checkout code in app directory + echo "cd /edx/app/" + echo "git clone https://github.com/edx/${app_repo}.git" + + # Replace deploy_host in app config file with sandbox DNS name + echo "sed -i 's/deploy_host/${dns_name}.${dns_zone}/g' /var/tmp/${app_service_name}.yml" + + # Install yq for yaml processing + echo "wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq" + + # Combine app config with jwt_signature config + echo "yq eval-all '. as \$item ireduce ({}; . *+ \$item)' /var/tmp/${app_service_name}.yml /tmp/lms_jwt_signature.yml > /edx/etc/${app_service_name}.yml" + + # Provision IDA User in LMS + echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production manage_user ${app_service_name}_worker ${app_service_name}_worker@example.com --staff --superuser" + + # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. + echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris 'https://${app_hostname}-${dns_name}.${dns_zone}/complete/edx-oauth2/' --client-id '${app_service_name}-sso-key' --client-secret '${app_service_name}-sso-secret' --scopes 'user_id' ${app_service_name}-sso ${app_service_name}_worker" + echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production create_dot_application --grant-type client-credentials --client-id '${app_service_name}-backend-service-key' --client-secret '${app_service_name}-backend-service-secret' ${app_service_name}-backend-service ${app_service_name}_worker" + + # Checkout code version + echo "cd /edx/app/${app_repo}" + echo "git checkout ${app_version}" + + # Create app database + echo "mysql -uroot -e \"CREATE DATABASE \\\`${app_service_name}\\\`;\"" + + # Build app docker image and run app container + echo "docker build . -t ${app_service_name}:latest" + echo "docker run --rm -it -d -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -e ${app_cfg}=/${app_service_name}.yml -v '/edx/app/${app_repo}:/edx/app/${app_repo}/' -v '/edx/etc/${app_service_name}.yml:/${app_service_name}.yml' -v '/edx/var/${app_service_name}/staticfiles/:/var/tmp/' --name ${app_service_name}.app --network=host --entrypoint '/bin/bash' ${app_service_name}:latest -c 'while true; do exec gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_repo}/${app_service_name}/docker_gunicorn_configuration.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application; sleep 2; done'" + + # Wait for app container + echo "sleep 5" + + # Run migrations + echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py migrate\"" + + # Run collectstatic + echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py collectstatic --noinput\"" + + # Create superuser + echo "docker exec -t ${app_service_name}.app bash -c \"echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\\\"edx\\\", \\\"edx@example.com\\\", \\\"edx\\\") if not User.objects.filter(username=\\\"edx\\\").exists() else None' | python /edx/app/${app_repo}/manage.py shell\"" + + # Create Nginx config + echo "site_config=/edx/app/nginx/sites-available/${app_service_name}" + echo "cat << 'EOF' > \$site_config + server { + server_name ~^((stage|prod)-)?${app_hostname}.*; + listen 80; + rewrite ^ https://\$host\$request_uri? permanent; + } + server { + server_name ~^((stage|prod)-)?${app_hostname}.*; + listen 443 ssl; + ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; + ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; + + location / { + try_files \$uri @proxy_to_app; + } + location ~ ^/(api)/ { + try_files \$uri @proxy_to_app; + } + location @proxy_to_app { + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header X-Forwarded-Port \$server_port; + proxy_set_header X-Forwarded-For \$remote_addr; + proxy_set_header Host \$http_host; + proxy_redirect off; + proxy_pass http://127.0.0.1:${app_gunicorn_port}; + } + location ~ ^/static/(?P.*) { + root /edx/var/${app_service_name}; + try_files /staticfiles/\$file =404; + } + } +EOF" + echo "ln -s /edx/app/nginx/sites-available/${app_service_name} /etc/nginx/sites-enabled/${app_service_name}" + echo "service nginx reload" +} + +if [[ $edx_exams == 'true' ]]; then + + app_hostname="edx-exams" + app_service_name="edx_exams" + app_repo="edx-exams" + app_version=$edx_exams_version + app_gunicorn_port=18740 + app_cfg=EDX_EXAMS_CFG + + provision_script="/var/tmp/provision-script-$$.sh" +cat << EOF > $provision_script +$(provision_containerized_app) +EOF + + # copy app config file and run script to deploy app + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${WORKSPACE}/configuration-internal/k8s-sandbox-config/${app_service_name}.yml dest=/var/tmp/${app_service_name}.yml" -u ubuntu -b + ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_script}" -u ubuntu -b + + rm -f "${provision_script}" +fi + rm -f "$extra_vars_file" rm -f ${extra_vars_file}_clean From 2cb7b007488329a6263a448a82e6946545c50111 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 3 Oct 2022 12:58:41 +0500 Subject: [PATCH 346/664] chore: Add condition to check if in docker for rsyslogd --- playbooks/roles/common/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 927a0c0e42c..c51564cc21d 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -217,6 +217,11 @@ - { src: 'etc/cron.hourly/logrotate.j2', dest: '/etc/cron.hourly/logrotate', mode: '0555' } - { src: 'etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2', dest: '/etc/logrotate.d/hourly/tracking.log' } +- name: Are we in a Docker container + shell: echo $(egrep -q 'docker' /proc/self/cgroup && echo 'yes' || echo 'no') + ignore_errors: yes + register: docker_container + # TODO: restarts no matter which template has changed, need to examine # the results - name: restart rsyslogd @@ -224,7 +229,7 @@ name: rsyslog state: restarted become: True - when: config_templates.changed + when: config_templates.changed and docker_container.stdout != 'no' - name: Add ntp alert script template: From 9b4630c5cb44e1695f3fd0e49a30538058046ce7 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 3 Oct 2022 14:08:12 +0500 Subject: [PATCH 347/664] fix: Fix rsys task condition --- playbooks/roles/common/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index c51564cc21d..b406c55a021 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -229,7 +229,7 @@ name: rsyslog state: restarted become: True - when: config_templates.changed and docker_container.stdout != 'no' + when: config_templates.changed and docker_container.stdout != 'yes' - name: Add ntp alert script template: From 6a1cc3ad667bc193158f7eed9a4b4f4f05f00196 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 11 Oct 2022 11:44:23 +0500 Subject: [PATCH 348/664] chore: Remove OpenSearch for edxapp from sandbox --- playbooks/openedx_native.yml | 3 --- playbooks/roles/edxapp/defaults/main.yml | 10 ---------- 2 files changed, 13 deletions(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index 4ecd54d3044..c8c9c5340e6 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -56,7 +56,6 @@ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false EDXAPP_ENABLE_MEMCACHE: true EDXAPP_ENABLE_ELASTIC_SEARCH: true - EDXAPP_ENABLE_OPEN_SEARCH: TRUE # Ecommerce ECOMMERCE_CORS_ORIGIN_WHITELIST: [ "{{ EDXAPP_LMS_BASE_SCHEME }}://{{ MFE_BASE }}", @@ -119,8 +118,6 @@ - oauth_client_setup - role: elasticsearch when: EDXAPP_ENABLE_ELASTIC_SEARCH - - role: opensearch - when: EDXAPP_ENABLE_OPEN_SEARCH - forum - role: discovery when: SANDBOX_ENABLE_DISCOVERY diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index ee4bafb4c7e..60648100264 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -123,10 +123,6 @@ EDXAPP_SEARCH_HOST: 'localhost' EDXAPP_SEARCH_PORT: 9200 EDXAPP_SEARCH_USE_SSL: false -EDXAPP_OPENSEARCH_HOST: 'localhost' -EDXAPP_OPENSEARCH_PORT: 9202 -EDXAPP_OPENSEARCH_USE_SSL: false - # list of dictionaries of the format # { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' } # http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch @@ -135,11 +131,6 @@ EDXAPP_ELASTIC_SEARCH_CONFIG: port: "{{ EDXAPP_SEARCH_PORT }}" use_ssl: "{{ EDXAPP_SEARCH_USE_SSL }}" -EDXAPP_OPEN_SEARCH_CONFIG: - - host: "{{ EDXAPP_OPENSEARCH_HOST }}" - port: "{{ EDXAPP_OPENSEARCH_PORT }}" - use_ssl: "{{ EDXAPP_OPENSEARCH_USE_SSL }}" - EDXAPP_SETTINGS: '{{ COMMON_EDXAPP_SETTINGS }}' EDXAPP_LMS_ENV: 'lms.envs.{{ EDXAPP_SETTINGS }}' @@ -1547,7 +1538,6 @@ generic_env_config: &edxapp_generic_env user: '{{ edxapp_sandbox_user }}' AFFILIATE_COOKIE_NAME: "{{ EDXAPP_AFFILIATE_COOKIE_NAME }}" ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}" - OPEN_SEARCH_CONFIG: "{{ EDXAPP_OPEN_SEARCH_CONFIG }}" PLATFORM_TWITTER_ACCOUNT: "{{ EDXAPP_PLATFORM_TWITTER_ACCOUNT }}" PLATFORM_FACEBOOK_ACCOUNT: "{{ EDXAPP_PLATFORM_FACEBOOK_ACCOUNT }}" HELP_TOKENS_BOOKS: "{{ EDXAPP_HELP_TOKENS_BOOKS }}" From b9afd13b98c1b5aa8c8d1578a5a5ac29fbfafe87 Mon Sep 17 00:00:00 2001 From: Justin Hynes Date: Wed, 12 Oct 2022 14:28:46 -0400 Subject: [PATCH 349/664] chore: Update the default Node version of the Credentials IDA [APER-2040] * Update the Credentials IDA to install Node16 by default, updating from Node 12.11.x. --- playbooks/roles/credentials/defaults/main.yml | 3 ++- playbooks/roles/credentials/meta/main.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/credentials/defaults/main.yml b/playbooks/roles/credentials/defaults/main.yml index d5b3a6af2b7..937f3797dbc 100644 --- a/playbooks/roles/credentials/defaults/main.yml +++ b/playbooks/roles/credentials/defaults/main.yml @@ -21,7 +21,8 @@ credentials_environment: CREDENTIALS_CFG: '{{ COMMON_CFG_DIR }}/{{ credentials_service_name }}.yml' credentials_gunicorn_port: 8150 -CREDENTIALS_NODE_VERSION: '12.11.1' +CREDENTIALS_NODE_VERSION: '16.14.0' +CREDENTIALS_NPM_VERSION: '8.5.5' # # OS packages diff --git a/playbooks/roles/credentials/meta/main.yml b/playbooks/roles/credentials/meta/main.yml index 3910c275ecd..26436591839 100644 --- a/playbooks/roles/credentials/meta/main.yml +++ b/playbooks/roles/credentials/meta/main.yml @@ -43,6 +43,7 @@ dependencies: edx_django_service_extra_requirements: '{{ CREDENTIALS_EXTRA_REQUIREMENTS }}' edx_django_service_session_expire_at_browser_close: '{{ CREDENTIALS_SESSION_EXPIRE_AT_BROWSER_CLOSE }}' edx_django_service_node_version: '{{ CREDENTIALS_NODE_VERSION }}' + edx_django_service_npm_version: '{{ CREDENTIALS_NPM_VERSION }}' edx_django_service_automated_users: '{{ CREDENTIALS_AUTOMATED_USERS }}' edx_django_service_cors_whitelist: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST }}' edx_django_service_post_migrate_commands: '{{ credentials_post_migrate_commands }}' From 16c6c306c21b0aec7f448030452d900cab23100f Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Wed, 12 Oct 2022 12:26:08 -0400 Subject: [PATCH 350/664] feat: use docker compose implace of docker run command --- util/jenkins/ansible-provision.sh | 40 ++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index b0ba1db08b3..28808baad19 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -669,12 +669,12 @@ function provision_containerized_app() { # Replace deploy_host in app config file with sandbox DNS name echo "sed -i 's/deploy_host/${dns_name}.${dns_zone}/g' /var/tmp/${app_service_name}.yml" - # Install yq for yaml processing + # Install yq for yaml processing echo "wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq" # Combine app config with jwt_signature config echo "yq eval-all '. as \$item ireduce ({}; . *+ \$item)' /var/tmp/${app_service_name}.yml /tmp/lms_jwt_signature.yml > /edx/etc/${app_service_name}.yml" - + # Provision IDA User in LMS echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production manage_user ${app_service_name}_worker ${app_service_name}_worker@example.com --staff --superuser" @@ -689,9 +689,34 @@ function provision_containerized_app() { # Create app database echo "mysql -uroot -e \"CREATE DATABASE \\\`${app_service_name}\\\`;\"" - # Build app docker image and run app container - echo "docker build . -t ${app_service_name}:latest" - echo "docker run --rm -it -d -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -e ${app_cfg}=/${app_service_name}.yml -v '/edx/app/${app_repo}:/edx/app/${app_repo}/' -v '/edx/etc/${app_service_name}.yml:/${app_service_name}.yml' -v '/edx/var/${app_service_name}/staticfiles/:/var/tmp/' --name ${app_service_name}.app --network=host --entrypoint '/bin/bash' ${app_service_name}:latest -c 'while true; do exec gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_repo}/${app_service_name}/docker_gunicorn_configuration.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application; sleep 2; done'" + # use heredoc to dynamically create docker compose file + echo "docker_compose_file=/var/tmp/docker-compose-${app_service_name}.yml" + echo "cat << 'EOF' > \$docker_compose_file + version: '2.1' + services: + app: + image: ${app_service_name}:latest + stdin_open: true + tty: true + build: + context: /edx/app/${app_repo} + dockerfile: Dockerfile + container_name: ${app_service_name}.app + command: bash -c 'while true; do exec gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_repo}/${app_service_name}/docker_gunicorn_configuration.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application; sleep 2; done' + network_mode: 'host' + environment: + DJANGO_SETTINGS_MODULE: ${app_service_name}.settings.production + DJANGO_WATCHMAN_TIMEOUT: 30 + ENABLE_DJANGO_TOOLBAR: 1 + ${app_cfg}: /${app_service_name}.yml + volumes: + - /edx/app/${app_repo}:/edx/app/${app_repo}/ + - /edx/etc/${app_service_name}.yml:/${app_service_name}.yml + - /edx/var/${app_service_name}/staticfiles/:/var/tmp/ +EOF" + + # run docker compose to spin up service container + echo "docker-compose -f \$docker_compose_file up -d" # Wait for app container echo "sleep 5" @@ -701,7 +726,6 @@ function provision_containerized_app() { # Run collectstatic echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py collectstatic --noinput\"" - # Create superuser echo "docker exec -t ${app_service_name}.app bash -c \"echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\\\"edx\\\", \\\"edx@example.com\\\", \\\"edx\\\") if not User.objects.filter(username=\\\"edx\\\").exists() else None' | python /edx/app/${app_repo}/manage.py shell\"" @@ -744,7 +768,7 @@ EOF" } if [[ $edx_exams == 'true' ]]; then - + app_hostname="edx-exams" app_service_name="edx_exams" app_repo="edx-exams" @@ -756,7 +780,7 @@ if [[ $edx_exams == 'true' ]]; then cat << EOF > $provision_script $(provision_containerized_app) EOF - + # copy app config file and run script to deploy app ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${WORKSPACE}/configuration-internal/k8s-sandbox-config/${app_service_name}.yml dest=/var/tmp/${app_service_name}.yml" -u ubuntu -b ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_script}" -u ubuntu -b From 3e51d0921906864786756b44f51de477404d0c33 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 7 Oct 2022 17:28:09 -0400 Subject: [PATCH 351/664] feat: Add management of recently changed users --- playbooks/manage_edxapp_users_and_groups.yml | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/playbooks/manage_edxapp_users_and_groups.yml b/playbooks/manage_edxapp_users_and_groups.yml index 0e7b12a9747..22b3b1d21e8 100644 --- a/playbooks/manage_edxapp_users_and_groups.yml +++ b/playbooks/manage_edxapp_users_and_groups.yml @@ -146,6 +146,27 @@ become: true become_user: "{{ common_web_user }}" + - name: Manage recent LMS/CMS users + tags: + - manage-recent-users-edxapp + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} lms --settings={{ deployment_settings }} + manage_user {{ item.username | quote }} {{ item.email | quote }} + {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + {% if item.get('superuser') %}--superuser{% endif %} + {% if item.get('staff') %}--staff{% endif %} + {% if item.get('unusable_password') %}--unusable-password{% endif %} + {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} + with_items: "{{ django_users }}" + register: manage_users_result + failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) + retries: 3 + until: manage_users_result is not failed + become: true + become_user: "{{ common_web_user }}" + + - name: Manage active LMS/CMS users tags: - manage-active-users-edxapp @@ -201,6 +222,26 @@ become: true become_user: "{{ common_web_user }}" + - name: Manage recent {{ service }} users + tags: + - manage-recent-users-ida + shell: > + . {{env_path}} && {{ python_path }} {{ manage_path }} + manage_user {{ item.username | quote }} {{ item.email | quote }} + {% if item.get('groups', []) | length %}--groups {{ item.groups | default([]) | map('quote') | join(' ') }}{% endif %} + {% if item.get('remove') %}--remove{% endif %} + {% if item.get('superuser') %}--superuser{% endif %} + {% if item.get('staff') %}--staff{% endif %} + {% if item.get('unusable_password') %}--unusable-password{% endif %} + {% if item.get('initial_password_hash') %}--initial-password-hash {{ item.initial_password_hash | quote }}{% endif %} + with_items: "{{ django_users }}" + register: manage_users_result + failed_when: (manage_users_result is failed) and not (ignore_user_creation_errors | bool) + retries: 3 + until: manage_users_result is not failed + become: true + become_user: "{{ common_web_user }}" + - name: Manage active {{ service }} users tags: - manage-active-users-ida From f9e3ecbadd983abe4d08dae97404a207f37f7c00 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 18 Oct 2022 17:44:07 +0500 Subject: [PATCH 352/664] chore: Added OpenSearch for discovery --- playbooks/openedx_native.yml | 3 +++ playbooks/roles/discovery/defaults/main.yml | 8 ++++++++ playbooks/roles/opensearch/tasks/main.yml | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/playbooks/openedx_native.yml b/playbooks/openedx_native.yml index c8c9c5340e6..009f8edfb07 100644 --- a/playbooks/openedx_native.yml +++ b/playbooks/openedx_native.yml @@ -52,6 +52,7 @@ SANDBOX_ENABLE_INSIGHTS: true SANDBOX_ENABLE_REDIS: true SANDBOX_ENABLE_NOTES: false + SANDBOX_ENABLE_OPEN_SEARCH: TRUE DEMO_ROLE_ENABLED: true ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false EDXAPP_ENABLE_MEMCACHE: true @@ -118,6 +119,8 @@ - oauth_client_setup - role: elasticsearch when: EDXAPP_ENABLE_ELASTIC_SEARCH + - role: opensearch + when: SANDBOX_ENABLE_OPEN_SEARCH - forum - role: discovery when: SANDBOX_ENABLE_DISCOVERY diff --git a/playbooks/roles/discovery/defaults/main.yml b/playbooks/roles/discovery/defaults/main.yml index 931e4ad7483..0660c208325 100644 --- a/playbooks/roles/discovery/defaults/main.yml +++ b/playbooks/roles/discovery/defaults/main.yml @@ -58,6 +58,10 @@ DISCOVERY_SSL_NGINX_PORT: "4{{ discovery_gunicorn_port }}" DISCOVERY_ELASTICSEARCH_URL: 'http://127.0.0.1:9200/' DISCOVERY_ELASTICSEARCH_INDEX_NAME: 'catalog' +DISCOVERY_OPENSEARCH_HOST: 'localhost' +DISCOVERY_OPENSEARCH_PORT: 9202 +DISCOVERY_OPENSEARCH_USE_SSL: false + DISCOVERY_MEMCACHE: [ 'memcache' ] DISCOVERY_VERSION: "master" @@ -156,6 +160,10 @@ discovery_service_config_overrides: ELASTICSEARCH_CLUSTER_URL: '{{ DISCOVERY_ELASTICSEARCH_URL }}' ELASTICSEARCH_INDEX_NAME: '{{ DISCOVERY_ELASTICSEARCH_INDEX_NAME }}' + DISCOVERY_OPENSEARCH_HOST: '{{ DISCOVERY_OPENSEARCH_HOST }}' + DISCOVERY_OPENSEARCH_PORT: '{{ DISCOVERY_OPENSEARCH_PORT }}' + DISCOVERY_OPENSEARCH_USE_SSL: '{{ DISCOVERY_OPENSEARCH_USE_SSL }}' + PLATFORM_NAME: '{{ DISCOVERY_PLATFORM_NAME }}' DEFAULT_PARTNER_ID: '{{ DISCOVERY_DEFAULT_PARTNER_ID }}' diff --git a/playbooks/roles/opensearch/tasks/main.yml b/playbooks/roles/opensearch/tasks/main.yml index 0767f902720..53aefa67f97 100644 --- a/playbooks/roles/opensearch/tasks/main.yml +++ b/playbooks/roles/opensearch/tasks/main.yml @@ -82,14 +82,14 @@ - install - install:base -- name: Get all the installed ES plugins +- name: Get all the installed OpenSearch plugins command: "/edx/app/opensearch/bin/opensearch-plugin list" register: list_plugins tags: - install - install:base -- name: Show all the installed ES plugins +- name: Show all the installed OpenSearch plugins debug: msg: "{{ list_plugins.stdout }}" tags: From 4a27d6c4bb7f829ff1d326974d83a0fd55d017dd Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Fri, 21 Oct 2022 14:12:34 -0400 Subject: [PATCH 353/664] feat: Comment this out since it's a footgun --- playbooks/splunk.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/playbooks/splunk.yml b/playbooks/splunk.yml index a4d149313ab..7555bf3aba7 100644 --- a/playbooks/splunk.yml +++ b/playbooks/splunk.yml @@ -1,8 +1,9 @@ -# Usage: ansible-playbook splunk.yml -e@/path/to/environment-deployment.yml +# This is commented out since it should not be run normally. +# # Usage: ansible-playbook splunk.yml -e@/path/to/environment-deployment.yml -- name: Deploy Splunk - hosts: all - become: True - gather_facts: True - roles: - - splunk-server +# - name: Deploy Splunk +# hosts: all +# become: True +# gather_facts: True +# roles: +# - splunk-server From cc6181f702b5ad92def18ef364386553c057ed24 Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Wed, 12 Oct 2022 14:21:04 +0500 Subject: [PATCH 354/664] feat: run edxapp workers in docker containers on sandbox --- playbooks/edx_continuous_integration.yml | 1 - playbooks/roles/edxapp/tasks/main.yml | 4 +- util/jenkins/ansible-provision.sh | 36 +++++++++++ util/jenkins/worker-container-provisioner.sh | 68 ++++++++++++++++++++ 4 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 util/jenkins/worker-container-provisioner.sh diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 97f6f24c0fa..24c53cafc8a 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,7 +25,6 @@ - memcache - mongo_4_2 - role: redis - - { role: 'edxapp', celery_worker: True } - edxapp - testcourses - oraclejdk diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index 8c8aacd187d..eb62c343f58 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -219,6 +219,6 @@ app_name: 'lms' app_config_file: "{{ COMMON_CFG_DIR }}/lms.yml" app_config_owner: "{{ edxapp_user }}" - app_config_group: root - app_config_mode: 0644 + app_config_group: "{{ common_web_group }}" + app_config_mode: 0640 CAN_GENERATE_NEW_JWT_SIGNATURE: True diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index b0ba1db08b3..a10f80302bb 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -602,6 +602,42 @@ done if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" || $server_type == "ubuntu_20.04" ]]; then cat $extra_vars_file run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu + # Export LC_* vars. To be passed to remote instance via SSH where SSH configuration allows LC_* to be accepted as environment variables. + # LC_* is normally used for passing through locale settings of SSH clients to SSH servers. + export LC_WORKER_CFG=$(cat < $1 +--- +version: "3.9" +services: +$( + for worker_config in "${worker_cfg[@]}"; do + worker_service_variant=$(echo "${worker_config}" | yq -e '.service_variant' -) + worker_queue=$(echo "${worker_config}" | yq -e '.queue' -) + worker_concurrency=$(echo "${worker_config}" | yq -e '.concurrency' -) + prefetch_optimization=$(echo "${worker_config}" | yq -e '.prefetch_optimization' -) + worker_service_name="${worker_service_variant}_${worker_queue}_${worker_concurrency}" + echo -e " ${worker_service_name}:" + echo -e " network_mode: host" + echo -e " image: ${LC_WORKER_IMAGE_NAME}:latest" + echo -e " container_name: $worker_service_name" + echo -e " user: \"www-data:www-data\"" + echo -e " command: ${worker_celery_path} --app=${worker_service_variant}.celery:APP worker --loglevel=info --queues=edx.${worker_service_variant}.core.${worker_queue} --hostname=edx.${worker_service_variant}.core.${worker_queue}.%%h --concurrency=${worker_concurrency} -O ${prefetch_optimization}" + echo -e " volumes:" + for volume_map in ${worker_service_volume_mappings[@]} ; do + echo -e " - ${volume_map}" + done + echo -e " environment:" + echo -e " - SERVICE_VARIANT=${worker_service_variant}" + echo -e " - DJANGO_SETTINGS_MODULE=${worker_service_variant}.envs.production" + echo -e " - EDX_REST_API_CLIENT_NAME=edx.${worker_service_variant}.core.${worker_queue}" + for env_map in ${worker_service_env_mappings[@]} ; do + echo -e " - ${env_map}" + done + done +) +EOF +} + +install_pre_reqs + +# Check if docker image already exists. If it doesn't, build it. +if ! $(docker image inspect ${LC_WORKER_IMAGE_NAME}:latest >/dev/null 2>&1 && echo true || echo false) ; then + cd /edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO} + docker build . -t ${LC_WORKER_IMAGE_NAME}:latest --target base +fi + +# Render a docker-compose file for workers +render_docker_compose "/home/$LC_SANDBOX_USER/docker-compose-${LC_WORKER_OF}-workers.yaml" + +# Run the docker-compose file +docker-compose -f "/home/$LC_SANDBOX_USER/docker-compose-${LC_WORKER_OF}-workers.yaml" up -d From 60bc30c919ed1ce3ebd38a930521ac730daccf3b Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Tue, 18 Oct 2022 14:01:14 -0400 Subject: [PATCH 355/664] feat: add fluentd container for logging --- util/jenkins/ansible-provision.sh | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 28808baad19..8e5965beb4c 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -788,5 +788,42 @@ EOF rm -f "${provision_script}" fi +function provision_fluentd() { + echo "#!/usr/bin/env bash" + echo "set -ex" + + echo "docker pull fluent/fluentd:edge-debian" + + echo "fluentd_config=/var/tmp/fluentd.conf" + echo "cat << 'EOF' > \$fluentd_config + + @type syslog + port 5140 + bind 0.0.0.0 + tag * + + + + @type none + + + + + @type stdout + +EOF" + echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" +} + +if [[ $fluentd_logging == 'true' ]]; then + provision_fluentd_script="/var/tmp/provision-fluentd-script.sh" +cat << EOF > $provision_fluentd_script +$(provision_fluentd) +EOF + ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_fluentd_script}" -u ubuntu -b + + rm -f "${provision_fluentd_script}" +fi + rm -f "$extra_vars_file" rm -f ${extra_vars_file}_clean From 9d918f139e003bd8031451b7d8d017d4f78b5999 Mon Sep 17 00:00:00 2001 From: Gabe Mulley Date: Fri, 28 Oct 2022 15:54:30 -0400 Subject: [PATCH 356/664] feat: support customizing the contentful environment used by prospectus --- playbooks/roles/prospectus/defaults/main.yml | 2 ++ util/jenkins/ansible-provision.sh | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 0a245116f43..7de92dedc8f 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -21,6 +21,7 @@ PROSPECTUS_NEW_RELIC_APP_ID: 'fake-id' PROSPECTUS_CONTENTFUL_SPACE_ID: 'fake-key' PROSPECTUS_CONTENTFUL_ACCESS_TOKEN: 'fake-key' +PROSPECTUS_CONTENTFUL_ENVIRONMENT: 'master' PROSPECTUS_SEGMENT_WRITE_KEY: 'fake-key' PROSPECTUS_ALGOLIA_BROWSE_KEY: 'fake-key' PROSPECTUS_ALGOLIA_ADMIN_KEY: 'fake-key' @@ -52,6 +53,7 @@ prospectus_env_vars: GATSBY_CONTENTFUL_ACCESS_TOKEN: "{{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }}" GATSBY_CSRF_TOKEN_API_PATH: '/csrf/api/v1/token' GATSBY_ACCESS_TOKEN_COOKIE_NAME: 'edx-jwt-cookie-header-payload' + CONTENTFUL_ENVIRONMENT: "{{ PROSPECTUS_CONTENTFUL_ENVIRONMENT }}" prospectus_git_identity: "{{ prospectus_app_dir }}/prospectus-git-identity" prospectus_code_dir: "{{ prospectus_app_dir }}/prospectus" prospectus_ssl_nginx_port: 443 diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 8e5965beb4c..06b6946b583 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -214,6 +214,10 @@ if [[ -z $prospectus_version ]]; then PROSPECTUS_VERSION="master" fi +if [[ -z $prospectus_contentful_environment ]]; then + prospectus_contentful_environment="master" +fi + if [[ $registrar == 'true' ]]; then program_console="true" fi @@ -314,6 +318,7 @@ PROSPECTUS_NGINX_PORT: 80 PROSPECTUS_SSL_NGINX_PORT: 443 PROSPECTUS_VERSION: $prospectus_version PROSPECTUS_ENABLED: $prospectus +PROSPECTUS_CONTENTFUL_ENVIRONMENT: $prospectus_contentful_environment PROSPECTUS_SANDBOX_BUILD: True AUTHN_NGINX_PORT: 80 From a22d7d2a85c9d5d8db43359ca4a28b6a639e44b2 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Thu, 3 Nov 2022 17:27:44 +0500 Subject: [PATCH 357/664] feat: PSRE-1899 MySQL 5.7 Dockerfile and github workflow --- .github/workflows/mysql-5.7-build.yml | 26 ++ Dockerfile | 109 +++++++ docker-entrypoint.sh | 437 ++++++++++++++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 .github/workflows/mysql-5.7-build.yml create mode 100644 Dockerfile create mode 100644 docker-entrypoint.sh diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml new file mode 100644 index 00000000000..cb4dd0566a1 --- /dev/null +++ b/.github/workflows/mysql-5.7-build.yml @@ -0,0 +1,26 @@ +name: MySQL5.7 image build + +on: + push: + branches: + - 'main' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v3 + with: + push: true + tags: edxops/mysql:5.7 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..f4d53d9bfae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +FROM debian:buster-slim + +# add gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 + +ENV MYSQL_MAJOR 5.7 +ENV MYSQL_MINOR ${MYSQL_MAJOR}.40 +ENV MYSQL_VERSION 5.7.40-1debian10 + +RUN set -eux; \ + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added + groupadd -r mysql && useradd -r -g mysql mysql; \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y --no-install-recommends gnupg && rm -rf /var/lib/apt/lists/*; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true; \ + mkdir /docker-entrypoint-initdb.d; \ + DEVELOPMENT_PACKAGES="pkg-config libssl-dev dpkg-dev devscripts equivs"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + bzip2 \ + openssl \ + # FATAL ERROR: please install the following Perl modules before executing /usr/local/mysql/scripts/mysql_install_db: + # File::Basename + # File::Copy + # Sys::Hostname + # Data::Dumper + perl \ + xz-utils \ + zstd \ + ${DEVELOPMENT_PACKAGES}; \ + rm -rf /var/lib/apt/lists/*; \ + # gpg: key 3A79BD29: public key "MySQL Release Engineering " imported + key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ + mkdir -p /etc/apt/keyrings; \ + gpg --batch --export "$key" > /etc/apt/keyrings/mysql.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + # RUN echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list + echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list; \ + echo 'deb-src [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' >> /etc/apt/sources.list.d/mysql.list; \ +# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter + { \ + echo mysql-community-server mysql-community-server/data-dir select ''; \ + echo mysql-community-server mysql-community-server/root-pass password ''; \ + echo mysql-community-server mysql-community-server/re-root-pass password ''; \ + echo mysql-community-server mysql-community-server/remove-test-db select false; \ + } | debconf-set-selections; \ + apt-get update; \ + mk-build-deps mysql-community-server --install --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"; \ + mkdir /usr/src/mysql; \ + cd /usr/src/mysql; \ + apt source mysql-community-server=${MYSQL_MINOR}; \ + cd mysql-community-${MYSQL_MINOR}; \ + sed -i -e "s/\-DWITH_BOOST\=\/usr\/global\/share/\-DWITH_BOOST\=\/tmp\/boost \-DDOWNLOAD_BOOST\=1/g" debian/rules; \ + sed -i -e "s/\-DWITH_SSL\=.* /-DWITH_SSL\=system /g" debian/rules; \ + debuild -b -uc -us; \ + MYSQL_DEB_VERSION=${MYSQL_VERSION}_${dpkgArch}; \ + cd ~; \ + dpkg --install \ + /usr/src/mysql/mysql-common_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-community-server_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-community-client_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-server_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-client_${MYSQL_DEB_VERSION}.deb; \ + rm -rf /usr/src/mysql /tmp/boost /mysql-community-build-deps*deb; \ + apt-get remove --purge -y mysql-community-build-deps ${DEVELOPMENT_PACKAGES}; \ + apt-get -y autoremove; \ + apt-get clean all; \ + # comment out a few problematic configuration values + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ + # don't reverse lookup hostnames, they are usually another container + echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld; \ + chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ + # ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + chmod 1777 /var/run/mysqld /var/lib/mysql + +VOLUME /var/lib/mysql + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 3306 33060 +CMD ["mysqld"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 00000000000..5819a38e93a --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,437 @@ +#!/bin/bash +set -eo pipefail +shopt -s nullglob + +# logging functions +mysql_log() { + local type="$1"; shift + # accept argument string or stdin + local text="$*"; if [ "$#" -eq 0 ]; then text="$(cat)"; fi + local dt; dt="$(date --rfc-3339=seconds)" + printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + mysql_error "Both $var and $fileVar are set (but are exclusive)" + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions +docker_process_init_files() { + # mysql here for backwards compatibility "${mysql[@]}" + mysql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + mysql_note "$0: running $f" + "$f" + else + mysql_note "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) mysql_note "$0: running $f"; docker_process_sql < "$f"; echo ;; + *.sql.bz2) mysql_note "$0: running $f"; bunzip2 -c "$f" | docker_process_sql; echo ;; + *.sql.gz) mysql_note "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) mysql_note "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *.sql.zst) mysql_note "$0: running $f"; zstd -dc "$f" | docker_process_sql; echo ;; + *) mysql_warn "$0: ignoring $f" ;; + esac + echo + done +} + +# arguments necessary to run "mysqld --verbose --help" successfully (used for testing configuration validity and for extracting default/configured values) +_verboseHelpArgs=( + --verbose --help + --log-bin-index="$(mktemp -u)" # https://github.com/docker-library/mysql/issues/136 +) + +mysql_check_config() { + local toRun=( "$@" "${_verboseHelpArgs[@]}" ) errors + if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then + mysql_error $'mysqld failed while attempting to check config\n\tcommand was: '"${toRun[*]}"$'\n\t'"$errors" + fi +} + +# Fetch value from server config +# We use mysqld --verbose --help instead of my_print_defaults because the +# latter only show values present in config files, and not server defaults +mysql_get_config() { + local conf="$1"; shift + "$@" "${_verboseHelpArgs[@]}" 2>/dev/null \ + | awk -v conf="$conf" '$1 == conf && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' + # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" +} + +# Ensure that the package default socket can also be used +# since rpm packages are compiled with a different socket location +# and "mysqlsh --mysql" doesn't read the [client] config +# related to https://github.com/docker-library/mysql/issues/829 +mysql_socket_fix() { + local defaultSocket + defaultSocket="$(mysql_get_config 'socket' mysqld --no-defaults)" + if [ "$defaultSocket" != "$SOCKET" ]; then + ln -sfTv "$SOCKET" "$defaultSocket" || : + fi +} + +# Do a temporary startup of the MySQL server, for init purposes +docker_temp_server_start() { + if [ "${MYSQL_MAJOR}" = '5.7' ]; then + "$@" --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}" & + mysql_note "Waiting for server startup" + local i + for i in {30..0}; do + # only use the root password if the database has already been initialized + # so that it won't try to fill in a password file when it hasn't been set yet + extraArgs=() + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + extraArgs+=( '--dont-use-mysql-root-password' ) + fi + if docker_process_sql "${extraArgs[@]}" --database=mysql <<<'SELECT 1' &> /dev/null; then + break + fi + sleep 1 + done + if [ "$i" = 0 ]; then + mysql_error "Unable to start server." + fi + else + # For 5.7+ the server is ready for use as soon as startup command unblocks + if ! "$@" --daemonize --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}"; then + mysql_error "Unable to start server." + fi + fi +} + +# Stop the server. When using a local socket file mysqladmin will block until +# the shutdown is complete. +docker_temp_server_stop() { + if ! mysqladmin --defaults-extra-file=<( _mysql_passfile ) shutdown -uroot --socket="${SOCKET}"; then + mysql_error "Unable to shut down server." + fi +} + +# Verify that the minimally required password settings are set for new databases. +docker_verify_minimum_env() { + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then + mysql_error <<-'EOF' + Database is uninitialized and password option is not specified + You need to specify one of the following: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # This will prevent the CREATE USER from failing (and thus exiting with a half-initialized database) + if [ "$MYSQL_USER" = 'root' ]; then + mysql_error <<-'EOF' + MYSQL_USER="root", MYSQL_USER and MYSQL_PASSWORD are for configuring a regular user and cannot be used for the root user + Remove MYSQL_USER="root" and use one of the following to control the root user password: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # warn when missing one of MYSQL_USER or MYSQL_PASSWORD + if [ -n "$MYSQL_USER" ] && [ -z "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_USER specified, but missing MYSQL_PASSWORD; MYSQL_USER will not be created' + elif [ -z "$MYSQL_USER" ] && [ -n "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_PASSWORD specified, but missing MYSQL_USER; MYSQL_PASSWORD will be ignored' + fi +} + +# creates folders for the database +# also ensures permission for user mysql of run as root +docker_create_db_directories() { + local user; user="$(id -u)" + + local -A dirs=( ["$DATADIR"]=1 ) + local dir + dir="$(dirname "$SOCKET")" + dirs["$dir"]=1 + + # "datadir" and "socket" are already handled above (since they were already queried previously) + local conf + for conf in \ + general-log-file \ + keyring_file_data \ + pid-file \ + secure-file-priv \ + slow-query-log-file \ + ; do + dir="$(mysql_get_config "$conf" "$@")" + + # skip empty values + if [ -z "$dir" ] || [ "$dir" = 'NULL' ]; then + continue + fi + case "$conf" in + secure-file-priv) + # already points at a directory + ;; + *) + # other config options point at a file, but we need the directory + dir="$(dirname "$dir")" + ;; + esac + + dirs["$dir"]=1 + done + + mkdir -p "${!dirs[@]}" + + if [ "$user" = "0" ]; then + # this will cause less disk access than `chown -R` + find "${!dirs[@]}" \! -user mysql -exec chown --no-dereference mysql '{}' + + fi +} + +# initializes the database directory +docker_init_database_dir() { + mysql_note "Initializing database files" + "$@" --initialize-insecure --default-time-zone=SYSTEM + mysql_note "Database files initialized" +} + +# Loads various settings that are used elsewhere in the script +# This should be called after mysql_check_config, but before any other functions +docker_setup_env() { + # Get config + declare -g DATADIR SOCKET + DATADIR="$(mysql_get_config 'datadir' "$@")" + SOCKET="$(mysql_get_config 'socket' "$@")" + + # Initialize values that might be stored in a file + file_env 'MYSQL_ROOT_HOST' '%' + file_env 'MYSQL_DATABASE' + file_env 'MYSQL_USER' + file_env 'MYSQL_PASSWORD' + file_env 'MYSQL_ROOT_PASSWORD' + + declare -g DATABASE_ALREADY_EXISTS + if [ -d "$DATADIR/mysql" ]; then + DATABASE_ALREADY_EXISTS='true' + fi +} + +# Execute sql script, passed via stdin +# usage: docker_process_sql [--dont-use-mysql-root-password] [mysql-cli-args] +# ie: docker_process_sql --database=mydb <<<'INSERT ...' +# ie: docker_process_sql --dont-use-mysql-root-password --database=mydb /dev/null + + docker_init_database_dir "$@" + + mysql_note "Starting temporary server" + docker_temp_server_start "$@" + mysql_note "Temporary server started." + + mysql_socket_fix + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + mysql_expire_root_user + + mysql_note "Stopping temporary server" + docker_temp_server_stop + mysql_note "Temporary server stopped" + + echo + mysql_note "MySQL init process done. Ready for start up." + echo + else + mysql_socket_fix + fi + fi + exec "$@" +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi From 5197885bd4c47106be3a1ef1189548ffb51a933a Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 3 Nov 2022 15:41:41 -0400 Subject: [PATCH 358/664] feat: Add tags to gocd build instances --- playbooks/continuous_delivery/launch_instance.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 4b8593b94b9..807b1b33f4d 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -90,6 +90,11 @@ state: present tags: Name: "{{ automation_prefix }} {{ unique_key_name.stdout }}" + gocd_pipeline_url: "{{ lookup('ansible.builtin.env', 'GO_SERVER_URL')|default('', True) }}/pipelines/value_stream_map/{{ lookup('ansible.builtin.env', 'GO_PIPELINE_NAME')|default('', True) }}/{{ lookup('ansible.builtin.env', 'GO_PIPELINE_COUNTER')|default('', True) }}" + gocd_pipepline_name: "{{ lookup('ansible.builtin.env', 'GO_PIPELINE_NAME')|default('', True) }}" + gocd_pipepline_label: "{{ lookup('ansible.builtin.env', 'GO_PIPELINE_LABEL')|default('', True) }}" + gocd_pipepline_counter: "{{ lookup('ansible.builtin.env', 'GO_PIPELINE_COUNTER')|default('', True) }}" + gocd_stage_counter: "{{ lookup('ansible.builtin.env', 'GO_STAGE_COUNTER')|default('', True) }}" retries: 6 delay: 15 From e0638fa18fd6a323266019bf6af5a8c47ef4a522 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Thu, 3 Nov 2022 17:27:44 +0500 Subject: [PATCH 359/664] feat: PSRE-1899 MySQL 5.7 Dockerfile and github workflow --- .github/workflows/mysql-5.7-build.yml | 26 ++ Dockerfile | 109 +++++++ docker-entrypoint.sh | 437 ++++++++++++++++++++++++++ 3 files changed, 572 insertions(+) create mode 100644 .github/workflows/mysql-5.7-build.yml create mode 100644 Dockerfile create mode 100644 docker-entrypoint.sh diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml new file mode 100644 index 00000000000..cb4dd0566a1 --- /dev/null +++ b/.github/workflows/mysql-5.7-build.yml @@ -0,0 +1,26 @@ +name: MySQL5.7 image build + +on: + push: + branches: + - 'main' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v3 + with: + push: true + tags: edxops/mysql:5.7 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..f4d53d9bfae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +FROM debian:buster-slim + +# add gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 + +ENV MYSQL_MAJOR 5.7 +ENV MYSQL_MINOR ${MYSQL_MAJOR}.40 +ENV MYSQL_VERSION 5.7.40-1debian10 + +RUN set -eux; \ + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added + groupadd -r mysql && useradd -r -g mysql mysql; \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y --no-install-recommends gnupg && rm -rf /var/lib/apt/lists/*; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true; \ + mkdir /docker-entrypoint-initdb.d; \ + DEVELOPMENT_PACKAGES="pkg-config libssl-dev dpkg-dev devscripts equivs"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + bzip2 \ + openssl \ + # FATAL ERROR: please install the following Perl modules before executing /usr/local/mysql/scripts/mysql_install_db: + # File::Basename + # File::Copy + # Sys::Hostname + # Data::Dumper + perl \ + xz-utils \ + zstd \ + ${DEVELOPMENT_PACKAGES}; \ + rm -rf /var/lib/apt/lists/*; \ + # gpg: key 3A79BD29: public key "MySQL Release Engineering " imported + key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ + mkdir -p /etc/apt/keyrings; \ + gpg --batch --export "$key" > /etc/apt/keyrings/mysql.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + # RUN echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list + echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list; \ + echo 'deb-src [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' >> /etc/apt/sources.list.d/mysql.list; \ +# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter + { \ + echo mysql-community-server mysql-community-server/data-dir select ''; \ + echo mysql-community-server mysql-community-server/root-pass password ''; \ + echo mysql-community-server mysql-community-server/re-root-pass password ''; \ + echo mysql-community-server mysql-community-server/remove-test-db select false; \ + } | debconf-set-selections; \ + apt-get update; \ + mk-build-deps mysql-community-server --install --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"; \ + mkdir /usr/src/mysql; \ + cd /usr/src/mysql; \ + apt source mysql-community-server=${MYSQL_MINOR}; \ + cd mysql-community-${MYSQL_MINOR}; \ + sed -i -e "s/\-DWITH_BOOST\=\/usr\/global\/share/\-DWITH_BOOST\=\/tmp\/boost \-DDOWNLOAD_BOOST\=1/g" debian/rules; \ + sed -i -e "s/\-DWITH_SSL\=.* /-DWITH_SSL\=system /g" debian/rules; \ + debuild -b -uc -us; \ + MYSQL_DEB_VERSION=${MYSQL_VERSION}_${dpkgArch}; \ + cd ~; \ + dpkg --install \ + /usr/src/mysql/mysql-common_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-community-server_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-community-client_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-server_${MYSQL_DEB_VERSION}.deb \ + /usr/src/mysql/mysql-client_${MYSQL_DEB_VERSION}.deb; \ + rm -rf /usr/src/mysql /tmp/boost /mysql-community-build-deps*deb; \ + apt-get remove --purge -y mysql-community-build-deps ${DEVELOPMENT_PACKAGES}; \ + apt-get -y autoremove; \ + apt-get clean all; \ + # comment out a few problematic configuration values + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ + # don't reverse lookup hostnames, they are usually another container + echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld; \ + chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ + # ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + chmod 1777 /var/run/mysqld /var/lib/mysql + +VOLUME /var/lib/mysql + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 3306 33060 +CMD ["mysqld"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 00000000000..5819a38e93a --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,437 @@ +#!/bin/bash +set -eo pipefail +shopt -s nullglob + +# logging functions +mysql_log() { + local type="$1"; shift + # accept argument string or stdin + local text="$*"; if [ "$#" -eq 0 ]; then text="$(cat)"; fi + local dt; dt="$(date --rfc-3339=seconds)" + printf '%s [%s] [Entrypoint]: %s\n' "$dt" "$type" "$text" +} +mysql_note() { + mysql_log Note "$@" +} +mysql_warn() { + mysql_log Warn "$@" >&2 +} +mysql_error() { + mysql_log ERROR "$@" >&2 + exit 1 +} + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + mysql_error "Both $var and $fileVar are set (but are exclusive)" + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions +docker_process_init_files() { + # mysql here for backwards compatibility "${mysql[@]}" + mysql=( docker_process_sql ) + + echo + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + mysql_note "$0: running $f" + "$f" + else + mysql_note "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) mysql_note "$0: running $f"; docker_process_sql < "$f"; echo ;; + *.sql.bz2) mysql_note "$0: running $f"; bunzip2 -c "$f" | docker_process_sql; echo ;; + *.sql.gz) mysql_note "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; + *.sql.xz) mysql_note "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; + *.sql.zst) mysql_note "$0: running $f"; zstd -dc "$f" | docker_process_sql; echo ;; + *) mysql_warn "$0: ignoring $f" ;; + esac + echo + done +} + +# arguments necessary to run "mysqld --verbose --help" successfully (used for testing configuration validity and for extracting default/configured values) +_verboseHelpArgs=( + --verbose --help + --log-bin-index="$(mktemp -u)" # https://github.com/docker-library/mysql/issues/136 +) + +mysql_check_config() { + local toRun=( "$@" "${_verboseHelpArgs[@]}" ) errors + if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then + mysql_error $'mysqld failed while attempting to check config\n\tcommand was: '"${toRun[*]}"$'\n\t'"$errors" + fi +} + +# Fetch value from server config +# We use mysqld --verbose --help instead of my_print_defaults because the +# latter only show values present in config files, and not server defaults +mysql_get_config() { + local conf="$1"; shift + "$@" "${_verboseHelpArgs[@]}" 2>/dev/null \ + | awk -v conf="$conf" '$1 == conf && /^[^ \t]/ { sub(/^[^ \t]+[ \t]+/, ""); print; exit }' + # match "datadir /some/path with/spaces in/it here" but not "--xyz=abc\n datadir (xyz)" +} + +# Ensure that the package default socket can also be used +# since rpm packages are compiled with a different socket location +# and "mysqlsh --mysql" doesn't read the [client] config +# related to https://github.com/docker-library/mysql/issues/829 +mysql_socket_fix() { + local defaultSocket + defaultSocket="$(mysql_get_config 'socket' mysqld --no-defaults)" + if [ "$defaultSocket" != "$SOCKET" ]; then + ln -sfTv "$SOCKET" "$defaultSocket" || : + fi +} + +# Do a temporary startup of the MySQL server, for init purposes +docker_temp_server_start() { + if [ "${MYSQL_MAJOR}" = '5.7' ]; then + "$@" --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}" & + mysql_note "Waiting for server startup" + local i + for i in {30..0}; do + # only use the root password if the database has already been initialized + # so that it won't try to fill in a password file when it hasn't been set yet + extraArgs=() + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + extraArgs+=( '--dont-use-mysql-root-password' ) + fi + if docker_process_sql "${extraArgs[@]}" --database=mysql <<<'SELECT 1' &> /dev/null; then + break + fi + sleep 1 + done + if [ "$i" = 0 ]; then + mysql_error "Unable to start server." + fi + else + # For 5.7+ the server is ready for use as soon as startup command unblocks + if ! "$@" --daemonize --skip-networking --default-time-zone=SYSTEM --socket="${SOCKET}"; then + mysql_error "Unable to start server." + fi + fi +} + +# Stop the server. When using a local socket file mysqladmin will block until +# the shutdown is complete. +docker_temp_server_stop() { + if ! mysqladmin --defaults-extra-file=<( _mysql_passfile ) shutdown -uroot --socket="${SOCKET}"; then + mysql_error "Unable to shut down server." + fi +} + +# Verify that the minimally required password settings are set for new databases. +docker_verify_minimum_env() { + if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then + mysql_error <<-'EOF' + Database is uninitialized and password option is not specified + You need to specify one of the following: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # This will prevent the CREATE USER from failing (and thus exiting with a half-initialized database) + if [ "$MYSQL_USER" = 'root' ]; then + mysql_error <<-'EOF' + MYSQL_USER="root", MYSQL_USER and MYSQL_PASSWORD are for configuring a regular user and cannot be used for the root user + Remove MYSQL_USER="root" and use one of the following to control the root user password: + - MYSQL_ROOT_PASSWORD + - MYSQL_ALLOW_EMPTY_PASSWORD + - MYSQL_RANDOM_ROOT_PASSWORD + EOF + fi + + # warn when missing one of MYSQL_USER or MYSQL_PASSWORD + if [ -n "$MYSQL_USER" ] && [ -z "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_USER specified, but missing MYSQL_PASSWORD; MYSQL_USER will not be created' + elif [ -z "$MYSQL_USER" ] && [ -n "$MYSQL_PASSWORD" ]; then + mysql_warn 'MYSQL_PASSWORD specified, but missing MYSQL_USER; MYSQL_PASSWORD will be ignored' + fi +} + +# creates folders for the database +# also ensures permission for user mysql of run as root +docker_create_db_directories() { + local user; user="$(id -u)" + + local -A dirs=( ["$DATADIR"]=1 ) + local dir + dir="$(dirname "$SOCKET")" + dirs["$dir"]=1 + + # "datadir" and "socket" are already handled above (since they were already queried previously) + local conf + for conf in \ + general-log-file \ + keyring_file_data \ + pid-file \ + secure-file-priv \ + slow-query-log-file \ + ; do + dir="$(mysql_get_config "$conf" "$@")" + + # skip empty values + if [ -z "$dir" ] || [ "$dir" = 'NULL' ]; then + continue + fi + case "$conf" in + secure-file-priv) + # already points at a directory + ;; + *) + # other config options point at a file, but we need the directory + dir="$(dirname "$dir")" + ;; + esac + + dirs["$dir"]=1 + done + + mkdir -p "${!dirs[@]}" + + if [ "$user" = "0" ]; then + # this will cause less disk access than `chown -R` + find "${!dirs[@]}" \! -user mysql -exec chown --no-dereference mysql '{}' + + fi +} + +# initializes the database directory +docker_init_database_dir() { + mysql_note "Initializing database files" + "$@" --initialize-insecure --default-time-zone=SYSTEM + mysql_note "Database files initialized" +} + +# Loads various settings that are used elsewhere in the script +# This should be called after mysql_check_config, but before any other functions +docker_setup_env() { + # Get config + declare -g DATADIR SOCKET + DATADIR="$(mysql_get_config 'datadir' "$@")" + SOCKET="$(mysql_get_config 'socket' "$@")" + + # Initialize values that might be stored in a file + file_env 'MYSQL_ROOT_HOST' '%' + file_env 'MYSQL_DATABASE' + file_env 'MYSQL_USER' + file_env 'MYSQL_PASSWORD' + file_env 'MYSQL_ROOT_PASSWORD' + + declare -g DATABASE_ALREADY_EXISTS + if [ -d "$DATADIR/mysql" ]; then + DATABASE_ALREADY_EXISTS='true' + fi +} + +# Execute sql script, passed via stdin +# usage: docker_process_sql [--dont-use-mysql-root-password] [mysql-cli-args] +# ie: docker_process_sql --database=mydb <<<'INSERT ...' +# ie: docker_process_sql --dont-use-mysql-root-password --database=mydb /dev/null + + docker_init_database_dir "$@" + + mysql_note "Starting temporary server" + docker_temp_server_start "$@" + mysql_note "Temporary server started." + + mysql_socket_fix + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + mysql_expire_root_user + + mysql_note "Stopping temporary server" + docker_temp_server_stop + mysql_note "Temporary server stopped" + + echo + mysql_note "MySQL init process done. Ready for start up." + echo + else + mysql_socket_fix + fi + fi + exec "$@" +} + +# If we are sourced from elsewhere, don't perform any further actions +if ! _is_sourced; then + _main "$@" +fi From 59410a7055e18554771ab90bd882777db4649d83 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Tue, 8 Nov 2022 13:38:20 +0500 Subject: [PATCH 360/664] feat: Github actions workflow added --- .github/workflows/edx-mysql-arm.yml | 50 ++++++++ .github/workflows/mysql-5.7-build.yml | 28 ++++- Dockerfile | 109 ---------------- docker/build/mysql/Dockerfile | 118 ++++++++++++++++-- .../build/mysql/docker-entrypoint.sh | 0 5 files changed, 184 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/edx-mysql-arm.yml delete mode 100644 Dockerfile rename docker-entrypoint.sh => docker/build/mysql/docker-entrypoint.sh (100%) diff --git a/.github/workflows/edx-mysql-arm.yml b/.github/workflows/edx-mysql-arm.yml new file mode 100644 index 00000000000..6c486e23d0d --- /dev/null +++ b/.github/workflows/edx-mysql-arm.yml @@ -0,0 +1,50 @@ +name: MySQL5.7 image build + +on: + push: + branches: + - 'main' + +jobs: + docker: + runs-on: ubuntu-latest + name: Build on ${{ matrix.distro }} ${{ matrix.arch }} + strategy: + matrix: + include: + - arch: aarch64 + distro: ubuntu-latest + - arch: armv7 + distro: ubuntu-latest + env: + DOCKER_TARGET_PLATFORM: linux/arm/v7 + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build and push amd64 image + uses: docker/build-push-action@v3 + with: + push: true + tags: user/app:latest + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + - + name: Build and push arm image + uses: docker/build-push-action@v3 + with: + push: true +# platforms: linux/arm/v7 + tags: edxops/mysql-arm:5.7 + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index cb4dd0566a1..7c314c8ba4b 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -8,10 +8,22 @@ on: jobs: docker: runs-on: ubuntu-latest + name: Build on ${{ matrix.distro }} ${{ matrix.arch }} + strategy: + matrix: + arch: [aarch64, armv6] + include: + - arch: aarch64 + distro: ubuntu-latest + - arch: armv6 + distro: raspios_lite:latest steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Login to Docker Hub uses: docker/login-action@v2 @@ -19,8 +31,20 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and push + name: Build and push ${{ matrix.arch }} uses: docker/build-push-action@v3 with: push: true - tags: edxops/mysql:5.7 + context: ./docker/build/mysql/ + tags: edxops/edx-mysql-${{ matrix.arch }}:5.7 + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} + - + name: Build and push ${{ matrix.arch }} + uses: docker/build-push-action@v3 + with: + push: true + context: ./docker/build/mysql/ + tags: edxops/edx-mysql-${{ matrix.arch }}:5.7 + arch: ${{ matrix.arch }} + distro: ${{ matrix.distro }} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f4d53d9bfae..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,109 +0,0 @@ -FROM debian:buster-slim - -# add gosu for easy step-down from root -# https://github.com/tianon/gosu/releases -ENV GOSU_VERSION 1.14 - -ENV MYSQL_MAJOR 5.7 -ENV MYSQL_MINOR ${MYSQL_MAJOR}.40 -ENV MYSQL_VERSION 5.7.40-1debian10 - -RUN set -eux; \ - # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added - groupadd -r mysql && useradd -r -g mysql mysql; \ - export DEBIAN_FRONTEND=noninteractive; \ - apt-get update && apt-get install -y --no-install-recommends gnupg && rm -rf /var/lib/apt/lists/*; \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates wget; \ - rm -rf /var/lib/apt/lists/*; \ - dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch"; \ - wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch.asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - chmod +x /usr/local/bin/gosu; \ - gosu --version; \ - gosu nobody true; \ - mkdir /docker-entrypoint-initdb.d; \ - DEVELOPMENT_PACKAGES="pkg-config libssl-dev dpkg-dev devscripts equivs"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - bzip2 \ - openssl \ - # FATAL ERROR: please install the following Perl modules before executing /usr/local/mysql/scripts/mysql_install_db: - # File::Basename - # File::Copy - # Sys::Hostname - # Data::Dumper - perl \ - xz-utils \ - zstd \ - ${DEVELOPMENT_PACKAGES}; \ - rm -rf /var/lib/apt/lists/*; \ - # gpg: key 3A79BD29: public key "MySQL Release Engineering " imported - key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ - mkdir -p /etc/apt/keyrings; \ - gpg --batch --export "$key" > /etc/apt/keyrings/mysql.gpg; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME"; \ - # RUN echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list - echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list; \ - echo 'deb-src [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' >> /etc/apt/sources.list.d/mysql.list; \ -# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) -# also, we set debconf keys to make APT a little quieter - { \ - echo mysql-community-server mysql-community-server/data-dir select ''; \ - echo mysql-community-server mysql-community-server/root-pass password ''; \ - echo mysql-community-server mysql-community-server/re-root-pass password ''; \ - echo mysql-community-server mysql-community-server/remove-test-db select false; \ - } | debconf-set-selections; \ - apt-get update; \ - mk-build-deps mysql-community-server --install --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"; \ - mkdir /usr/src/mysql; \ - cd /usr/src/mysql; \ - apt source mysql-community-server=${MYSQL_MINOR}; \ - cd mysql-community-${MYSQL_MINOR}; \ - sed -i -e "s/\-DWITH_BOOST\=\/usr\/global\/share/\-DWITH_BOOST\=\/tmp\/boost \-DDOWNLOAD_BOOST\=1/g" debian/rules; \ - sed -i -e "s/\-DWITH_SSL\=.* /-DWITH_SSL\=system /g" debian/rules; \ - debuild -b -uc -us; \ - MYSQL_DEB_VERSION=${MYSQL_VERSION}_${dpkgArch}; \ - cd ~; \ - dpkg --install \ - /usr/src/mysql/mysql-common_${MYSQL_DEB_VERSION}.deb \ - /usr/src/mysql/mysql-community-server_${MYSQL_DEB_VERSION}.deb \ - /usr/src/mysql/mysql-community-client_${MYSQL_DEB_VERSION}.deb \ - /usr/src/mysql/mysql-server_${MYSQL_DEB_VERSION}.deb \ - /usr/src/mysql/mysql-client_${MYSQL_DEB_VERSION}.deb; \ - rm -rf /usr/src/mysql /tmp/boost /mysql-community-build-deps*deb; \ - apt-get remove --purge -y mysql-community-build-deps ${DEVELOPMENT_PACKAGES}; \ - apt-get -y autoremove; \ - apt-get clean all; \ - # comment out a few problematic configuration values - find /etc/mysql/ -name '*.cnf' -print0 \ - | xargs -0 grep -lZE '^(bind-address|log)' \ - | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ - # don't reverse lookup hostnames, they are usually another container - echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf; \ - rm -rf /var/lib/apt/lists/*; \ - rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld; \ - chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ - # ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime - chmod 1777 /var/run/mysqld /var/lib/mysql - -VOLUME /var/lib/mysql - -COPY docker-entrypoint.sh /usr/local/bin/ -RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat -ENTRYPOINT ["docker-entrypoint.sh"] - -EXPOSE 3306 33060 -CMD ["mysqld"] diff --git a/docker/build/mysql/Dockerfile b/docker/build/mysql/Dockerfile index 435a4d953e7..28f8551ce71 100644 --- a/docker/build/mysql/Dockerfile +++ b/docker/build/mysql/Dockerfile @@ -1,10 +1,108 @@ -FROM edxops/xenial-common:latest -LABEL maintainer="edxops" - -ADD . /edx/app/edx_ansible/edx_ansible -COPY docker/build/mysql/ansible_overrides.yml / -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook -vvvv mysql.yml \ - -i '127.0.0.1,' -c local \ - -e@/ansible_overrides.yml -WORKDIR /edx/app +FROM debian:buster-slim + +# add gosu for easy step-down from root +# https://github.com/tianon/gosu/releases +ENV GOSU_VERSION 1.14 + +ENV MYSQL_MAJOR 5.7 +ENV MYSQL_MINOR ${MYSQL_MAJOR}.39 +ENV MYSQL_VERSION 5.7.39-1debian10 + +RUN set -eux; \ + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added + groupadd -r mysql && useradd -r -g mysql mysql; \ + export DEBIAN_FRONTEND=noninteractive; \ + apt-get update && apt-get install -y --no-install-recommends gnupg && rm -rf /var/lib/apt/lists/*; \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates wget; \ + rm -rf /var/lib/apt/lists/*; \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-$dpkgArch.asc"; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + apt-mark auto '.*' > /dev/null; \ + [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + chmod +x /usr/local/bin/gosu; \ + gosu --version; \ + gosu nobody true; \ + mkdir /docker-entrypoint-initdb.d; \ + DEVELOPMENT_PACKAGES="pkg-config libssl-dev dpkg-dev devscripts equivs"; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + bzip2 \ + openssl \ + # FATAL ERROR: please install the following Perl modules before executing /usr/local/mysql/scripts/mysql_install_db: + # File::Basename + # File::Copy + # Sys::Hostname + # Data::Dumper + perl \ + xz-utils \ + zstd \ + ${DEVELOPMENT_PACKAGES}; \ + rm -rf /var/lib/apt/lists/*; \ + # gpg: key 3A79BD29: public key "MySQL Release Engineering " imported + key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \ + mkdir -p /etc/apt/keyrings; \ + gpg --batch --export "$key" > /etc/apt/keyrings/mysql.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + # RUN echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list + echo 'deb [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' > /etc/apt/sources.list.d/mysql.list; \ + echo 'deb-src [ signed-by=/etc/apt/keyrings/mysql.gpg ] http://repo.mysql.com/apt/debian/ buster mysql-5.7' >> /etc/apt/sources.list.d/mysql.list; \ +# the "/var/lib/mysql" stuff here is because the mysql-server postinst doesn't have an explicit way to disable the mysql_install_db codepath besides having a database already "configured" (ie, stuff in /var/lib/mysql/mysql) +# also, we set debconf keys to make APT a little quieter + { \ + echo mysql-community-server mysql-community-server/data-dir select ''; \ + echo mysql-community-server mysql-community-server/root-pass password ''; \ + echo mysql-community-server mysql-community-server/re-root-pass password ''; \ + echo mysql-community-server mysql-community-server/remove-test-db select false; \ + } | debconf-set-selections; \ + apt-get update; \ + mk-build-deps mysql-community-server --install --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"; \ + mkdir /usr/src/mysql; \ + cd /usr/src/mysql; \ + apt source mysql-community-server; \ + cd mysql-community-*; \ + sed -i -e "s/\-DWITH_BOOST\=\/usr\/global\/share/\-DWITH_BOOST\=\/tmp\/boost \-DDOWNLOAD_BOOST\=1/g" debian/rules; \ + sed -i -e "s/\-DWITH_SSL\=.* /-DWITH_SSL\=system /g" debian/rules; \ + debuild -b -uc -us; \ + cd ~; \ + dpkg --install \ + /usr/src/mysql/mysql-common_*.deb \ + /usr/src/mysql/mysql-community-server_*.deb \ + /usr/src/mysql/mysql-community-client_*.deb \ + /usr/src/mysql/mysql-server_*.deb \ + /usr/src/mysql/mysql-client_*.deb; \ + rm -rf /usr/src/mysql /tmp/boost /mysql-community-build-deps*deb; \ + apt-get remove --purge -y mysql-community-build-deps ${DEVELOPMENT_PACKAGES}; \ + apt-get -y autoremove; \ + apt-get clean all; \ + # comment out a few problematic configuration values + find /etc/mysql/ -name '*.cnf' -print0 \ + | xargs -0 grep -lZE '^(bind-address|log)' \ + | xargs -rt -0 sed -Ei 's/^(bind-address|log)/#&/'; \ + # don't reverse lookup hostnames, they are usually another container + echo '[mysqld]\nskip-host-cache\nskip-name-resolve' > /etc/mysql/conf.d/docker.cnf; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld; \ + chown -R mysql:mysql /var/lib/mysql /var/run/mysqld; \ + # ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instance ends up having at runtime + chmod 1777 /var/run/mysqld /var/lib/mysql + +VOLUME /var/lib/mysql + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +ENTRYPOINT ["docker-entrypoint.sh"] + +EXPOSE 3306 33060 +CMD ["mysqld"] diff --git a/docker-entrypoint.sh b/docker/build/mysql/docker-entrypoint.sh similarity index 100% rename from docker-entrypoint.sh rename to docker/build/mysql/docker-entrypoint.sh From e98a73938fbaebf324d755cbdbf93756cb88bb29 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Thu, 10 Nov 2022 17:09:42 +0500 Subject: [PATCH 361/664] feat: mysql image for amd64 --- .github/workflows/mysql-5.7-build.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index 52ffb9d65cf..14b1dbf1fcb 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -11,23 +11,25 @@ on: - .github/workflows/mysql-5.7-build.yml jobs: - docker: + build: + strategy: + matrix: + build_platform: ["linux/amd64"] runs-on: ubuntu-latest name: Build arm/amd mysql images steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v3 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Set up QEMU + + - name: Setup QEMU uses: docker/setup-qemu-action@v2 with: - platforms: 'arm64,arm' - - - name: Login to Docker Hub + platforms: ${{ matrix.build_platform }} + + - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -39,4 +41,4 @@ jobs: push: true context: docker/build/mysql/ tags: edxops/edx-mysql:5.7 - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.build_platform }} From 11c265739513cc6dc92bdcf9d0fedf58a9202314 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Fri, 11 Nov 2022 16:24:41 +0500 Subject: [PATCH 362/664] fix: trigger event changes --- .github/workflows/mysql-5.7-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index 52ffb9d65cf..d3b126ba7d1 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -1,17 +1,17 @@ name: MySQL5.7 image build on: - push: - paths: - - "docker/build/mysql/Dockerfile" - - .github/workflows/mysql-5.7-build.yml pull_request: + types: + - closed paths: - "docker/build/mysql/Dockerfile" - .github/workflows/mysql-5.7-build.yml jobs: - docker: + build: + if_merged: + if: github.event.pull_request.merged == true runs-on: ubuntu-latest name: Build arm/amd mysql images steps: From fea45c0084a9dbcbd5af4091f90c5d189869c54f Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Fri, 11 Nov 2022 16:43:58 +0500 Subject: [PATCH 363/664] fix: Run action of PR merge --- .github/workflows/mysql-5.7-build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index d3b126ba7d1..a82d63e80a5 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -12,8 +12,11 @@ jobs: build: if_merged: if: github.event.pull_request.merged == true + strategy: + matrix: + build_platform: ["linux/amd64"] runs-on: ubuntu-latest - name: Build arm/amd mysql images + name: Build amd64 mysql image steps: - name: Checkout @@ -25,7 +28,7 @@ jobs: name: Set up QEMU uses: docker/setup-qemu-action@v2 with: - platforms: 'arm64,arm' + platforms: ${{ matrix.build_platform }} - name: Login to Docker Hub uses: docker/login-action@v2 @@ -39,4 +42,4 @@ jobs: push: true context: docker/build/mysql/ tags: edxops/edx-mysql:5.7 - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.build_platform }} From 40bdb535bcc50febb594f712e16bade44405a578 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Mon, 14 Nov 2022 17:35:01 +0500 Subject: [PATCH 364/664] fix: fixed pipeline --- .github/workflows/mysql-5.7-build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index a82d63e80a5..84fc6af7456 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -1,9 +1,8 @@ name: MySQL5.7 image build on: - pull_request: - types: - - closed + push: + branches-ignore: ['master'] paths: - "docker/build/mysql/Dockerfile" - .github/workflows/mysql-5.7-build.yml @@ -18,19 +17,20 @@ jobs: runs-on: ubuntu-latest name: Build amd64 mysql image steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v3 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Set up QEMU + + - name: Setup QEMU uses: docker/setup-qemu-action@v2 with: platforms: ${{ matrix.build_platform }} - name: Login to Docker Hub + + - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} From 4234718887e6ab048241f9f998ae6a0f20055a32 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Mon, 14 Nov 2022 17:49:09 +0500 Subject: [PATCH 365/664] fix: run action on PR merge --- .github/workflows/mysql-5.7-build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index 84fc6af7456..cf69eb1a133 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -2,15 +2,14 @@ name: MySQL5.7 image build on: push: - branches-ignore: ['master'] + branches: + - master paths: - "docker/build/mysql/Dockerfile" - .github/workflows/mysql-5.7-build.yml jobs: build: - if_merged: - if: github.event.pull_request.merged == true strategy: matrix: build_platform: ["linux/amd64"] From 0279dbbddd1c6a0e25d0b9c14c90243dcfc02e36 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Mon, 14 Nov 2022 17:58:08 +0500 Subject: [PATCH 366/664] fix: run action on PR merge --- .github/workflows/mysql-5.7-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/mysql-5.7-build.yml b/.github/workflows/mysql-5.7-build.yml index cf69eb1a133..6aeca0cf202 100644 --- a/.github/workflows/mysql-5.7-build.yml +++ b/.github/workflows/mysql-5.7-build.yml @@ -26,9 +26,6 @@ jobs: uses: docker/setup-qemu-action@v2 with: platforms: ${{ matrix.build_platform }} - - - name: Login to Docker Hub - - name: Login to DockerHub uses: docker/login-action@v2 with: From 1355ceee1c7aa4341c63e785f55f5ec895a83da1 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Mon, 14 Nov 2022 11:09:56 -0500 Subject: [PATCH 367/664] temp: enable debug logging for the New Relic agent on stage (#6836) --- playbooks/roles/edxapp/templates/newrelic.ini.j2 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/roles/edxapp/templates/newrelic.ini.j2 b/playbooks/roles/edxapp/templates/newrelic.ini.j2 index 26bb86bf2a3..342cb6aeb45 100644 --- a/playbooks/roles/edxapp/templates/newrelic.ini.j2 +++ b/playbooks/roles/edxapp/templates/newrelic.ini.j2 @@ -27,3 +27,11 @@ # `course_id`. # browser_monitoring.attributes.enabled=true + +{# Temporarily turn on logging to debug https://github.com/openedx/event-bus-kafka/issues/45. If that ticket is closed, + this if-block can be removed -#} +{%- if COMMON_ENVIRONMENT == "stage" %} +audit_log_file = /tmp/newrelic-python-agent-audit.log +log_file = /tmp/newrelic-python-agent-debug.log +log_level = debug +{%- endif %} From d4221d58871ba6c36e8eb87c0fa0ea40b9b0e11b Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Mon, 14 Nov 2022 14:52:14 -0500 Subject: [PATCH 368/664] fix: change fluentD configs for tail logs --- util/jenkins/ansible-provision.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 06b6946b583..61d00b4a813 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -802,9 +802,8 @@ function provision_fluentd() { echo "fluentd_config=/var/tmp/fluentd.conf" echo "cat << 'EOF' > \$fluentd_config - @type syslog - port 5140 - bind 0.0.0.0 + @type tail + path /var/tmp/logs.log tag * @@ -817,7 +816,7 @@ function provision_fluentd() { @type stdout EOF" - echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" + echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log" } if [[ $fluentd_logging == 'true' ]]; then From d3decf93dcb5e0538b2d30cfe2bd6e26aa70ca90 Mon Sep 17 00:00:00 2001 From: Phillip Shiu Date: Tue, 15 Nov 2022 17:59:39 +0300 Subject: [PATCH 369/664] feat: add enterprise_catalog to update.sh (#6838) --- playbooks/roles/edx_ansible/templates/update.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index 51d3cdc764e..6c40c60d4b5 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -16,7 +16,7 @@ IFS="," - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, configuration, read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, discovery, video_web_frontend, video_delivery_worker, veda_pipeline_worker, video_encode_worker, veda_ffmpeg, - registrar, program_console, learner_portal + registrar, program_console, learner_portal, prospectus, authn, payment, learning, ora_grading, enterprise_catalog - can be a commit or tag - specify extra_vars to any of the ansible plays with the -e switch and then ecaptulating your vars in "double quotes" example: update -e "-e 'hallo=bye' -e 'bye=hallo'" @@ -82,6 +82,7 @@ repos_to_cmd["authn"]="$edx_ansible_cmd authn_frontend.yml -e 'AUTHN_MFE_VERSION repos_to_cmd["payment"]="$edx_ansible_cmd payment.yml -e 'PAYMENT_MFE_VERSION=$2'" repos_to_cmd["learning"]="$edx_ansible_cmd learning.yml -e 'LEARNING_MFE_VERSION=$2'" repos_to_cmd["ora_grading"]="$edx_ansible_cmd ora_grading.yml -e 'ORA_GRADING_MFE_VERSION=$2'" +repos_to_cmd["enterprise_catalog"]="$edx_ansible_cmd enterprise_catalog.yml -e 'ENTERPRISE_CATALOG_MFE_VERSION=$2'" if [[ -z $1 || -z $2 ]]; then echo From aa5df39af7422412cd70e9b9255c90d9943c023b Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Tue, 15 Nov 2022 14:17:20 -0500 Subject: [PATCH 370/664] Revert "temp: enable debug logging for the New Relic agent on stage (#6836)" (#6837) This reverts commit 1355ceee1c7aa4341c63e785f55f5ec895a83da1. --- playbooks/roles/edxapp/templates/newrelic.ini.j2 | 8 -------- 1 file changed, 8 deletions(-) diff --git a/playbooks/roles/edxapp/templates/newrelic.ini.j2 b/playbooks/roles/edxapp/templates/newrelic.ini.j2 index 342cb6aeb45..26bb86bf2a3 100644 --- a/playbooks/roles/edxapp/templates/newrelic.ini.j2 +++ b/playbooks/roles/edxapp/templates/newrelic.ini.j2 @@ -27,11 +27,3 @@ # `course_id`. # browser_monitoring.attributes.enabled=true - -{# Temporarily turn on logging to debug https://github.com/openedx/event-bus-kafka/issues/45. If that ticket is closed, - this if-block can be removed -#} -{%- if COMMON_ENVIRONMENT == "stage" %} -audit_log_file = /tmp/newrelic-python-agent-audit.log -log_file = /tmp/newrelic-python-agent-debug.log -log_level = debug -{%- endif %} From 992e5199a18d3b124f8eb10a8c03289135e6c0f5 Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Tue, 15 Nov 2022 14:23:20 -0500 Subject: [PATCH 371/664] fix: change docker command to mount file and run as root --- util/jenkins/ansible-provision.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 61d00b4a813..ba2bc528ee6 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -804,9 +804,8 @@ function provision_fluentd() { @type tail path /var/tmp/logs.log + pos_file /var/log/td-agent/log_file.pos tag * - - @type none @@ -816,7 +815,7 @@ function provision_fluentd() { @type stdout EOF" - echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log" + echo "docker run --user root -d --network host --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" } if [[ $fluentd_logging == 'true' ]]; then From ec4b50a51d09efe0b577aac96b6723a92f1870cc Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Thu, 10 Nov 2022 16:42:13 +0500 Subject: [PATCH 372/664] chore: Added toggle that Disables Deployment of Ansible based Celery Workers --- playbooks/edx_continuous_integration.yml | 1 + util/jenkins/ansible-provision.sh | 32 +++++++++++--------- util/jenkins/worker-container-provisioner.sh | 2 +- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 24c53cafc8a..e7cbefcfa62 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,6 +25,7 @@ - memcache - mongo_4_2 - role: redis + - { role: 'edxapp', celery_worker: True, when: edxapp_celery_worker is defined and edxapp_celery_worker } - edxapp - testcourses - oraclejdk diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index a10f80302bb..a5315a2a501 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -601,10 +601,11 @@ done # run non-deploy tasks for all plays if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" || $server_type == "ubuntu_20.04" ]]; then cat $extra_vars_file - run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu - # Export LC_* vars. To be passed to remote instance via SSH where SSH configuration allows LC_* to be accepted as environment variables. - # LC_* is normally used for passing through locale settings of SSH clients to SSH servers. - export LC_WORKER_CFG=$(cat </dev/null 2>&1 && echo true || echo false) ; then cd /edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO} - docker build . -t ${LC_WORKER_IMAGE_NAME}:latest --target base + time DOCKER_BUILDKIT=1 docker build . -t ${LC_WORKER_IMAGE_NAME}:latest --target base fi # Render a docker-compose file for workers From cfa05a78222b66bfe6e2c47aa9e985bc68da225e Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Thu, 17 Nov 2022 15:43:46 -0500 Subject: [PATCH 373/664] fix: change pos file location --- util/jenkins/ansible-provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index ba2bc528ee6..9a6bf97aed1 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -804,7 +804,7 @@ function provision_fluentd() { @type tail path /var/tmp/logs.log - pos_file /var/log/td-agent/log_file.pos + pos_file /var/log/log_file.pos tag * @type none @@ -815,7 +815,7 @@ function provision_fluentd() { @type stdout EOF" - echo "docker run --user root -d --network host --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" + echo "docker run -d --network host --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" } if [[ $fluentd_logging == 'true' ]]; then From 5d522359661d2c8c746af73e6f82d8b443b76071 Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Mon, 21 Nov 2022 11:41:51 -0500 Subject: [PATCH 374/664] fix: mount tmp volume instead of bind mount file --- util/jenkins/ansible-provision.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 9a6bf97aed1..b1d89bf03b1 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -803,8 +803,8 @@ function provision_fluentd() { echo "cat << 'EOF' > \$fluentd_config @type tail - path /var/tmp/logs.log - pos_file /var/log/log_file.pos + path /var/tmp/tracking_logs.log + pos_file /var/log/tracking_logs.pos tag * @type none @@ -815,7 +815,7 @@ function provision_fluentd() { @type stdout EOF" - echo "docker run -d --network host --mount type=bind,source=/var/tmp/logs.log,target=/var/tmp/logs.log -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" + echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf -v /var/tmp:/var/tmp fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" } if [[ $fluentd_logging == 'true' ]]; then From e5ca81fb066864dc17b2c7d1ad75d18ce5d232a2 Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Tue, 22 Nov 2022 10:08:14 -0500 Subject: [PATCH 375/664] fix: add rotate_wait config 10 seconds --- util/jenkins/ansible-provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index b1d89bf03b1..d1d3c36a469 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -805,6 +805,7 @@ function provision_fluentd() { @type tail path /var/tmp/tracking_logs.log pos_file /var/log/tracking_logs.pos + rotate_wait 10 tag * @type none From 93f6d26438d3226b4de88ae7cac4a69ef323fcd9 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 29 Nov 2022 10:55:57 -0500 Subject: [PATCH 376/664] fix: no nginx when building service images for devstack we do not run nginx inside these containers, so this step was always moot JIRA:DOS-3493 --- docker/plays/credentials.yml | 3 --- docker/plays/designer.yml | 3 --- docker/plays/discovery.yml | 3 --- docker/plays/ecommerce.yml | 3 --- docker/plays/edxapp.yml | 8 -------- docker/plays/enterprise_catalog.yml | 3 --- docker/plays/notes.yml | 3 --- docker/plays/registrar.yml | 3 --- 8 files changed, 29 deletions(-) diff --git a/docker/plays/credentials.yml b/docker/plays/credentials.yml index ca52d521a14..0845efe5e64 100644 --- a/docker/plays/credentials.yml +++ b/docker/plays/credentials.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - credentials - credentials diff --git a/docker/plays/designer.yml b/docker/plays/designer.yml index 516cebce9e3..163d7496bb3 100644 --- a/docker/plays/designer.yml +++ b/docker/plays/designer.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - designer - designer diff --git a/docker/plays/discovery.yml b/docker/plays/discovery.yml index 35236f62082..f4737f7fb35 100644 --- a/docker/plays/discovery.yml +++ b/docker/plays/discovery.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - discovery - discovery diff --git a/docker/plays/ecommerce.yml b/docker/plays/ecommerce.yml index 5e6aa5b839d..14987139153 100644 --- a/docker/plays/ecommerce.yml +++ b/docker/plays/ecommerce.yml @@ -6,9 +6,6 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - ecommerce - devpi_consumer - ecommerce - sqlite_fix diff --git a/docker/plays/edxapp.yml b/docker/plays/edxapp.yml index d6199572486..ccdc314f9f9 100644 --- a/docker/plays/edxapp.yml +++ b/docker/plays/edxapp.yml @@ -8,12 +8,4 @@ roles: - common - devpi_consumer - - role: nginx - nginx_sites: - - lms - - cms - nginx_default_sites: - - lms - nginx_extra_sites: "{{ NGINX_EDXAPP_EXTRA_SITES }}" - nginx_extra_configs: "{{ NGINX_EDXAPP_EXTRA_CONFIGS }}" - edxapp diff --git a/docker/plays/enterprise_catalog.yml b/docker/plays/enterprise_catalog.yml index ba72826b0a4..499ff24cfb3 100644 --- a/docker/plays/enterprise_catalog.yml +++ b/docker/plays/enterprise_catalog.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - enterprise_catalog - enterprise_catalog diff --git a/docker/plays/notes.yml b/docker/plays/notes.yml index 8dbce12e19a..94ff04e5899 100644 --- a/docker/plays/notes.yml +++ b/docker/plays/notes.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_sites: - - edx_notes_api - role: edx_notes_api diff --git a/docker/plays/registrar.yml b/docker/plays/registrar.yml index 96ca892b2f3..894c098bde7 100644 --- a/docker/plays/registrar.yml +++ b/docker/plays/registrar.yml @@ -6,7 +6,4 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - role: nginx - nginx_default_sites: - - registrar - registrar From ca4d6c47d7a9c828a57027859b1e111466916252 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan Date: Thu, 1 Dec 2022 16:12:55 +0500 Subject: [PATCH 377/664] feat: remove Insights Devstack code --- docker/build/insights/Dockerfile | 31 ----------- docker/build/insights/ansible_overrides.yml | 24 --------- docker/build/insights/devstack.sh | 26 --------- docker/build/insights/insights.yml | 59 --------------------- docker/plays/insights.yml | 10 ---- 5 files changed, 150 deletions(-) delete mode 100644 docker/build/insights/Dockerfile delete mode 100644 docker/build/insights/ansible_overrides.yml delete mode 100644 docker/build/insights/devstack.sh delete mode 100644 docker/build/insights/insights.yml delete mode 100644 docker/plays/insights.yml diff --git a/docker/build/insights/Dockerfile b/docker/build/insights/Dockerfile deleted file mode 100644 index eb0805698da..00000000000 --- a/docker/build/insights/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/insights/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" - -ADD . /edx/app/edx_ansible/edx_ansible -COPY docker/build/insights/ansible_overrides.yml / - -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays - -ARG OPENEDX_RELEASE=master -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook insights.yml \ - -i '127.0.0.1,' -c local \ - -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" \ - --extra-vars="INSIGHTS_VERSION=${OPENEDX_RELEASE}" \ - --extra-vars="@/ansible_overrides.yml" -COPY docker/build/insights/insights.yml /edx/etc/insights.yml -ADD docker/build/insights/devstack.sh /edx/app/insights/devstack.sh -RUN chown insights:insights /edx/app/insights/devstack.sh && chmod a+x /edx/app/insights/devstack.sh -ENTRYPOINT ["/edx/app/insights/devstack.sh"] -CMD ["start"] -EXPOSE 8110 18110 diff --git a/docker/build/insights/ansible_overrides.yml b/docker/build/insights/ansible_overrides.yml deleted file mode 100644 index 480e0b1fd3c..00000000000 --- a/docker/build/insights/ansible_overrides.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- - -DOCKER_TLD: "edx" - -INSIGHTS_LMS_BASE: "http://lms.{{ DOCKER_TLD }}:8000" -INSIGHTS_CMS_BASE: "http://cms.{{ DOCKER_TLD }}:8010" -INSIGHTS_BASE_URL: "http://insights.{{ DOCKER_TLD }}:8110" -INSIGHTS_MEMCACHE: - - "memcache.{{ DOCKER_TLD }}:11211" -ANALYTICS_API_ENDPOINT: "http://analtyicsapi.{{ DOCKER_TLD }}:8100/api/v0" - -insights_django_settings: "analytics_dashboard.settings.devstack" - -INSIGHTS_DATABASES: - # rw user - default: - ENGINE: 'django.db.backends.mysql' - NAME: '{{ INSIGHTS_DATABASE_NAME }}' - USER: 'rosencrantz' - PASSWORD: 'secret' - HOST: "db.{{ DOCKER_TLD }}" - PORT: '3306' - -edx_django_service_is_devstack: true diff --git a/docker/build/insights/devstack.sh b/docker/build/insights/devstack.sh deleted file mode 100644 index 46addc019bb..00000000000 --- a/docker/build/insights/devstack.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -COMMAND=$1 - -case $COMMAND in - start) - /edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf - ;; - open) - . /edx/app/insights/venvs/insights/bin/activate - cd /edx/app/insights/insights - - /bin/bash - ;; - exec) - shift - - . /edx/app/insights/venvs/insights/bin/activate - cd /edx/app/insights/insights - - "$@" - ;; - *) - "$@" - ;; -esac diff --git a/docker/build/insights/insights.yml b/docker/build/insights/insights.yml deleted file mode 100644 index dbe361e8a21..00000000000 --- a/docker/build/insights/insights.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- - -APPLICATION_NAME: Insights -BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://edx.devstack.lms:18000/oauth2 -CACHES: - default: - BACKEND: django.core.cache.backends.memcached.MemcachedCache - KEY_PREFIX: default_env-default_deployment-insights - LOCATION: - - edx.devstack.memcached:11211 -CDN_DOMAIN: null -CMS_COURSE_SHORTCUT_BASE_URL: http://edx.devstack.lms:18000/course -COURSE_API_URL: http://edx.devstack.lms:18000/api/courses/v1/ -CSRF_COOKIE_NAME: insights_csrftoken -CSRF_COOKIE_SECURE: false -DATABASES: - default: - ENGINE: django.db.backends.mysql - HOST: edx.devstack.mysql - NAME: dashboard - PASSWORD: secret - PORT: '3306' - USER: rosencrantz -DATA_API_AUTH_TOKEN: edx -DATA_API_URL: http://edx.devstack.analyticsapi:18100/api/v0 -DOCUMENTATION_LOAD_ERROR_URL: http://127.0.0.1/en/latest/Reference.html#error-conditions -EMAIL_HOST: smtp.example.com -EMAIL_HOST_PASSWORD: mail_password -EMAIL_HOST_USER: mail_user -EMAIL_PORT: 587 -ENABLE_AUTO_AUTH: true -GRADING_POLICY_API_URL: http://edx.devstack.lms:18000/api/grades/v1/ -HELP_URL: http://127.0.0.1/en/latest -LANGUAGE_CODE: en-us -LANGUAGE_COOKIE_NAME: insights_language -LEARNER_API_LIST_DOWNLOAD_FIELDS: null -LMS_COURSE_SHORTCUT_BASE_URL: URL_FOR_LMS_COURSE_LIST_PAGE -MODULE_PREVIEW_URL: http://edx.devstack.lms:18000/xblock -OPEN_SOURCE_URL: http://set-me-please -PLATFORM_NAME: edX -PRIVACY_POLICY_URL: http://example.com/privacy-policy -RESEARCH_URL: https://www.edx.org/research-pedagogy -SECRET_KEY: YOUR_SECRET_KEY_HERE -SEGMENT_IGNORE_EMAIL_REGEX: null -SEGMENT_IO_KEY: YOUR_KEY -SESSION_COOKIE_NAME: insights_sessionid -SESSION_EXPIRE_AT_BROWSER_CLOSE: false -SOCIAL_AUTH_REDIRECT_IS_HTTPS: false -SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://localhost:18000 -SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://edx.devstack.lms:18000 -SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT: http://localhost:18000 -SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://localhost:18000/logout -STATICFILES_DIRS: -- /edx/app/insights/edx_analytics_dashboard/analytics_dashboard/static -STATIC_ROOT: /edx/var/insights/staticfiles -SUPPORT_EMAIL: '' -TERMS_OF_SERVICE_URL: http://example.com/terms-service -TIME_ZONE: UTC - diff --git a/docker/plays/insights.yml b/docker/plays/insights.yml deleted file mode 100644 index e31939a774a..00000000000 --- a/docker/plays/insights.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deploy Insights - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - common - - insights From 7354cba4721033692d420eab259ac264d6597b85 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Wed, 26 Oct 2022 17:32:00 -0400 Subject: [PATCH 378/664] feat: Add stub devstack ami playbook ISRE-1389 --- playbooks/devstack_ami.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 playbooks/devstack_ami.yml diff --git a/playbooks/devstack_ami.yml b/playbooks/devstack_ami.yml new file mode 100644 index 00000000000..51ecd227d43 --- /dev/null +++ b/playbooks/devstack_ami.yml @@ -0,0 +1,11 @@ +- name: Build cloud devstack AMI + hosts: all + become: True + gather_facts: True + vars: + ENABLE_DATADOG: False + ENABLE_NEWRELIC: False + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - role: common From 000ffb9bef183dc3167aca7f28c9d2de582a9073 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 3 Nov 2022 11:59:43 -0400 Subject: [PATCH 379/664] DO NOT MERGE: Use public ip --- playbooks/continuous_delivery/launch_instance.yml | 4 +++- .../templates/local/launch_template.yml.j2 | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 807b1b33f4d..0b1e9a478ea 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -100,7 +100,9 @@ - name: Wait for SSH to come up wait_for: - host: "{{ ec2_instance_register.instances[0].private_ip }}" +# jdmulloy temporarily use public ip for testing from laptop +# host: "{{ ec2_instance_register.instances[0].private_ip }}" + host: "{{ ec2_instance_register.instances[0].public_ip }}" port: 22 delay: 60 timeout: "{{ ec2_timeout }}" diff --git a/playbooks/continuous_delivery/templates/local/launch_template.yml.j2 b/playbooks/continuous_delivery/templates/local/launch_template.yml.j2 index c43731639f0..f9f954caea6 100644 --- a/playbooks/continuous_delivery/templates/local/launch_template.yml.j2 +++ b/playbooks/continuous_delivery/templates/local/launch_template.yml.j2 @@ -1,4 +1,4 @@ keypair_id: {{ unique_key_name.stdout }} key_material_file: {{ artifact_path }}/key.pem instance_id: {{ ec2_instance_register.instances[0].id }} -instance_ip: {{ ec2_instance_register.instances[0].private_ip }} \ No newline at end of file +instance_ip: {{ ec2_instance_register.instances[0].public_ip }} From ce1e5dc675a1cf3760cf1ad89467db78667d68e8 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 3 Nov 2022 14:51:28 -0400 Subject: [PATCH 380/664] DO NOT MERGE: Use public ip 2 --- playbooks/continuous_delivery/templates/local/inventory.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/continuous_delivery/templates/local/inventory.j2 b/playbooks/continuous_delivery/templates/local/inventory.j2 index c22f36001ce..9857d34eb19 100644 --- a/playbooks/continuous_delivery/templates/local/inventory.j2 +++ b/playbooks/continuous_delivery/templates/local/inventory.j2 @@ -1,3 +1,3 @@ {% for instance in ec2_instance_register.instances %} -{{ instance.private_ip }} -{% endfor %} \ No newline at end of file +{{ instance.public_ip }} +{% endfor %} From 00a671ce0a8dac517da2c680f96da808c9539d55 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 2 Dec 2022 10:27:58 -0500 Subject: [PATCH 381/664] feat: Use private or public ip based on variable --- playbooks/continuous_delivery/launch_instance.yml | 14 +++++++++++--- .../templates/local/inventory.j2 | 4 ++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 0b1e9a478ea..f6450436a5d 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -98,15 +98,23 @@ retries: 6 delay: 15 - - name: Wait for SSH to come up + - name: Wait for SSH to come up (private ip) + wait_for: + host: "{{ ec2_instance_register.instances[0].private_ip }}" + port: 22 + delay: 60 + timeout: "{{ ec2_timeout }}" + state: started + when: not ec2_assign_public_ip + + - name: Wait for SSH to come up (public ip) wait_for: -# jdmulloy temporarily use public ip for testing from laptop -# host: "{{ ec2_instance_register.instances[0].private_ip }}" host: "{{ ec2_instance_register.instances[0].public_ip }}" port: 22 delay: 60 timeout: "{{ ec2_timeout }}" state: started + when: ec2_assign_public_ip - name: Generate artifact for jobs down stream template: diff --git a/playbooks/continuous_delivery/templates/local/inventory.j2 b/playbooks/continuous_delivery/templates/local/inventory.j2 index 9857d34eb19..a96ba9c5b48 100644 --- a/playbooks/continuous_delivery/templates/local/inventory.j2 +++ b/playbooks/continuous_delivery/templates/local/inventory.j2 @@ -1,3 +1,7 @@ {% for instance in ec2_instance_register.instances %} +{% if ec2_assign_public_ip %} {{ instance.public_ip }} +{% else %} +{{ instance.private_ip }} +{% endif %} {% endfor %} From 880092dfc886a78097824a76a832c489e986d471 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 19 Dec 2022 17:47:12 -0500 Subject: [PATCH 382/664] feat: Add aws_devstack playbook --- playbooks/devstack_ami.yml | 2 +- playbooks/roles/aws_devstack/meta/main.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 playbooks/roles/aws_devstack/meta/main.yml diff --git a/playbooks/devstack_ami.yml b/playbooks/devstack_ami.yml index 51ecd227d43..8f8631f8a67 100644 --- a/playbooks/devstack_ami.yml +++ b/playbooks/devstack_ami.yml @@ -8,4 +8,4 @@ roles: - role: aws when: COMMON_ENABLE_AWS_ROLE - - role: common + - role: aws_devstack diff --git a/playbooks/roles/aws_devstack/meta/main.yml b/playbooks/roles/aws_devstack/meta/main.yml new file mode 100644 index 00000000000..f779906f1e9 --- /dev/null +++ b/playbooks/roles/aws_devstack/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: common + - role: docker-tools From 6f950c6929c976492ad43c02ecdce46149dd9813 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 19 Dec 2022 17:47:27 -0500 Subject: [PATCH 383/664] chore: Update docker package list List from https://docs.docker.com/engine/install/ubuntu/ includes pacakges docker-ce docker-ce-cli containerd.io docker-compose-plugin --- playbooks/roles/docker-tools/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml index 6a3ab29e00f..41eae58c295 100644 --- a/playbooks/roles/docker-tools/tasks/main.yml +++ b/playbooks/roles/docker-tools/tasks/main.yml @@ -55,7 +55,7 @@ - name: install docker-engine apt: - name: "docker-ce" + name: ["docker-ce", "docker-ce-cli", "containerd.io", "docker-compose-plugin"] update_cache: yes tags: - install From dbab0da587fb3206d7e65896b7fbcc00834573c8 Mon Sep 17 00:00:00 2001 From: Demid Date: Wed, 21 Dec 2022 01:44:00 +0200 Subject: [PATCH 384/664] refactor: rename hidden_module.HiddenDescriptor -> hidden_block.HiddenBlock (#6833) --- docker/build/edxapp/lms.yml | 4 ++-- docker/build/edxapp/studio.yml | 4 ++-- playbooks/roles/edxapp/defaults/main.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/build/edxapp/lms.yml b/docker/build/edxapp/lms.yml index 70983473f39..402e7fd296a 100644 --- a/docker/build/edxapp/lms.yml +++ b/docker/build/edxapp/lms.yml @@ -380,7 +380,7 @@ MODULESTORE: ENGINE: xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore NAME: split OPTIONS: - default_class: xmodule.hidden_module.HiddenDescriptor + default_class: xmodule.hidden_block.HiddenBlock fs_root: /edx/var/edxapp/data render_template: common.djangoapps.edxmako.shortcuts.render_to_string - DOC_STORE_CONFIG: @@ -400,7 +400,7 @@ MODULESTORE: ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore NAME: draft OPTIONS: - default_class: xmodule.hidden_module.HiddenDescriptor + default_class: xmodule.hidden_block.HiddenBlock fs_root: /edx/var/edxapp/data render_template: common.djangoapps.edxmako.shortcuts.render_to_string OAUTH_DELETE_EXPIRED: true diff --git a/docker/build/edxapp/studio.yml b/docker/build/edxapp/studio.yml index 5e1e4443698..be56083c979 100644 --- a/docker/build/edxapp/studio.yml +++ b/docker/build/edxapp/studio.yml @@ -342,7 +342,7 @@ MODULESTORE: ENGINE: xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore NAME: split OPTIONS: - default_class: xmodule.hidden_module.HiddenDescriptor + default_class: xmodule.hidden_block.HiddenBlock fs_root: /edx/var/edxapp/data render_template: common.djangoapps.edxmako.shortcuts.render_to_string - DOC_STORE_CONFIG: @@ -362,7 +362,7 @@ MODULESTORE: ENGINE: xmodule.modulestore.mongo.DraftMongoModuleStore NAME: draft OPTIONS: - default_class: xmodule.hidden_module.HiddenDescriptor + default_class: xmodule.hidden_block.HiddenBlock fs_root: /edx/var/edxapp/data render_template: common.djangoapps.edxmako.shortcuts.render_to_string ORA2_FILE_PREFIX: default_env-default_deployment/ora2 diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 60648100264..9ecf1aa3d20 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1593,14 +1593,14 @@ lms_auth_config: ENGINE: 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore' DOC_STORE_CONFIG: "{{ EDXAPP_LMS_SPLIT_DOC_STORE_CONFIG }}" OPTIONS: - default_class: 'xmodule.hidden_module.HiddenDescriptor' + default_class: 'xmodule.hidden_block.HiddenBlock' fs_root: "{{ edxapp_course_data_dir }}" render_template: 'common.djangoapps.edxmako.shortcuts.render_to_string' - NAME: 'draft' ENGINE: 'xmodule.modulestore.mongo.DraftMongoModuleStore' DOC_STORE_CONFIG: "{{ EDXAPP_LMS_DRAFT_DOC_STORE_CONFIG }}" OPTIONS: - default_class: 'xmodule.hidden_module.HiddenDescriptor' + default_class: 'xmodule.hidden_block.HiddenBlock' fs_root: "{{ edxapp_course_data_dir }}" render_template: 'common.djangoapps.edxmako.shortcuts.render_to_string' SOCIAL_AUTH_OAUTH_SECRETS: "{{ EDXAPP_SOCIAL_AUTH_OAUTH_SECRETS }}" @@ -1695,14 +1695,14 @@ cms_auth_config: ENGINE: 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore' DOC_STORE_CONFIG: "{{ EDXAPP_CMS_DOC_STORE_CONFIG }}" OPTIONS: - default_class: 'xmodule.hidden_module.HiddenDescriptor' + default_class: 'xmodule.hidden_block.HiddenBlock' fs_root: "{{ edxapp_course_data_dir }}" render_template: 'common.djangoapps.edxmako.shortcuts.render_to_string' - NAME: 'draft' ENGINE: 'xmodule.modulestore.mongo.DraftMongoModuleStore' DOC_STORE_CONFIG: "{{ EDXAPP_CMS_DOC_STORE_CONFIG }}" OPTIONS: - default_class: 'xmodule.hidden_module.HiddenDescriptor' + default_class: 'xmodule.hidden_block.HiddenBlock' fs_root: "{{ edxapp_course_data_dir }}" render_template: 'common.djangoapps.edxmako.shortcuts.render_to_string' SEGMENT_KEY: "{{ EDXAPP_CMS_SEGMENT_KEY }}" From a7ca597477c6757d99da6069cbd18b822185d956 Mon Sep 17 00:00:00 2001 From: attiyaishaque Date: Wed, 18 Jan 2023 21:24:00 +0500 Subject: [PATCH 385/664] fix: add the CONFIGURABLE_EDX_FIELDS settings --- playbooks/roles/authn/defaults/main.yml | 2 ++ playbooks/roles/authn/tasks/main.yml | 1 + 2 files changed, 3 insertions(+) create mode 100644 playbooks/roles/authn/defaults/main.yml diff --git a/playbooks/roles/authn/defaults/main.yml b/playbooks/roles/authn/defaults/main.yml new file mode 100644 index 00000000000..c7a95c4039e --- /dev/null +++ b/playbooks/roles/authn/defaults/main.yml @@ -0,0 +1,2 @@ +authn_env_extra: + SHOW_CONFIGURABLE_EDX_FIELDS: "{{ AUTHN_SHOW_CONFIGURABLE_EDX_FIELDS }}" diff --git a/playbooks/roles/authn/tasks/main.yml b/playbooks/roles/authn/tasks/main.yml index 4a1aeb518fd..8bfaab1af34 100644 --- a/playbooks/roles/authn/tasks/main.yml +++ b/playbooks/roles/authn/tasks/main.yml @@ -6,3 +6,4 @@ MFE_VERSION: '{{ AUTHN_MFE_VERSION }}' MFE_SITE_NAME: 'Your Platform Name Here' MFE_NPM_OVERRIDES: '{{ AUTHN_MFE_NPM_OVERRIDES | default(MFE_DEPLOY_NPM_OVERRIDES) }}' + MFE_ENVIRONMENT_EXTRA: '{{ authn_env_extra | default(MFE_DEPLOY_ENVIRONMENT_EXTRA) }}' From 350af10475e655185d2131f958ca7abb8c1de3a1 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 18 Jan 2023 13:00:57 -0500 Subject: [PATCH 386/664] fix: update runbook to use python3-compatible python library JIRA:PSRE-2153 --- playbooks/create_db_and_users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/create_db_and_users.yml b/playbooks/create_db_and_users.yml index c3841d70bb4..5324f5048c0 100644 --- a/playbooks/create_db_and_users.yml +++ b/playbooks/create_db_and_users.yml @@ -59,7 +59,7 @@ - name: install python mysqldb module pip: name={{item}} state=present with_items: - - MySQL-python + - mysqlclient - name: create mysql databases mysql_db: From e14220bb51440b23134e830130e4c6072ad8ccb4 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 25 Jan 2023 14:49:20 +0500 Subject: [PATCH 387/664] fix: use get_url to download and install docker-compose --- playbooks/roles/docker-tools/defaults/main.yml | 1 + playbooks/roles/docker-tools/tasks/main.yml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/docker-tools/defaults/main.yml b/playbooks/roles/docker-tools/defaults/main.yml index 3e56fd9be31..8d46f0513d6 100644 --- a/playbooks/roles/docker-tools/defaults/main.yml +++ b/playbooks/roles/docker-tools/defaults/main.yml @@ -10,6 +10,7 @@ docker_tools_deps_deb_pkgs_focal: - ca-certificates - python3-pip +docker_compose_pkg_url: "https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-x86_64" docker_apt_key_url: "https://download.docker.com/linux/ubuntu/gpg" docker_repos: - "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml index 6a3ab29e00f..34e010eb769 100644 --- a/playbooks/roles/docker-tools/tasks/main.yml +++ b/playbooks/roles/docker-tools/tasks/main.yml @@ -80,8 +80,10 @@ - install:configuration - name: install docker-compose - pip: - name: docker-compose + get_url: + dest: /usr/local/bin/docker-compose + url: "{{ docker_compose_pkg_url }}" + mode: 0755 tags: - install - install:system-requirements From 2e8855d7b6a6d7c26c764a37eac928a567d33954 Mon Sep 17 00:00:00 2001 From: Darius Zivertas Date: Wed, 25 Jan 2023 15:33:04 +0200 Subject: [PATCH 388/664] chore: Update PROSPECTUS_NODE_VERSION 16.13.0 -> 18.13.0 https://2u-internal.atlassian.net/browse/SEO-90 --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 7de92dedc8f..bda2ef045f8 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '16.13.0' +PROSPECTUS_NODE_VERSION: '18.13.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From 1b2c58771d920072abd268c8f1171f910fd5bd21 Mon Sep 17 00:00:00 2001 From: Mohammad Ahtasham ul Hassan <60315450+aht007@users.noreply.github.com> Date: Mon, 30 Jan 2023 18:30:36 +0500 Subject: [PATCH 389/664] build: remove ansible free services code (#6871) --- docker/build/analytics_api/Dockerfile | 34 ------- docker/build/analytics_api/analytics_api.yml | 77 ---------------- .../build/analytics_api/ansible_overrides.yml | 25 ------ docker/build/discovery/Dockerfile | 33 ------- docker/build/discovery/ansible_overrides.yml | 15 ---- docker/build/discovery/discovery.yml | 90 ------------------- docker/plays/analytics_api.yml | 10 --- docker/plays/discovery.yml | 9 -- 8 files changed, 293 deletions(-) delete mode 100644 docker/build/analytics_api/Dockerfile delete mode 100644 docker/build/analytics_api/analytics_api.yml delete mode 100644 docker/build/analytics_api/ansible_overrides.yml delete mode 100644 docker/build/discovery/Dockerfile delete mode 100644 docker/build/discovery/ansible_overrides.yml delete mode 100644 docker/build/discovery/discovery.yml delete mode 100644 docker/plays/analytics_api.yml delete mode 100644 docker/plays/discovery.yml diff --git a/docker/build/analytics_api/Dockerfile b/docker/build/analytics_api/Dockerfile deleted file mode 100644 index 0503b5b3bc9..00000000000 --- a/docker/build/analytics_api/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/analytics_api/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" - -RUN apt-get update - -ADD . /edx/app/edx_ansible/edx_ansible -COPY docker/build/analytics_api/ansible_overrides.yml / -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays - -COPY docker/build/analytics_api/ansible_overrides.yml / -COPY docker/build/analytics_api/analytics_api.yml /edx/etc/analytics_api.yml - -ARG OPENEDX_RELEASE=master -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook analytics_api.yml \ - -i '127.0.0.1,' \ - -c local \ - -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code,devstack:install" \ - --extra-vars="ANALYTICS_API_VERSION=${OPENEDX_RELEASE}" \ - --extra-vars="@/ansible_overrides.yml" -WORKDIR /edx/app/ -ENTRYPOINT ["/edx/app/analytics_api/devstack.sh"] -CMD ["start"] -EXPOSE 443 80 diff --git a/docker/build/analytics_api/analytics_api.yml b/docker/build/analytics_api/analytics_api.yml deleted file mode 100644 index 612c0591911..00000000000 --- a/docker/build/analytics_api/analytics_api.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- - -AGGREGATE_PAGE_SIZE: 10 -ANALYTICS_DATABASE: reports -API_AUTH_TOKEN: put-your-api-token-here -API_ROOT: null -BACKEND_SERVICE_EDX_OAUTH2_KEY: analytics_api-backend-service-key -BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://127.0.0.1:8000/oauth2 -BACKEND_SERVICE_EDX_OAUTH2_SECRET: analytics_api-backend-service-secret -CACHES: - default: - BACKEND: django.core.cache.backends.memcached.MemcachedCache - KEY_PREFIX: analytics_api - LOCATION: - - memcache -CSRF_COOKIE_SECURE: false -DATABASES: - default: - ENGINE: django.db.backends.mysql - HOST: db.edx - NAME: analytics-api - PASSWORD: password - PORT: '3306' - USER: api001 - reports: - ENGINE: django.db.backends.mysql - HOST: db.edx - NAME: reports - PASSWORD: password - PORT: '3306' - USER: reports001 -DATETIME_FORMAT: '%Y-%m-%dT%H%M%S' -DATE_FORMAT: '%Y-%m-%d' -DEFAULT_PAGE_SIZE: 25 -EDX_DRF_EXTENSIONS: - OAUTH2_USER_INFO_URL: http://127.0.0.1:8000/oauth2/user_info -ELASTICSEARCH_AWS_ACCESS_KEY_ID: null -ELASTICSEARCH_AWS_SECRET_ACCESS_KEY: null -ELASTICSEARCH_CONNECTION_CLASS: null -ELASTICSEARCH_CONNECTION_DEFAULT_REGION: us-east-1 -ELASTICSEARCH_LEARNERS_HOST: edx.devstack.elasticsearch -ELASTICSEARCH_LEARNERS_INDEX: roster_1_2 -ELASTICSEARCH_LEARNERS_UPDATE_INDEX: index_updates -EXTRA_APPS: [] -JWT_AUTH: - JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload - JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature - JWT_ISSUERS: - - AUDIENCE: SET-ME-PLEASE - ISSUER: http://127.0.0.1:8000/oauth2 - SECRET_KEY: SET-ME-PLEASE - JWT_PUBLIC_SIGNING_JWK_SET: '' -LANGUAGE_CODE: en-us -LMS_BASE_URL: http://127.0.0.1:8000/ -MAX_PAGE_SIZE: 100 -MEDIA_STORAGE_BACKEND: - DEFAULT_FILE_STORAGE: django.core.files.storage.FileSystemStorage - MEDIA_ROOT: /edx/var/analytics_api/media - MEDIA_URL: /media/ -REPORT_DOWNLOAD_BACKEND: - COURSE_REPORT_FILE_LOCATION_TEMPLATE: '{course_id}_{report_name}.csv' - DEFAULT_FILE_STORAGE: django.core.files.storage.FileSystemStorage - MEDIA_ROOT: /edx/var/analytics_api/static/reports - MEDIA_URL: http://localhost:8100/static/reports/ -SECRET_KEY: Your secret key here -SESSION_EXPIRE_AT_BROWSER_CLOSE: false -SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://127.0.0.1:8000 -SOCIAL_AUTH_EDX_OAUTH2_KEY: analytics_api-sso-key -SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://127.0.0.1:8000/logout -SOCIAL_AUTH_EDX_OAUTH2_SECRET: analytics_api-sso-secret -SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://127.0.0.1:8000 -SOCIAL_AUTH_REDIRECT_IS_HTTPS: false -STATICFILES_DIRS: -- static -STATICFILES_STORAGE: django.contrib.staticfiles.storage.StaticFilesStorage -STATIC_ROOT: /edx/var/analytics_api/staticfiles -TIME_ZONE: UTC diff --git a/docker/build/analytics_api/ansible_overrides.yml b/docker/build/analytics_api/ansible_overrides.yml deleted file mode 100644 index d5b6dc54258..00000000000 --- a/docker/build/analytics_api/ansible_overrides.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -edx_django_service_is_devstack: True - -ANALYTICS_API_DJANGO_SETTINGS_MODULE: "analyticsdataserver.settings.devstack" - -DOCKER_TLD: "edx" - -ANALYTICS_API_DATABASES: - # rw user - default: - ENGINE: 'django.db.backends.mysql' - NAME: '{{ ANALYTICS_API_DEFAULT_DB_NAME }}' - USER: 'api001' - PASSWORD: 'password' - HOST: 'db.{{ DOCKER_TLD }}' - PORT: '3306' - # read-only user - reports: - ENGINE: 'django.db.backends.mysql' - NAME: '{{ ANALYTICS_API_REPORTS_DB_NAME }}' - USER: 'reports001' - PASSWORD: 'password' - HOST: "db.{{ DOCKER_TLD }}" - PORT: '3306' diff --git a/docker/build/discovery/Dockerfile b/docker/build/discovery/Dockerfile deleted file mode 100644 index 8a3dc3578ec..00000000000 --- a/docker/build/discovery/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/discovery/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" -USER root -ENTRYPOINT ["/edx/app/discovery/devstack.sh"] -CMD ["start"] - -ADD . /edx/app/edx_ansible/edx_ansible -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays - -COPY docker/build/discovery/ansible_overrides.yml / -COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml -COPY docker/build/discovery/discovery.yml /edx/etc/discovery.yml - -ARG OPENEDX_RELEASE=master -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook discovery.yml \ - -c local -i '127.0.0.1,' \ - -t 'install,assets,devstack' \ - --extra-vars="DISCOVERY_VERSION=${OPENEDX_RELEASE}" \ - --extra-vars="@/ansible_overrides.yml" \ - --extra-vars="@/devstack/ansible_overrides.yml" - -EXPOSE 18381 diff --git a/docker/build/discovery/ansible_overrides.yml b/docker/build/discovery/ansible_overrides.yml deleted file mode 100644 index 10c12f7a52d..00000000000 --- a/docker/build/discovery/ansible_overrides.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -COMMON_GIT_PATH: 'edx' - -COMMON_MYSQL_MIGRATE_USER: '{{ DISCOVERY_MYSQL_USER }}' -COMMON_MYSQL_MIGRATE_PASS: '{{ DISCOVERY_MYSQL_PASSWORD }}' - -DISCOVERY_MYSQL: 'edx.devstack.mysql57' -DISCOVERY_MYSQL_REPLICA_HOST: 'edx.devstack.mysql57' -DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack' -DISCOVERY_ELASTICSEARCH_HOST: 'edx.devstack.elasticsearch' -DISCOVERY_GUNICORN_EXTRA: '--reload' -DISCOVERY_MEMCACHE: ['edx.devstack.memcached:11211'] -DISCOVERY_EXTRA_APPS: ['course_discovery.apps.edx_catalog_extensions'] - -edx_django_service_is_devstack: true diff --git a/docker/build/discovery/discovery.yml b/docker/build/discovery/discovery.yml deleted file mode 100644 index 3836223b4cd..00000000000 --- a/docker/build/discovery/discovery.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- - -API_ROOT: null -AWS_SES_REGION_ENDPOINT: email.us-east-1.amazonaws.com -AWS_SES_REGION_NAME: us-east-1 -BACKEND_SERVICE_EDX_OAUTH2_KEY: discovery-backend-service-key -BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL: http://localhost:18000/oauth2 -BACKEND_SERVICE_EDX_OAUTH2_SECRET: discovery-backend-service-secret -CACHES: - default: - BACKEND: django.core.cache.backends.memcached.MemcachedCache - KEY_PREFIX: discovery - LOCATION: - - edx.devstack.memcached:11211 -CORS_ORIGIN_WHITELIST: [] -CSRF_COOKIE_SECURE: false -DATABASES: - default: - ATOMIC_REQUESTS: 'false' - CONN_MAX_AGE: 60 - ENGINE: django.db.backends.mysql - HOST: edx.devstack.mysql57 - NAME: discovery - OPTIONS: - connect_timeout: 10 - init_command: SET sql_mode='STRICT_TRANS_TABLES' - PASSWORD: password - PORT: 3306 - USER: discov001 - read_replica: - ATOMIC_REQUESTS: 'false' - CONN_MAX_AGE: 60 - ENGINE: django.db.backends.mysql - HOST: edx.devstack.mysql57 - NAME: discovery - OPTIONS: - connect_timeout: 10 - init_command: SET sql_mode='STRICT_TRANS_TABLES' - PASSWORD: password - PORT: 3306 - USER: discov001 -DEFAULT_PARTNER_ID: 1 -EDX_DRF_EXTENSIONS: - OAUTH2_USER_INFO_URL: http://edx.devstack.lms:18000/oauth2/user_info -ELASTICSEARCH_INDEX_NAME: catalog -ELASTICSEARCH_URL: http://127.0.0.1:9200/ -EMAIL_BACKEND: django_ses.SESBackend -EMAIL_HOST: localhost -EMAIL_HOST_PASSWORD: '' -EMAIL_HOST_USER: '' -EMAIL_PORT: 25 -EMAIL_USE_TLS: false -ENABLE_PUBLISHER: false -EXTRA_APPS: -- course_discovery.apps.edx_catalog_extensions -JWT_AUTH: - JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload - JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature - JWT_ISSUERS: - - AUDIENCE: lms-key - ISSUER: http://localhost:18000/oauth2 - SECRET_KEY: lms-secret - JWT_PUBLIC_SIGNING_JWK_SET: '' -LANGUAGE_CODE: en -MEDIA_STORAGE_BACKEND: - DEFAULT_FILE_STORAGE: django.core.files.storage.FileSystemStorage - MEDIA_ROOT: /edx/var/discovery/media - MEDIA_URL: /media/ -OPENEXCHANGERATES_API_KEY: '' -PARLER_DEFAULT_LANGUAGE_CODE: en -PARLER_LANGUAGES: - 1: - - code: en - default: - fallbacks: - - en - hide_untranslated: 'False' -PLATFORM_NAME: Your Platform Name Here -PUBLISHER_FROM_EMAIL: null -SECRET_KEY: Your secret key here -SESSION_EXPIRE_AT_BROWSER_CLOSE: false -SOCIAL_AUTH_EDX_OAUTH2_ISSUER: http://127.0.0.1:8000 -SOCIAL_AUTH_EDX_OAUTH2_KEY: discovery-sso-key -SOCIAL_AUTH_EDX_OAUTH2_LOGOUT_URL: http://localhost:18000/logout -SOCIAL_AUTH_EDX_OAUTH2_SECRET: discovery-sso-secret -SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT: http://127.0.0.1:8000 -STATICFILES_STORAGE: django.contrib.staticfiles.storage.StaticFilesStorage -STATIC_ROOT: /edx/var/discovery/staticfiles -TIME_ZONE: UTC -USERNAME_REPLACEMENT_WORKER: OVERRIDE THIS WITH A VALID USERNAME diff --git a/docker/plays/analytics_api.yml b/docker/plays/analytics_api.yml deleted file mode 100644 index 25565cb0d46..00000000000 --- a/docker/plays/analytics_api.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deploy Analytics API - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - common - - analytics_api diff --git a/docker/plays/discovery.yml b/docker/plays/discovery.yml deleted file mode 100644 index f4737f7fb35..00000000000 --- a/docker/plays/discovery.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: Deploy Discovery - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - discovery From 830c1442d9cb0ed5a86f8766d165d567fb59028c Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Fri, 20 Jan 2023 17:31:08 +0500 Subject: [PATCH 390/664] feat: updated sandbox deployment scripts to setup edx-platform in docker containers. The scripts do not use Ansible to deploy edx-platform in containers and the entire automation is writen in bash. The mechanism uses Fluend for separating tracking logs. --- CHANGELOG.md | 4 + playbooks/edx_continuous_integration.yml | 8 +- playbooks/edx_provision.yml | 1 + util/jenkins/ansible-provision.sh | 359 ++++++++++--------- util/jenkins/app-container-provisioner.sh | 247 +++++++++++++ util/jenkins/demo-course-provisioner.sh | 39 ++ util/jenkins/worker-container-provisioner.sh | 17 +- 7 files changed, 490 insertions(+), 185 deletions(-) create mode 100644 util/jenkins/app-container-provisioner.sh create mode 100644 util/jenkins/demo-course-provisioner.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 444110d661a..68ece1587a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-01-18 + - Updated sandbox deployment scripts to setup edx-platform in docker containers. + - Added Fluentd for checking tracking logs when running edx-platform in docker containers. + - 2022-07-25 - Add new routing key for individual learner course regrade queue - 2022-06-08 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index e7cbefcfa62..0928b05a9cf 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,9 +25,9 @@ - memcache - mongo_4_2 - role: redis - - { role: 'edxapp', celery_worker: True, when: edxapp_celery_worker is defined and edxapp_celery_worker } - - edxapp - - testcourses + - { role: "edxapp", celery_worker: True, when: edxapp_containerized is defined and not edxapp_containerized } + - { role: "edxapp", when: edxapp_containerized is defined and not edxapp_containerized } + - { role: "testcourses", when: edxapp_containerized is defined and not edxapp_containerized } - oraclejdk - elasticsearch - opensearch @@ -63,7 +63,7 @@ - role: mfe MFE_NAME: ora-grading when: ORA_GRADING_MFE_ENABLED - - oauth_client_setup + - { role: oauth_client_setup, when: edxapp_containerized is defined and not edxapp_containerized } - role: datadog when: COMMON_ENABLE_DATADOG - role: splunkforwarder diff --git a/playbooks/edx_provision.yml b/playbooks/edx_provision.yml index da84cc38e27..8b80c25ae08 100644 --- a/playbooks/edx_provision.yml +++ b/playbooks/edx_provision.yml @@ -82,6 +82,7 @@ - forum nginx_default_sites: - lms + - docker-tools post_tasks: - name: get instance id for elb registration local_action: diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 103f1da5b18..77539eef411 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -46,6 +46,40 @@ run_ansible() { fi } +# Install yq +wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_amd64 -O $WORKSPACE/yq && chmod +x $WORKSPACE/yq + +function provision_fluentd() { + echo "#!/usr/bin/env bash" + echo "set -ex" + + # add tracking log file to host instance + echo "touch /var/tmp/tracking_logs.log" + echo "chown www-data:www-data /var/tmp/tracking_logs.log" + + echo "docker pull fluent/fluentd:edge-debian" + + # create fluentd config + echo "fluentd_config=/var/tmp/fluentd.conf" + echo "cat << 'EOF' > \$fluentd_config + + @type tail + path /var/tmp/tracking_logs.log + pos_file /var/tmp/tracking_logs.pos + rotate_wait 10 + tag * + + @type none + + + + + @type stdout + +EOF" + echo "docker run -d --name fluentd --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf -v /var/tmp:/var/tmp fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" +} + # This DATE_TIME will be used as instance launch time tag if [[ ! -n ${sandbox_life//[0-9]/} ]] && [[ ${sandbox_life} -le 30 ]]; then TERMINATION_DATE_TIME=`date +"%m-%d-%Y %T" --date "${sandbox_life=7} days"` @@ -67,8 +101,12 @@ fi if [[ -z $WORKSPACE ]]; then dir=$(dirname $0) source "$dir/ascii-convert.sh" + source "$dir/app-container-provisioner.sh" + source "$dir/demo-course-provisioner.sh" else source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh" + source "$WORKSPACE/configuration/util/jenkins/app-container-provisioner.sh" + source "$WORKSPACE/configuration/util/jenkins/demo-course-provisioner.sh" fi if [[ -z $static_url_base ]]; then @@ -606,46 +644,159 @@ done # run non-deploy tasks for all plays if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" || $server_type == "ubuntu_20.04" ]]; then cat $extra_vars_file - if [[ $edxapp_workers_docker_container_enabled == "true" ]]; then - run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg -e edxapp_celery_worker=false --user ubuntu + if [[ $edxapp_container_enabled == "true" ]]; then + cat << EOF > $WORKSPACE/edxapp_extra_var.yml +edxapp_containerized: true +CAN_GENERATE_NEW_JWT_SIGNATURE: false +EOF + ansible -i "${deploy_host}," $deploy_host -m include_role -a "name=memcache" -u ubuntu -b + for playbook in redis mongo_4_2; do + run_ansible $playbook.yml -i "${deploy_host}," $extra_var_arg --user ubuntu + done + run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu --tags "edxlocal" + # create fluentd container for processing tracking logs + provision_fluentd_script="/var/tmp/provision-fluentd-script.sh" + cat << EOF > $provision_fluentd_script +$(provision_fluentd) +EOF + ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_fluentd_script}" -u ubuntu -b + + rm -f "${provision_fluentd_script}" + + # decrypt lms config file + asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/lms.yml --private_key_path $WORKSPACE/configuration-secure/ansible/keys/sandbox-remote-config/sandbox/private.key --outfile_path $WORKSPACE/lms.yml + # decrypt cms config file + asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/studio.yml --private_key_path $WORKSPACE/configuration-secure/ansible/keys/sandbox-remote-config/sandbox/private.key --outfile_path $WORKSPACE/cms.yml + + sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/lms.yml + sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/cms.yml + + # copy app config file + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/lms.yml dest=/var/tmp/lms.yml" -u ubuntu -b + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/cms.yml dest=/var/tmp/cms.yml" -u ubuntu -b + + set +x + app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" + + # specify variable names + app_hostname="courses" + app_service_name="lms" + app_name="edxapp" + app_repo="edx-platform" + app_version=$edxapp_version + app_gunicorn_port=8000 + app_cfg=LMS_CFG + app_admin_password=SANDBOX_ADMIN_PASSWORD + + app_provision_script="/var/tmp/app-container-provision-script-$$.sh" + + write_app_deployment_script $app_provision_script + set -x + + ssh \ + -o ControlMaster=auto \ + -o ControlPersist=60s \ + -o "ControlPath=/tmp/${app_service_name}-ssh-%h-%p-%r" \ + -o ServerAliveInterval=30 \ + -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ubuntu@${deploy_host} "sudo -n -s bash" < $app_provision_script + + rm -f "${app_provision_script}" + + # create CMS provision script + # specify variable names + app_hostname="studio" + app_service_name="cms" + app_name="edxapp" + app_repo="edx-platform" + app_version=$edxapp_version + app_gunicorn_port=8010 + app_cfg=CMS_CFG + + app_provision_script="/var/tmp/app-container-provision-script-$$.sh" + + write_app_deployment_script $app_provision_script + set -x + + ssh \ + -o ControlMaster=auto \ + -o ControlPersist=60s \ + -o "ControlPath=/tmp/${app_service_name}-ssh-%h-%p-%r" \ + -o ServerAliveInterval=30 \ + -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + ubuntu@${deploy_host} "sudo -n -s bash" < $app_provision_script + + rm -f "${app_provision_script}" + + # set admin password for demo users + set +x + admin_hashed_password="$($WORKSPACE/yq '.SANDBOX_ADMIN_PASSWORD' $WORKSPACE/configuration-internal/ansible/vars/developer-sandbox.yml)" + + # create demo course and test users + demo_course_provision_script="/var/tmp/demo-provision-script.sh" + write_demo_course_script $demo_course_provision_script + set -x + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${deploy_host} "sudo -n -s bash" < $demo_course_provision_script + + rm -f "${demo_course_provision_script}" + + # edxapp celery workers # Export LC_* vars. To be passed to remote instance via SSH where SSH configuration allows LC_* to be accepted as environment variables. # LC_* is normally used for passing through locale settings of SSH clients to SSH servers. export LC_WORKER_CFG=$(cat < $WORKSPACE/edxapp_extra_var.yml +edxapp_containerized: false +EOF + run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg -e @$WORKSPACE/edxapp_extra_var.yml --user ubuntu fi fi @@ -671,10 +822,10 @@ if [[ $ret -ne 0 ]]; then exit $ret fi -if [[ $run_oauth == "true" ]]; then - # Setup the OAuth2 clients - run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu -fi +#if [[ $run_oauth == "true" ]]; then +# # Setup the OAuth2 clients +# run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu +#fi # set the hostname run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu @@ -700,118 +851,6 @@ if [[ $enable_newrelic == "true" ]]; then run_ansible run_role.yml -i "${deploy_host}," -e role=newrelic_infrastructure $extra_var_arg --user ubuntu fi -function provision_containerized_app() { - echo "#!/usr/bin/env bash" - echo "set -ex" - - # Create app staticfiles dir - echo "mkdir /edx/var/${app_service_name}/staticfiles/ -p && chmod 777 /edx/var/${app_service_name} -R" - - # Checkout code in app directory - echo "cd /edx/app/" - echo "git clone https://github.com/edx/${app_repo}.git" - - # Replace deploy_host in app config file with sandbox DNS name - echo "sed -i 's/deploy_host/${dns_name}.${dns_zone}/g' /var/tmp/${app_service_name}.yml" - - # Install yq for yaml processing - echo "wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq" - - # Combine app config with jwt_signature config - echo "yq eval-all '. as \$item ireduce ({}; . *+ \$item)' /var/tmp/${app_service_name}.yml /tmp/lms_jwt_signature.yml > /edx/etc/${app_service_name}.yml" - - # Provision IDA User in LMS - echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production manage_user ${app_service_name}_worker ${app_service_name}_worker@example.com --staff --superuser" - - # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. - echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris 'https://${app_hostname}-${dns_name}.${dns_zone}/complete/edx-oauth2/' --client-id '${app_service_name}-sso-key' --client-secret '${app_service_name}-sso-secret' --scopes 'user_id' ${app_service_name}-sso ${app_service_name}_worker" - echo "source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=production create_dot_application --grant-type client-credentials --client-id '${app_service_name}-backend-service-key' --client-secret '${app_service_name}-backend-service-secret' ${app_service_name}-backend-service ${app_service_name}_worker" - - # Checkout code version - echo "cd /edx/app/${app_repo}" - echo "git checkout ${app_version}" - - # Create app database - echo "mysql -uroot -e \"CREATE DATABASE \\\`${app_service_name}\\\`;\"" - - # use heredoc to dynamically create docker compose file - echo "docker_compose_file=/var/tmp/docker-compose-${app_service_name}.yml" - echo "cat << 'EOF' > \$docker_compose_file - version: '2.1' - services: - app: - image: ${app_service_name}:latest - stdin_open: true - tty: true - build: - context: /edx/app/${app_repo} - dockerfile: Dockerfile - container_name: ${app_service_name}.app - command: bash -c 'while true; do exec gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_repo}/${app_service_name}/docker_gunicorn_configuration.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application; sleep 2; done' - network_mode: 'host' - environment: - DJANGO_SETTINGS_MODULE: ${app_service_name}.settings.production - DJANGO_WATCHMAN_TIMEOUT: 30 - ENABLE_DJANGO_TOOLBAR: 1 - ${app_cfg}: /${app_service_name}.yml - volumes: - - /edx/app/${app_repo}:/edx/app/${app_repo}/ - - /edx/etc/${app_service_name}.yml:/${app_service_name}.yml - - /edx/var/${app_service_name}/staticfiles/:/var/tmp/ -EOF" - - # run docker compose to spin up service container - echo "docker-compose -f \$docker_compose_file up -d" - - # Wait for app container - echo "sleep 5" - - # Run migrations - echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py migrate\"" - - # Run collectstatic - echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py collectstatic --noinput\"" - # Create superuser - echo "docker exec -t ${app_service_name}.app bash -c \"echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\\\"edx\\\", \\\"edx@example.com\\\", \\\"edx\\\") if not User.objects.filter(username=\\\"edx\\\").exists() else None' | python /edx/app/${app_repo}/manage.py shell\"" - - # Create Nginx config - echo "site_config=/edx/app/nginx/sites-available/${app_service_name}" - echo "cat << 'EOF' > \$site_config - server { - server_name ~^((stage|prod)-)?${app_hostname}.*; - listen 80; - rewrite ^ https://\$host\$request_uri? permanent; - } - server { - server_name ~^((stage|prod)-)?${app_hostname}.*; - listen 443 ssl; - ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; - ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; - - location / { - try_files \$uri @proxy_to_app; - } - location ~ ^/(api)/ { - try_files \$uri @proxy_to_app; - } - location @proxy_to_app { - proxy_set_header X-Forwarded-Proto \$scheme; - proxy_set_header X-Forwarded-Port \$server_port; - proxy_set_header X-Forwarded-For \$remote_addr; - proxy_set_header Host \$http_host; - proxy_redirect off; - proxy_pass http://127.0.0.1:${app_gunicorn_port}; - } - location ~ ^/static/(?P.*) { - root /edx/var/${app_service_name}; - try_files /staticfiles/\$file =404; - } - } -EOF" - echo "ln -s /edx/app/nginx/sites-available/${app_service_name} /etc/nginx/sites-enabled/${app_service_name}" - echo "service nginx reload" -} - if [[ $edx_exams == 'true' ]]; then app_hostname="edx-exams" @@ -833,41 +872,5 @@ EOF rm -f "${provision_script}" fi -function provision_fluentd() { - echo "#!/usr/bin/env bash" - echo "set -ex" - - echo "docker pull fluent/fluentd:edge-debian" - - echo "fluentd_config=/var/tmp/fluentd.conf" - echo "cat << 'EOF' > \$fluentd_config - - @type tail - path /var/tmp/tracking_logs.log - pos_file /var/log/tracking_logs.pos - rotate_wait 10 - tag * - - @type none - - - - - @type stdout - -EOF" - echo "docker run -d --network host -v /var/tmp/fluentd.conf:/fluentd/etc/fluentd.conf -v /var/tmp:/var/tmp fluent/fluentd:edge-debian -c /fluentd/etc/fluentd.conf" -} - -if [[ $fluentd_logging == 'true' ]]; then - provision_fluentd_script="/var/tmp/provision-fluentd-script.sh" -cat << EOF > $provision_fluentd_script -$(provision_fluentd) -EOF - ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_fluentd_script}" -u ubuntu -b - - rm -f "${provision_fluentd_script}" -fi - rm -f "$extra_vars_file" rm -f ${extra_vars_file}_clean diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh new file mode 100644 index 00000000000..8b4f62e3aa1 --- /dev/null +++ b/util/jenkins/app-container-provisioner.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash + +set -ex + +function write_app_deployment_script() { + cat < "$1" +#!/usr/bin/env bash + +set -ex + +# Install yq for yaml processing +wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq + +chown :www-data /var/tmp/${app_service_name}.yml + +if [[ ${app_service_name} == 'lms' ]] ; then + chown :www-data /var/tmp/cms.yml +fi + +if [[ ${app_service_name} != 'cms' && ${app_service_name} != 'lms' ]] ; then + # Create app staticfiles dir + mkdir /edx/var/${app_name}/staticfiles/ -p && chmod 777 /edx/var/${app_name} -R +fi + +# if application is lms, download and setup themes +if [[ ${app_service_name} == 'lms' && ! -d /edx/var/edx-themes ]] ; then + set +x + echo -e "${app_git_ssh_key}" > /tmp/theme_ssh_key + set -x + chmod 0600 /tmp/theme_ssh_key + useradd -m -d /edx/var/edx-themes edx-themes -G www-data + GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/theme_ssh_key" git clone git@github.com:edx/edx-themes.git /edx/var/edx-themes/edx-themes + cd /edx/var/edx-themes/edx-themes && git checkout ${themes_version} + chown -R edx-themes:www-data /edx/var/edx-themes + sudo -u edx-themes git config --global safe.directory '/edx/var/edx-themes/edx-themes' + rm -rf /tmp/theme_ssh_key +fi + +# checkout git repo +if [ ! -d "/edx/app/${app_name}" ]; then + mkdir /edx/app/${app_name} +fi + +if [[ ! -d "/edx/app/${app_name}/${app_repo}" ]] ; then + git clone https://github.com/edx/${app_repo}.git /edx/app/${app_name}/${app_repo} + cd /edx/app/${app_name}/${app_repo} && git checkout ${app_version} +fi + +# Generate container image if it doesn't already exist +if ! $(docker image inspect ${app_image_name} >/dev/null 2>&1 && echo true || echo false) ; then + cd /edx/app/${app_name}/${app_repo} + export DOCKER_BUILDKIT=1 + if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then + docker build . -t ${app_repo}:latest --target base + else + docker build . -t ${app_repo}:latest + fi +fi + +# if lms, create image (if it doesn't exist) and generate JWT credentials +if [[ ${app_service_name} == 'lms' ]]; then + touch /tmp/lms_jwt_signature.yml && chmod 777 /tmp/lms_jwt_signature.yml + # generate JWT token, ensure JWT file is mounted as volume + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /tmp/lms_jwt_signature.yml:/tmp/lms_jwt_signature.yml -v /var/tmp/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes ${app_repo}:latest python3 manage.py lms generate_jwt_signing_key --output-file /tmp/lms_jwt_signature.yml --strip-key-prefix +fi + +# Combine app config with jwt_signature config +cat /var/tmp/${app_service_name}.yml /tmp/lms_jwt_signature.yml > /edx/etc/${app_service_name}.yml + +chown :www-data /edx/etc/${app_service_name}.yml + +if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then + # run migrations + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database default + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database default --noinput + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} showmigrations --database student_module_history + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput +else + # Run app migrations + docker run --network=host --rm -u='www-data' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py migrate + # Generate static assets + docker run --network=host --rm -u='root' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py collectstatic --noinput +fi + +# Setup oauth clients for service other than CMS as part of the LMS setup +if [[ ${app_service_name} == 'lms' ]]; then + service_worker_users=(enterprise veda discovery credentials insights registrar designer license_manager commerce_coordinator enterprise_catalog ecommerce retirement) + # Provision IDA User in LMS + for service_worker in "\${service_worker_users[@]}"; do + app_hostname=\${service_worker/_/-} + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms manage_user \${service_worker}_worker \${service_worker}_worker@example.com --staff --superuser + + # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris "https://\${app_hostname}-${dns_name}.${dns_zone}/complete/edx-oauth2/" --client-id "\${service_worker}-sso-key" --client-secret "\${service_worker}-sso-secret" --scopes 'user_id' \${service_worker}-sso \${service_worker}_worker + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms create_dot_application --grant-type client-credentials --client-id "\${service_worker}-backend-service-key" --client-secret "\${service_worker}-backend-service-secret" \${service_worker}-backend-service \${service_worker}_worker + done +fi + +# oauth client setup +if [[ ${app_service_name} != 'lms' ]]; then + # Provision IDA User in LMS + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms manage_user $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker@example.com --staff --superuser + + # Create the DOT applications - one for single sign-on and one for backend service IDA-to-IDA authentication. + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms create_dot_application --grant-type authorization-code --skip-authorization --redirect-uris 'https://${app_hostname}-${dns_name}.${dns_zone}/complete/edx-oauth2/' --client-id '$(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-sso-key' --client-secret '$(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-sso-secret' --scopes 'user_id' $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-sso $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker + docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms create_dot_application --grant-type client-credentials --client-id '$(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-backend-service-key' --client-secret '$(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-backend-service-secret' $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}-; fi)${app_service_name}-backend-service $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker +fi + +# generate lms/cms static assets +if [[ ${app_service_name} == 'lms' ]]; then + # temporary hack, create npm-install.log file + touch /edx/app/edxapp/edx-platform/test_root/log/npm-install.log + docker run --network=host --rm -u='root' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/${app_service_name}.yml -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /var/tmp/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/${app_name}:/edx/var/${app_name} -v /edx/app/edxapp/edx-platform/test_root/log/npm-install.log:/edx/app/edxapp/edx-platform/test_root/log/npm-install.log -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest paver update_assets --debug-collect --settings=docker-production +fi + +# Generate docker-compose file for app service +cat < /home/$github_username/docker-compose-${app_service_name}.yml +version: "3.9" +services: + ${app_service_name}: + image: ${app_repo}:latest + stdin_open: true + tty: true + container_name: ${app_service_name} + command: bash -c "gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_name}/${app_repo}/${app_service_name}/docker_${app_service_name}_gunicorn.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application" + user: "www-data:www-data" + network_mode: 'host' + restart: on-failure + environment: + - ${app_cfg}=/edx/etc/${app_service_name}.yml + - EDX_REST_API_CLIENT_NAME=sandbox-edx-${app_service_name} +$( + if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then + echo -e " - DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production" + echo -e " - EDX_PLATFORM_SETTINGS=docker-production" + echo -e " - SERVICE_VARIANT=${app_service_name}" + else + echo -e " - DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production" + fi +) + volumes: + - /edx/var/${app_name}:/edx/var/${app_name} + - /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml + - /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock +$( + if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then + echo -e " - /edx/var/edx-themes:/edx/var/edx-themes" + echo -e " - /var/tmp/tracking_logs.log:/var/tmp/tracking_logs.log" + fi +) +$( + if [[ ${app_service_name} == 'cms' ]]; then + echo -e " - /edx/app/demo:/edx/app/demo" + fi +) +EOT + +docker-compose -f /home/$github_username/docker-compose-${app_service_name}.yml up -d + +EOF + + + + + + + +# # Create app database +# echo "mysql -uroot -e \"CREATE DATABASE \\\`${app_service_name}\\\`;\"" +# +# # use heredoc to dynamically create docker compose file +# echo "docker_compose_file=/var/tmp/docker-compose-${app_service_name}.yml" +# echo "cat << 'EOF' > \$docker_compose_file +# version: '2.1' +# services: +# app: +# image: ${app_service_name}:latest +# stdin_open: true +# tty: true +# build: +# context: /edx/app/${app_repo} +# dockerfile: Dockerfile +# container_name: ${app_service_name}.app +# command: bash -c 'while true; do exec gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_repo}/${app_service_name}/docker_gunicorn_configuration.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application; sleep 2; done' +# network_mode: 'host' +# environment: +# DJANGO_SETTINGS_MODULE: ${app_service_name}.settings.production +# DJANGO_WATCHMAN_TIMEOUT: 30 +# ENABLE_DJANGO_TOOLBAR: 1 +# ${app_cfg}: /${app_service_name}.yml +# volumes: +# - /edx/app/${app_repo}:/edx/app/${app_repo}/ +# - /edx/etc/${app_service_name}.yml:/${app_service_name}.yml +# - /edx/var/${app_service_name}/staticfiles/:/var/tmp/ +#EOF" +# +# # run docker compose to spin up service container +# echo "docker-compose -f \$docker_compose_file up -d" +# +# # Wait for app container +# echo "sleep 5" +# +# # Run migrations +# echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py migrate\"" +# +# # Run collectstatic +# echo "docker exec -t ${app_service_name}.app bash -c \"python3 manage.py collectstatic --noinput\"" +# # Create superuser +# echo "docker exec -t ${app_service_name}.app bash -c \"echo 'from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(\\\"edx\\\", \\\"edx@example.com\\\", \\\"edx\\\") if not User.objects.filter(username=\\\"edx\\\").exists() else None' | python /edx/app/${app_repo}/manage.py shell\"" +# +# # Create Nginx config +# echo "site_config=/edx/app/nginx/sites-available/${app_service_name}" +# echo "cat << 'EOF' > \$site_config +# server { +# server_name ~^((stage|prod)-)?${app_hostname}.*; +# listen 80; +# rewrite ^ https://\$host\$request_uri? permanent; +# } +# server { +# server_name ~^((stage|prod)-)?${app_hostname}.*; +# listen 443 ssl; +# ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; +# ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; +# +# location / { +# try_files \$uri @proxy_to_app; +# } +# location ~ ^/(api)/ { +# try_files \$uri @proxy_to_app; +# } +# location @proxy_to_app { +# proxy_set_header X-Forwarded-Proto \$scheme; +# proxy_set_header X-Forwarded-Port \$server_port; +# proxy_set_header X-Forwarded-For \$remote_addr; +# proxy_set_header Host \$http_host; +# proxy_redirect off; +# proxy_pass http://127.0.0.1:${app_gunicorn_port}; +# } +# location ~ ^/static/(?P.*) { +# root /edx/var/${app_service_name}; +# try_files /staticfiles/\$file =404; +# } +# } +#EOF" +# echo "ln -s /edx/app/nginx/sites-available/${app_service_name} /etc/nginx/sites-enabled/${app_service_name}" +# echo "service nginx reload" +} diff --git a/util/jenkins/demo-course-provisioner.sh b/util/jenkins/demo-course-provisioner.sh new file mode 100644 index 00000000000..0715f1f30d1 --- /dev/null +++ b/util/jenkins/demo-course-provisioner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -ex + +function write_demo_course_script() { + cat < "$1" +#!/usr/bin/env bash + +set -ex + +demo_hashed_password='pbkdf2_sha256\$20000\$TjE34FJjc3vv\$0B7GUmH8RwrOc/BvMoxjb5j8EgnWTt3sxorDANeF7Qw=' +admin_password='${admin_hashed_password}' + +# Clone demo course +mkdir /edx/var/edxapp/data +chmod 777 /edx/var/edxapp/data +git clone https://github.com/openedx/openedx-demo-course.git /edx/app/demo/edx-demo-course + +# import demo course +docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e CMS_CFG=/edx/etc/cms.yml -e DJANGO_SETTINGS_MODULE=cms.envs.docker-production -e SERVICE_VARIANT=cms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/cms.yml:/edx/etc/cms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/app/demo:/edx/app/demo -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py cms import /edx/var/edxapp/data /edx/app/demo/edx-demo-course + +# Create admin and demo users +docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms manage_user edx edx@example.com --initial-password-hash \$admin_password --superuser --staff +for user in honor audit verified staff ; do + email="\$user@example.com" + # Set staff flag for staff user + if [[ \$user == "staff" ]] ; then + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms manage_user \$user \$email --initial-password-hash \$demo_hashed_password --staff + else + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms manage_user \$user \$email --initial-password-hash \$demo_hashed_password + fi + # Enroll users in the demo course + docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms enroll_user_in_course -e \$email -c course-v1:edX+DemoX+Demo_Course +done + +# Seed forums for the demo course +docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms seed_permissions_roles course-v1:edX+DemoX+Demo_Course +EOF +} diff --git a/util/jenkins/worker-container-provisioner.sh b/util/jenkins/worker-container-provisioner.sh index 00271f7d8a6..dd38e67a4ad 100644 --- a/util/jenkins/worker-container-provisioner.sh +++ b/util/jenkins/worker-container-provisioner.sh @@ -13,8 +13,8 @@ function install_pre_reqs() { function render_docker_compose() { # Set common environment variables and volumes for edxapp celery workers if [ "${LC_WORKER_OF}" == "edxapp" ] ; then - worker_service_volume_mappings=("/edx/var/edxapp:/edx/var/edxapp" "/edx/app/edxapp/edx-platform:/edx/app/edxapp/edx-platform" "/edx/etc/lms.yml:/edx/etc/lms.yml" "/edx/etc/studio.yml:/edx/etc/studio.yml" "/edx/app/${LC_WORKER_OF}/.boto:/edx/app/${LC_WORKER_OF}/.boto" "/var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock" "/dev/log:/dev/log") - worker_service_env_mappings=("CONCURRENCY=1" "LOGLEVEL=info" "LANG=en_US.UTF-8" "PYTHONPATH=/edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO}" "BOTO_CONFIG=/edx/app/${LC_WORKER_OF}/.boto" "LMS_CFG=/edx/etc/lms.yml" "STUDIO_CFG=/edx/etc/studio.yml" "CMS_CFG=/edx/etc/studio.yml") + worker_service_volume_mappings=("/edx/var/edxapp:/edx/var/edxapp" "/edx/etc/lms.yml:/edx/etc/lms.yml" "/edx/etc/cms.yml:/edx/etc/cms.yml" "/edx/app/${LC_WORKER_OF}/.boto:/edx/app/${LC_WORKER_OF}/.boto" "/var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock") + worker_service_env_mappings=("CONCURRENCY=1" "LOGLEVEL=info" "LANG=en_US.UTF-8" "PYTHONPATH=/edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO}" "BOTO_CONFIG=/edx/app/${LC_WORKER_OF}/.boto" "LMS_CFG=/edx/etc/lms.yml" "STUDIO_CFG=/edx/etc/cms.yml" "CMS_CFG=/edx/etc/cms.yml") fi worker_celery_path="/edx/app/${LC_WORKER_OF}/venvs/${LC_WORKER_OF}/bin/celery" @@ -43,7 +43,8 @@ $( done echo -e " environment:" echo -e " - SERVICE_VARIANT=${worker_service_variant}" - echo -e " - DJANGO_SETTINGS_MODULE=${worker_service_variant}.envs.production" + echo -e " - DJANGO_SETTINGS_MODULE=${worker_service_variant}.envs.docker-production" + echo -e " - EDX_PLATFORM_SETTINGS=docker-production" echo -e " - EDX_REST_API_CLIENT_NAME=edx.${worker_service_variant}.core.${worker_queue}" for env_map in ${worker_service_env_mappings[@]} ; do echo -e " - ${env_map}" @@ -55,6 +56,16 @@ EOF install_pre_reqs +# checkout git repo +if [ ! -d "/edx/app/${LC_WORKER_OF}" ]; then + mkdir /edx/app/${LC_WORKER_OF} +fi + +if [ ! -d "/edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO}" ]; then + git clone https://github.com/edx/${LC_WORKER_SERVICE_REPO}.git /edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO} + cd /edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO} && git checkout ${LC_WORKER_SERVICE_REPO_VERSION} +fi + # Check if docker image already exists. If it doesn't, build it. if ! $(docker image inspect ${LC_WORKER_IMAGE_NAME}:latest >/dev/null 2>&1 && echo true || echo false) ; then cd /edx/app/${LC_WORKER_OF}/${LC_WORKER_SERVICE_REPO} From 0eae1fe0a4410d358d5a121d9ecd921058723274 Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Wed, 11 Jan 2023 13:43:20 -0500 Subject: [PATCH 391/664] fix: update edx-exams for new script --- playbooks/edx_continuous_integration.yml | 1 + playbooks/roles/edxapp/defaults/main.yml | 4 +++ playbooks/roles/edxlocal/defaults/main.yml | 6 ++++ .../app/nginx/sites-available/edx_exams.j2 | 30 +++++++++++++++++ .../oauth_client_setup/defaults/main.yml | 10 ++++++ util/jenkins/ansible-provision.sh | 33 ++++++++++++++----- util/jenkins/app-container-provisioner.sh | 24 ++++++++------ 7 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_exams.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 0928b05a9cf..0f4ca29cc7b 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -16,6 +16,7 @@ - learner_portal - program_console - prospectus + - edx_exams nginx_default_sites: - lms - docker-tools diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 9ecf1aa3d20..1dfe19c3f92 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1870,6 +1870,10 @@ SERVICE_WORKER_USERS: username: "{{ ENTERPRISE_CATALOG_SERVICE_USER_NAME }}" is_staff: true is_superuser: false + - email: "{{ EDX_EXAMS_SERVICE_USER_EMAIL }}" + username: "{{ EDX_EXAMS_SERVICE_USER_NAME }}" + is_staff: true + is_superuser: false - email: "{{ ECOMMERCE_SERVICE_USER_EMAIL }}" username: "{{ ECOMMERCE_SERVICE_USER_NAME }}" is_staff: true diff --git a/playbooks/roles/edxlocal/defaults/main.yml b/playbooks/roles/edxlocal/defaults/main.yml index afa5ff878eb..69ddc288b9e 100644 --- a/playbooks/roles/edxlocal/defaults/main.yml +++ b/playbooks/roles/edxlocal/defaults/main.yml @@ -20,6 +20,7 @@ edxlocal_databases: - "{{ LICENSE_MANAGER_DEFAULT_DB_NAME | default(None) }}" - "{{ ENTERPRISE_CATALOG_DEFAULT_DB_NAME | default(None) }}" - "{{ COMMERCE_COORDINATOR_DEFAULT_DB_NAME | default(None) }}" + - "{{ EDX_EXAMS_DEFAULT_DB_NAME | default(None) }}" edxlocal_database_users: - { @@ -97,3 +98,8 @@ edxlocal_database_users: user: "{{ COMMERCE_COORDINATOR_MYSQL_USER | default(None) }}", pass: "{{ COMMERCE_COORDINATOR_MYSQL_PASSWORD | default(None) }}" } + - { + db: "{{ EDX_EXAMS_DEFAULT_DB_NAME | default(None) }}", + user: "{{ EDX_EXAMS_MYSQL_USER | default(None) }}", + pass: "{{ EDX_EXAMS_MYSQL_PASSWORD | default(None) }}" + } diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_exams.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_exams.j2 new file mode 100644 index 00000000000..b754bcb7497 --- /dev/null +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_exams.j2 @@ -0,0 +1,30 @@ +server { + server_name ~^((stage|prod)-)?edx-exams.*; + listen 80; + rewrite ^ https://$host$request_uri? permanent; +} +server { + server_name ~^((stage|prod)-)?edx-exams.*; + listen 443 ssl; + ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; + ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; + + location / { + try_files $uri @proxy_to_app; + } + location ~ ^/(api)/ { + try_files $uri @proxy_to_app; + } + location @proxy_to_app { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://127.0.0.1:18740; + } + location ~ ^/static/(?P.*) { + root /edx/var/edx_exams; + try_files /staticfiles/$file =404; + } +} diff --git a/playbooks/roles/oauth_client_setup/defaults/main.yml b/playbooks/roles/oauth_client_setup/defaults/main.yml index 6823542f706..1d2fa3a3002 100644 --- a/playbooks/roles/oauth_client_setup/defaults/main.yml +++ b/playbooks/roles/oauth_client_setup/defaults/main.yml @@ -132,6 +132,16 @@ oauth_client_setup_oauth2_clients: logout_uri: "{{ COMMERCE_COORDINATOR_LOGOUT_URL | default('None') }}", username: "{{ COMMERCE_COORDINATOR_SERVICE_USER_NAME | default('None') }}", } + - { + name: "{{ edx_exams_service_name | default('None') }}", + url_root: "{{ EDX_EXAMS_URL_ROOT | default('None') }}", + sso_id: "{{ EDX_EXAMS_SOCIAL_AUTH_EDX_OAUTH2_KEY | default('None') }}", + sso_secret: "{{ EDX_EXAMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET | default('None') }}", + backend_service_id: "{{ EDX_EXAMS_BACKEND_SERVICE_EDX_OAUTH2_KEY | default('None') }}", + backend_service_secret: "{{ EDX_EXAMS_BACKEND_SERVICE_EDX_OAUTH2_SECRET | default('None') }}", + logout_uri: "{{ EDX_EXAMS_LOGOUT_URL | default('None') }}", + username: "{{ EDX_EXAMS_SERVICE_USER_NAME | default('None') }}", + } # # OS packages # diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 77539eef411..55e801976f2 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -388,6 +388,21 @@ COMMERCE_COORDINATOR_ENABLED: $commerce_coordinator COMMERCE_COORDINATOR_DECRYPT_CONFIG_ENABLED: true COMMERCE_COORDINATOR_COPY_CONFIG_ENABLED: true +EDX_EXAMS_NGINX_PORT: 80 +EDX_EXAMS_SSL_NGINX_PORT: 443 +EDX_EXAMS_DEFAULT_DB_NAME: 'edx_exams' +EDX_EXAMS_MYSQL_USER: 'edx_exams001' +EDX_EXAMS_MYSQL_PASSWORD: 'password' +edx_exams_service_name: 'edx_exams' +EDX_EXAMS_URL_ROOT: https://edx-exams-${deploy_host} +EDX_EXAMS_SOCIAL_AUTH_EDX_OAUTH2_KEY: 'edx_exams-sso-key' +EDX_EXAMS_SOCIAL_AUTH_EDX_OAUTH2_SECRET: 'edx_exams-sso-secret' +EDX_EXAMS_BACKEND_SERVICE_EDX_OAUTH2_KEY: 'edx_exams-backend-service-key' +EDX_EXAMS_BACKEND_SERVICE_EDX_OAUTH2_SECRET: 'edx_exams-backend-service-secret' +EDX_EXAMS_LOGOUT_URL: '{{ EDX_EXAMS_URL_ROOT }}/logout/' +EDX_EXAMS_SERVICE_USER_EMAIL: 'edx_exams_worker@example.com' +EDX_EXAMS_SERVICE_USER_NAME: 'edx_exams_worker' + ENTERPRISE_CATALOG_NGINX_PORT: 80 ENTERPRISE_CATALOG_SSL_NGINX_PORT: 443 ENTERPRISE_CATALOG_VERSION: $enterprise_catalog_version @@ -426,6 +441,8 @@ ORA_GRADING_SANDBOX_BUILD: True mysql_server_version_5_7: True +edxapp_container_enabled: $edxapp_container_enabled + # User provided extra vars $extra_vars EOF @@ -855,21 +872,21 @@ if [[ $edx_exams == 'true' ]]; then app_hostname="edx-exams" app_service_name="edx_exams" + app_name="edx-exams" app_repo="edx-exams" app_version=$edx_exams_version app_gunicorn_port=18740 app_cfg=EDX_EXAMS_CFG - provision_script="/var/tmp/provision-script-$$.sh" -cat << EOF > $provision_script -$(provision_containerized_app) -EOF + app_provision_script="/var/tmp/app-container-provision-script-$$.sh" - # copy app config file and run script to deploy app - ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${WORKSPACE}/configuration-internal/k8s-sandbox-config/${app_service_name}.yml dest=/var/tmp/${app_service_name}.yml" -u ubuntu -b - ansible -c ssh -i "${deploy_host}," $deploy_host -m script -a "${provision_script}" -u ubuntu -b + write_app_deployment_script $app_provision_script + set -x - rm -f "${provision_script}" + sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/configuration-internal/k8s-sandbox-config/$app_service_name.yml + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${WORKSPACE}/configuration-internal/k8s-sandbox-config/${app_service_name}.yml dest=/var/tmp/${app_service_name}.yml" -u ubuntu -b + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${deploy_host} "sudo -n -s bash" < $app_provision_script + rm -f "${app_provision_script}" fi rm -f "$extra_vars_file" diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh index 8b4f62e3aa1..6eccf33260c 100644 --- a/util/jenkins/app-container-provisioner.sh +++ b/util/jenkins/app-container-provisioner.sh @@ -19,7 +19,7 @@ fi if [[ ${app_service_name} != 'cms' && ${app_service_name} != 'lms' ]] ; then # Create app staticfiles dir - mkdir /edx/var/${app_name}/staticfiles/ -p && chmod 777 /edx/var/${app_name} -R + mkdir /edx/var/${app_service_name}/staticfiles/ -p && chmod 777 /edx/var/${app_service_name} -R fi # if application is lms, download and setup themes @@ -77,14 +77,14 @@ if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production -e SERVICE_VARIANT=${app_service_name} -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py ${app_service_name} migrate --database student_module_history --noinput else # Run app migrations - docker run --network=host --rm -u='www-data' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py migrate + docker run --network=host --rm -u='www-data' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py migrate # Generate static assets - docker run --network=host --rm -u='root' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_name}:/edx/var/${app_name} -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py collectstatic --noinput + docker run --network=host --rm -u='root' -e ${app_cfg}=/edx/etc/${app_service_name}.yml -e DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production -v /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml -v /edx/var/${app_service_name}/staticfiles/:/var/tmp/ -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py collectstatic --noinput fi # Setup oauth clients for service other than CMS as part of the LMS setup if [[ ${app_service_name} == 'lms' ]]; then - service_worker_users=(enterprise veda discovery credentials insights registrar designer license_manager commerce_coordinator enterprise_catalog ecommerce retirement) + service_worker_users=(enterprise veda discovery credentials insights registrar designer license_manager commerce_coordinator enterprise_catalog ecommerce retirement edx_exams) # Provision IDA User in LMS for service_worker in "\${service_worker_users[@]}"; do app_hostname=\${service_worker/_/-} @@ -97,7 +97,7 @@ if [[ ${app_service_name} == 'lms' ]]; then fi # oauth client setup -if [[ ${app_service_name} != 'lms' ]]; then +if [[ ${app_service_name} != 'lms' && ${edxapp_container_enabled} == 'true' ]]; then # Provision IDA User in LMS docker run --network=host --rm -u='www-data' -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock edx-platform:latest python3 manage.py lms manage_user $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker $(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}_; fi)${app_service_name}_worker@example.com --staff --superuser @@ -122,30 +122,34 @@ services: stdin_open: true tty: true container_name: ${app_service_name} - command: bash -c "gunicorn --workers=2 --name ${app_service_name} -c /edx/app/${app_name}/${app_repo}/${app_service_name}/docker_${app_service_name}_gunicorn.py --log-file - --max-requests=1000 ${app_service_name}.wsgi:application" + command: bash -c "gunicorn --workers=2 --name ${app_service_name} -c /edx/app/$(if [[ ${app_name} == 'edxapp' ]]; then echo ${app_name}/; fi)${app_repo}/${app_service_name}/$(if [[ ${app_name} == 'edxapp' ]]; then echo docker_${app_service_name}_gunicorn.py; else echo docker_gunicorn_configuration.py; fi) --log-file - --max-requests=1000 ${app_service_name}.wsgi:application" user: "www-data:www-data" network_mode: 'host' restart: on-failure environment: - - ${app_cfg}=/edx/etc/${app_service_name}.yml - EDX_REST_API_CLIENT_NAME=sandbox-edx-${app_service_name} $( if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then echo -e " - DJANGO_SETTINGS_MODULE=${app_service_name}.envs.docker-production" echo -e " - EDX_PLATFORM_SETTINGS=docker-production" echo -e " - SERVICE_VARIANT=${app_service_name}" + echo -e " - ${app_cfg}=/edx/etc/${app_service_name}.yml" else - echo -e " - DJANGO_SETTINGS_MODULE=${app_service_name}.envs.production" + echo -e " - DJANGO_SETTINGS_MODULE=${app_service_name}.settings.production" + echo -e " - ${app_cfg}=/${app_service_name}.yml" fi ) volumes: - - /edx/var/${app_name}:/edx/var/${app_name} - - /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml - /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock $( if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then + echo -e " - /edx/var/${app_name}:/edx/var/${app_name}" echo -e " - /edx/var/edx-themes:/edx/var/edx-themes" echo -e " - /var/tmp/tracking_logs.log:/var/tmp/tracking_logs.log" + echo -e " - /edx/etc/${app_service_name}.yml:/edx/etc/${app_service_name}.yml" + else + echo -e " - /edx/var/${app_service_name}/staticfiles/:/var/tmp/" + echo -e " - /edx/etc/${app_service_name}.yml:/${app_service_name}.yml" fi ) $( From 523f0cafe10c638ffa5d13cd71612a496014c601 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 23 Feb 2023 15:34:46 -0500 Subject: [PATCH 392/664] build: Creating a missing workflow file `self-assign-issue.yml`. The .github/workflows/self-assign-issue.yml workflow is missing or needs an update to stay in sync with the current standard for this workflow as defined in the `.github` repo of the `openedx` GitHub org. --- .github/workflows/self-assign-issue.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/self-assign-issue.yml diff --git a/.github/workflows/self-assign-issue.yml b/.github/workflows/self-assign-issue.yml new file mode 100644 index 00000000000..37522fd57b1 --- /dev/null +++ b/.github/workflows/self-assign-issue.yml @@ -0,0 +1,12 @@ +# This workflow runs when a comment is made on the ticket +# If the comment starts with "assign me" it assigns the author to the +# ticket (case insensitive) + +name: Assign comment author to ticket if they say "assign me" +on: + issue_comment: + types: [created] + +jobs: + self_assign_by_comment: + uses: openedx/.github/.github/workflows/self-assign-issue.yml@master From 1e8263b06338fa25f68f1818751ebb105cc66567 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 23 Feb 2023 15:34:46 -0500 Subject: [PATCH 393/664] build: Creating a missing workflow file `add-depr-ticket-to-depr-board.yml`. The .github/workflows/add-depr-ticket-to-depr-board.yml workflow is missing or needs an update to stay in sync with the current standard for this workflow as defined in the `.github` repo of the `openedx` GitHub org. --- .../add-depr-ticket-to-depr-board.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/add-depr-ticket-to-depr-board.yml diff --git a/.github/workflows/add-depr-ticket-to-depr-board.yml b/.github/workflows/add-depr-ticket-to-depr-board.yml new file mode 100644 index 00000000000..250e394abc1 --- /dev/null +++ b/.github/workflows/add-depr-ticket-to-depr-board.yml @@ -0,0 +1,19 @@ +# Run the workflow that adds new tickets that are either: +# - labelled "DEPR" +# - title starts with "[DEPR]" +# - body starts with "Proposal Date" (this is the first template field) +# to the org-wide DEPR project board + +name: Add newly created DEPR issues to the DEPR project board + +on: + issues: + types: [opened] + +jobs: + routeissue: + uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master + secrets: + GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} + GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} From a741ac3596652a98b391e35bb416170e4b5dcab2 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 23 Feb 2023 15:34:47 -0500 Subject: [PATCH 394/664] build: Creating a missing workflow file `add-remove-label-on-comment.yml`. The .github/workflows/add-remove-label-on-comment.yml workflow is missing or needs an update to stay in sync with the current standard for this workflow as defined in the `.github` repo of the `openedx` GitHub org. --- .../workflows/add-remove-label-on-comment.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/add-remove-label-on-comment.yml diff --git a/.github/workflows/add-remove-label-on-comment.yml b/.github/workflows/add-remove-label-on-comment.yml new file mode 100644 index 00000000000..0f369db7d29 --- /dev/null +++ b/.github/workflows/add-remove-label-on-comment.yml @@ -0,0 +1,20 @@ +# This workflow runs when a comment is made on the ticket +# If the comment starts with "label: " it tries to apply +# the label indicated in rest of comment. +# If the comment starts with "remove label: ", it tries +# to remove the indicated label. +# Note: Labels are allowed to have spaces and this script does +# not parse spaces (as often a space is legitimate), so the command +# "label: really long lots of words label" will apply the +# label "really long lots of words label" + +name: Allows for the adding and removing of labels via comment + +on: + issue_comment: + types: [created] + +jobs: + add_remove_labels: + uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master + From 0d6f6dc5472eb2da50f0989385f1cb85cc9c79df Mon Sep 17 00:00:00 2001 From: Phillip Shiu Date: Thu, 23 Feb 2023 17:41:13 -0500 Subject: [PATCH 395/664] fix: add PAYMENT_STRIPE_PUBLISHABLE_KEY & PAYMENT_STRIPE_RESPONSE_URL to Payment MFE default vars (#6882) --- CHANGELOG.md | 5 +++++ playbooks/roles/payment/defaults/main.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68ece1587a1..c80887fd3d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-02-23 + - Role: payment + - Add `PAYMENT_STRIPE_PUBLISHABLE_KEY` & `PAYMENT_STRIPE_RESPONSE_URL` + settings for operators running Payment MFE with Stripe. + - 2023-01-18 - Updated sandbox deployment scripts to setup edx-platform in docker containers. - Added Fluentd for checking tracking logs when running edx-platform in docker containers. diff --git a/playbooks/roles/payment/defaults/main.yml b/playbooks/roles/payment/defaults/main.yml index 779d2398f8d..70512b3827f 100644 --- a/playbooks/roles/payment/defaults/main.yml +++ b/playbooks/roles/payment/defaults/main.yml @@ -1,2 +1,4 @@ payment_env_extra: APPLE_PAY_COUNTRY_CODE: "{{ PAYMENT_APPLE_PAY_COUNTRY_CODE }}" + STRIPE_PUBLISHABLE_KEY: "{{ PAYMENT_STRIPE_PUBLISHABLE_KEY }}" + STRIPE_RESPONSE_URL: "{{ PAYMENT_STRIPE_RESPONSE_URL }}" From 7610d2841212ead67884fb0768df96f5d2e727cb Mon Sep 17 00:00:00 2001 From: Piotr Surowiec Date: Thu, 2 Mar 2023 01:35:45 +0100 Subject: [PATCH 396/664] fix: enable Celery workers in supervisor for edx_django_service role (#6884) Previously, these configurations were added only to `supervisor_available_dir`. --- CHANGELOG.md | 5 +++++ playbooks/roles/edx_django_service/tasks/main.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c80887fd3d2..52744394921 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2023-02-27 + - Role: edx_django_service + - Enable Celery workers in supervisor when + `edx_django_service_enable_celery_workers` is true. + - 2023-02-23 - Role: payment - Add `PAYMENT_STRIPE_PUBLISHABLE_KEY` & `PAYMENT_STRIPE_RESPONSE_URL` diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index 62616bbf9ae..c471cc0f97d 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -347,6 +347,17 @@ - install - install:configuration +- name: enable celery worker supervisor script + when: edx_django_service_enable_celery_workers + file: + src: "{{ supervisor_available_dir }}/{{ edx_django_service_workers_supervisor_conf }}" + dest: "{{ supervisor_cfg_dir }}/{{ edx_django_service_workers_supervisor_conf }}" + state: link + force: yes + tags: + - install + - install:configuration + - name: update supervisor configuration command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update" when: not disable_edx_services From 4346f92b7349903ecdee5ded3f6b2f5b16ef322b Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Fri, 3 Mar 2023 12:08:45 -0500 Subject: [PATCH 397/664] feat: add subscriptions to sandbox (#6885) * feat: add subscriptions to sandbox * fix: add newline at end of file --- playbooks/edx_continuous_integration.yml | 1 + playbooks/roles/edxlocal/defaults/main.yml | 6 ++++ .../nginx/sites-available/subscriptions.j2 | 30 +++++++++++++++++++ util/jenkins/ansible-provision.sh | 25 ++++++++++++++++ util/jenkins/app-container-provisioner.sh | 2 +- 5 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 playbooks/roles/nginx/templates/edx/app/nginx/sites-available/subscriptions.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 0f4ca29cc7b..4d6b532d7d7 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -17,6 +17,7 @@ - program_console - prospectus - edx_exams + - subscriptions nginx_default_sites: - lms - docker-tools diff --git a/playbooks/roles/edxlocal/defaults/main.yml b/playbooks/roles/edxlocal/defaults/main.yml index 69ddc288b9e..8a2d2787ac8 100644 --- a/playbooks/roles/edxlocal/defaults/main.yml +++ b/playbooks/roles/edxlocal/defaults/main.yml @@ -21,6 +21,7 @@ edxlocal_databases: - "{{ ENTERPRISE_CATALOG_DEFAULT_DB_NAME | default(None) }}" - "{{ COMMERCE_COORDINATOR_DEFAULT_DB_NAME | default(None) }}" - "{{ EDX_EXAMS_DEFAULT_DB_NAME | default(None) }}" + - "{{ SUBSCRIPTIONS_DEFAULT_DB_NAME | default(None) }}" edxlocal_database_users: - { @@ -102,4 +103,9 @@ edxlocal_database_users: db: "{{ EDX_EXAMS_DEFAULT_DB_NAME | default(None) }}", user: "{{ EDX_EXAMS_MYSQL_USER | default(None) }}", pass: "{{ EDX_EXAMS_MYSQL_PASSWORD | default(None) }}" + } + - { + db: "{{ SUBSCRIPTIONS_DEFAULT_DB_NAME | default(None) }}", + user: "{{ SUBSCRIPTIONS_MYSQL_USER | default(None) }}", + pass: "{{ SUBSCRIPTIONS_MYSQL_PASSWORD | default(None) }}" } diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/subscriptions.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/subscriptions.j2 new file mode 100644 index 00000000000..35841adbe51 --- /dev/null +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/subscriptions.j2 @@ -0,0 +1,30 @@ +server { + server_name ~^((stage|prod)-)?subscriptions.*; + listen 80; + rewrite ^ https://$host$request_uri? permanent; +} +server { + server_name ~^((stage|prod)-)?subscriptions.*; + listen 443 ssl; + ssl_certificate /etc/ssl/certs/wildcard.sandbox.edx.org.pem; + ssl_certificate_key /etc/ssl/private/wildcard.sandbox.edx.org.key; + + location / { + try_files $uri @proxy_to_app; + } + location ~ ^/(api)/ { + try_files $uri @proxy_to_app; + } + location @proxy_to_app { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://127.0.0.1:18750; + } + location ~ ^/static/(?P.*) { + root /edx/var/subscriptions; + try_files /staticfiles/$file =404; + } +} \ No newline at end of file diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 55e801976f2..e2401fad766 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -403,6 +403,10 @@ EDX_EXAMS_LOGOUT_URL: '{{ EDX_EXAMS_URL_ROOT }}/logout/' EDX_EXAMS_SERVICE_USER_EMAIL: 'edx_exams_worker@example.com' EDX_EXAMS_SERVICE_USER_NAME: 'edx_exams_worker' +SUBSCRIPTIONS_DEFAULT_DB_NAME: 'subscriptions' +SUBSCRIPTIONS_MYSQL_USER: 'subscriptions001' +SUBSCRIPTIONS_MYSQL_PASSWORD: 'password' + ENTERPRISE_CATALOG_NGINX_PORT: 80 ENTERPRISE_CATALOG_SSL_NGINX_PORT: 443 ENTERPRISE_CATALOG_VERSION: $enterprise_catalog_version @@ -889,5 +893,26 @@ if [[ $edx_exams == 'true' ]]; then rm -f "${app_provision_script}" fi +if [[ $subscriptions == 'true' ]]; then + + app_hostname="subscriptions" + app_service_name="subscriptions" + app_name="subscriptions" + app_repo="subscriptions" + app_version=$subscriptions_version + app_gunicorn_port=18750 + app_cfg=SUBSCRIPTIONS_CFG + + app_provision_script="/var/tmp/app-container-provision-script-$$.sh" + + write_app_deployment_script $app_provision_script + set -x + + sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/configuration-internal/k8s-sandbox-config/$app_service_name.yml + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${WORKSPACE}/configuration-internal/k8s-sandbox-config/${app_service_name}.yml dest=/var/tmp/${app_service_name}.yml" -u ubuntu -b + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${deploy_host} "sudo -n -s bash" < $app_provision_script + rm -f "${app_provision_script}" +fi + rm -f "$extra_vars_file" rm -f ${extra_vars_file}_clean diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh index 6eccf33260c..300a14127dd 100644 --- a/util/jenkins/app-container-provisioner.sh +++ b/util/jenkins/app-container-provisioner.sh @@ -84,7 +84,7 @@ fi # Setup oauth clients for service other than CMS as part of the LMS setup if [[ ${app_service_name} == 'lms' ]]; then - service_worker_users=(enterprise veda discovery credentials insights registrar designer license_manager commerce_coordinator enterprise_catalog ecommerce retirement edx_exams) + service_worker_users=(enterprise veda discovery credentials insights registrar designer license_manager commerce_coordinator enterprise_catalog ecommerce retirement edx_exams subscriptions) # Provision IDA User in LMS for service_worker in "\${service_worker_users[@]}"; do app_hostname=\${service_worker/_/-} From 98461687512fdd81ba94234b531e9fe0e44212b2 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Wed, 8 Mar 2023 08:54:02 -0500 Subject: [PATCH 398/664] feat: allow provisioning of private repos in sandboxes (#6888) * feat: allow provisioning of private repos in sandboxes --- util/jenkins/ansible-provision.sh | 1 + util/jenkins/app-container-provisioner.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index e2401fad766..b3c76b550e6 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -902,6 +902,7 @@ if [[ $subscriptions == 'true' ]]; then app_version=$subscriptions_version app_gunicorn_port=18750 app_cfg=SUBSCRIPTIONS_CFG + app_repo_is_private=true app_provision_script="/var/tmp/app-container-provision-script-$$.sh" diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh index 300a14127dd..de65a947547 100644 --- a/util/jenkins/app-container-provisioner.sh +++ b/util/jenkins/app-container-provisioner.sh @@ -42,7 +42,13 @@ if [ ! -d "/edx/app/${app_name}" ]; then fi if [[ ! -d "/edx/app/${app_name}/${app_repo}" ]] ; then - git clone https://github.com/edx/${app_repo}.git /edx/app/${app_name}/${app_repo} + + # use SSH to clone if repo is private + if [[ "$app_repo_is_private" = true ]] ; then + git clone git@github.com:edx/${app_repo}.git /edx/app/${app_name}/${app_repo} + else + git clone https://github.com/edx/${app_repo}.git /edx/app/${app_name}/${app_repo} + fi cd /edx/app/${app_name}/${app_repo} && git checkout ${app_version} fi From bc039de0a7b447d9c75316eceff966e6f2c77511 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Wed, 8 Mar 2023 16:09:02 -0500 Subject: [PATCH 399/664] fix: "Host key verification failed." error for sandbox build with private repo (#6890) --- util/jenkins/app-container-provisioner.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh index de65a947547..03d19f0ed52 100644 --- a/util/jenkins/app-container-provisioner.sh +++ b/util/jenkins/app-container-provisioner.sh @@ -45,7 +45,12 @@ if [[ ! -d "/edx/app/${app_name}/${app_repo}" ]] ; then # use SSH to clone if repo is private if [[ "$app_repo_is_private" = true ]] ; then - git clone git@github.com:edx/${app_repo}.git /edx/app/${app_name}/${app_repo} + set +x + echo -e "${app_git_ssh_key}" > /tmp/${app_service_name}_ssh_key + set -x + chmod 0600 /tmp/${app_service_name}_ssh_key + useradd -m -d /edx/var/${app_service_name} ${app_service_name} -G www-data + GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /tmp/${app_service_name}_ssh_key" git clone git@github.com:edx/${app_repo}.git /edx/app/${app_name}/${app_repo} else git clone https://github.com/edx/${app_repo}.git /edx/app/${app_name}/${app_repo} fi From f443d992f0f827cb888904cd1a9baa7ebec9c26e Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 9 Mar 2023 13:18:07 +0500 Subject: [PATCH 400/664] feat: add request tracing in nginx logs using X-Request-ID header --- .../edx/app/nginx/sites-available/app.j2 | 2 ++ .../sites-available/concerns/proxy-to-app.j2 | 2 ++ .../sites-available/concerns/x-request-id-map.j2 | 14 ++++++++++++++ .../edx/app/nginx/sites-available/cms.j2 | 15 +++++++++++++++ .../edx/app/nginx/sites-available/cms_proxy.j2 | 2 ++ .../edx/app/nginx/sites-available/lms.j2 | 14 ++++++++++++++ .../edx/app/nginx/sites-available/lms_proxy.j2 | 2 ++ .../roles/nginx/templates/etc/nginx/nginx.conf.j2 | 2 +- 8 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 index d051e6adc62..af3604e68e9 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 @@ -4,6 +4,7 @@ {% include "concerns/upstream.j2"%} {% include "concerns/cors-build-map.j2" %} +{% include "concerns/x-request-id-map.j2" %} server { server_name {{ edx_django_service_hostname }}; @@ -28,6 +29,7 @@ server { ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; + add_header X-Request-ID $uuid; {% include "concerns/app-common.j2" %} } diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 index 2271dbbf082..5f14a256e5f 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 @@ -22,10 +22,12 @@ location @proxy_to_app { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 new file mode 100644 index 00000000000..45e0fd894b8 --- /dev/null +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 @@ -0,0 +1,14 @@ + + +# Maps to trace requests and identifying them across different components. First map creates a new variable named +# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is +# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending +# on whether trace_id is set or not in first map. +map $http_x_request_id $trace_id { + "" "${http_cf_ray}"; + default "${http_x_request_id}"; +} +map $trace_id $uuid { + "" "${request_id}"; + default $trace_id; +} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 index c0c94614a45..aa94e0f35c2 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 @@ -25,6 +25,18 @@ upstream cms-backend { } {% endif %} +# Maps to trace requests and identifying them across different components. First map creates a new variable named +# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is +# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending +# on whether trace_id is set or not in first map. +map $http_x_request_id $trace_id { + "" "${http_cf_ray}"; + default "${http_x_request_id}"; +} +map $trace_id $uuid { + "" "${request_id}"; + default $trace_id; +} server { # CMS configuration file for nginx, templated by ansible @@ -80,6 +92,9 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; + # To track requests + add_header X-Request-ID $uuid; + {% include "handle-tls-redirect-and-ip-disclosure.j2" %} server_name {{ CMS_HOSTNAME }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 index c9cd09ca160..22e41e9ff2d 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 @@ -2,10 +2,12 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 index b55931a7f99..bff3db43900 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 @@ -59,6 +59,18 @@ geo $http_x_forwarded_for $embargo { } {% endif %} +# Maps to trace requests and identifying them across different components. First map creates a new variable named +# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is +# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending +# on whether trace_id is set or not in first map. +map $http_x_request_id $trace_id { + "" "${http_cf_ray}"; + default "${http_x_request_id}"; +} +map $trace_id $uuid { + "" "${request_id}"; + default $trace_id; +} server { # LMS configuration file for nginx, templated by ansible @@ -112,6 +124,8 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; + # To track requests + add_header X-Request-ID $uuid; {% include "handle-tls-redirect-and-ip-disclosure.j2" %} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 index fc2dfd4d7d0..38c25828544 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 @@ -2,10 +2,12 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index ccec62f53a7..cef245289dd 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -39,7 +39,7 @@ http { # Logging Settings ## - log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user $http_x_forwarded_proto [$time_local] ' + log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user $http_x_forwarded_proto [$time_local] $uuid ' '"$request" $status $body_bytes_sent $request_time ' '"$http_referer" "$http_user_agent"'; From 27ae481d039612eb91ed7c47ec729248bddeda3e Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Thu, 9 Mar 2023 15:15:28 +0500 Subject: [PATCH 401/664] Revert "feat: add request tracing in nginx logs using X-Request-ID header" --- .../edx/app/nginx/sites-available/app.j2 | 2 -- .../sites-available/concerns/proxy-to-app.j2 | 2 -- .../sites-available/concerns/x-request-id-map.j2 | 14 -------------- .../edx/app/nginx/sites-available/cms.j2 | 15 --------------- .../edx/app/nginx/sites-available/cms_proxy.j2 | 2 -- .../edx/app/nginx/sites-available/lms.j2 | 14 -------------- .../edx/app/nginx/sites-available/lms_proxy.j2 | 2 -- .../roles/nginx/templates/etc/nginx/nginx.conf.j2 | 2 +- 8 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 index af3604e68e9..d051e6adc62 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 @@ -4,7 +4,6 @@ {% include "concerns/upstream.j2"%} {% include "concerns/cors-build-map.j2" %} -{% include "concerns/x-request-id-map.j2" %} server { server_name {{ edx_django_service_hostname }}; @@ -29,7 +28,6 @@ server { ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; - add_header X-Request-ID $uuid; {% include "concerns/app-common.j2" %} } diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 index 5f14a256e5f..2271dbbf082 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 @@ -22,12 +22,10 @@ location @proxy_to_app { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; - proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 deleted file mode 100644 index 45e0fd894b8..00000000000 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/x-request-id-map.j2 +++ /dev/null @@ -1,14 +0,0 @@ - - -# Maps to trace requests and identifying them across different components. First map creates a new variable named -# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is -# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending -# on whether trace_id is set or not in first map. -map $http_x_request_id $trace_id { - "" "${http_cf_ray}"; - default "${http_x_request_id}"; -} -map $trace_id $uuid { - "" "${request_id}"; - default $trace_id; -} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 index aa94e0f35c2..c0c94614a45 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 @@ -25,18 +25,6 @@ upstream cms-backend { } {% endif %} -# Maps to trace requests and identifying them across different components. First map creates a new variable named -# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is -# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending -# on whether trace_id is set or not in first map. -map $http_x_request_id $trace_id { - "" "${http_cf_ray}"; - default "${http_x_request_id}"; -} -map $trace_id $uuid { - "" "${request_id}"; - default $trace_id; -} server { # CMS configuration file for nginx, templated by ansible @@ -92,9 +80,6 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; - # To track requests - add_header X-Request-ID $uuid; - {% include "handle-tls-redirect-and-ip-disclosure.j2" %} server_name {{ CMS_HOSTNAME }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 index 22e41e9ff2d..c9cd09ca160 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 @@ -2,12 +2,10 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; - proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 index bff3db43900..b55931a7f99 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 @@ -59,18 +59,6 @@ geo $http_x_forwarded_for $embargo { } {% endif %} -# Maps to trace requests and identifying them across different components. First map creates a new variable named -# trace_id, which is set to either the value of X-REQUEST-ID or Cloudflare CF-ray headers depending on whether http_x_request_id is -# empty or not. Second map creates a new variable named uuid, which is set to either the value of trace_id or request_id, depending -# on whether trace_id is set or not in first map. -map $http_x_request_id $trace_id { - "" "${http_cf_ray}"; - default "${http_x_request_id}"; -} -map $trace_id $uuid { - "" "${request_id}"; - default $trace_id; -} server { # LMS configuration file for nginx, templated by ansible @@ -124,8 +112,6 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; - # To track requests - add_header X-Request-ID $uuid; {% include "handle-tls-redirect-and-ip-disclosure.j2" %} diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 index 38c25828544..fc2dfd4d7d0 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 @@ -2,12 +2,10 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Request-ID $uuid; {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; - proxy_set_header X-Request-ID $uuid; {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index cef245289dd..ccec62f53a7 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -39,7 +39,7 @@ http { # Logging Settings ## - log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user $http_x_forwarded_proto [$time_local] $uuid ' + log_format p_combined '$http_x_forwarded_for - $remote_addr - $remote_user $http_x_forwarded_proto [$time_local] ' '"$request" $status $body_bytes_sent $request_time ' '"$http_referer" "$http_user_agent"'; From c85ec2afcf1de8e3f293fd062392cc9bc62c2b76 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 9 Mar 2023 09:44:00 -0500 Subject: [PATCH 402/664] docs: Remove repo specific CONTRIBUTING.md (#6887) We now have a org wide CONTRIBUTING.md that points to our correct general contributing guidelines. We don't need repo specific ones that forward to other contributing docs. --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 10681d7979b..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1 +0,0 @@ -Do not use GitHub issues for Open edX support. The mailing list and Slack channels are explained here: http://open.edx.org/getting-help. If it turns out there's a bug in the configuration scripts, we can open an issue or PR here. From c2c8d3122c1efd1dfc0f0a6fdada1740838696d3 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 10 Mar 2023 15:15:28 +0500 Subject: [PATCH 403/664] feat: add request tracing in nginx logs using X-Request-ID header --- .../edx/app/nginx/sites-available/app.j2 | 4 ++++ .../sites-available/concerns/proxy-to-app.j2 | 6 +++++ playbooks/roles/nginx/defaults/main.yml | 1 + .../edx/app/nginx/sites-available/cms.j2 | 5 +++++ .../app/nginx/sites-available/cms_proxy.j2 | 6 +++++ .../edx/app/nginx/sites-available/lms.j2 | 6 ++++- .../app/nginx/sites-available/lms_proxy.j2 | 6 +++++ .../nginx/templates/etc/nginx/nginx.conf.j2 | 22 ++++++++++++++++++- 8 files changed, 54 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 index d051e6adc62..7dc6897a949 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 @@ -28,6 +28,10 @@ server { ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + # To Track requests + add_header X-Request-ID $request_tracking_id; + {% endif %} {% include "concerns/app-common.j2" %} } diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 index 2271dbbf082..73b38a5967a 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/proxy-to-app.j2 @@ -22,10 +22,16 @@ location @proxy_to_app { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index c43f56031c8..4b5be5f65af 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -67,6 +67,7 @@ NGINX_SSL_PROTOCOLS: "TLSv1.1 TLSv1.2" NGINX_DH_PARAMS_PATH: "/etc/ssl/private/dhparams.pem" NGINX_DH_KEYSIZE: 2048 +NGINX_ENABLE_REQUEST_TRACKING_ID: False # This can be one of 'p_combined' or 'ssl_combined' by default. If you # wish to specify your own format then define it in a configuration file # located under `nginx_conf_dir` and then use the format name specified diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 index c0c94614a45..c6616c38800 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2 @@ -80,6 +80,11 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + # To track requests + add_header X-Request-ID $request_tracking_id; + {% endif %} + {% include "handle-tls-redirect-and-ip-disclosure.j2" %} server_name {{ CMS_HOSTNAME }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 index c9cd09ca160..664f9f5ee15 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms_proxy.j2 @@ -2,10 +2,16 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 index b55931a7f99..2dbc86f3212 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2 @@ -112,7 +112,11 @@ error_page {{ k }} {{ v }}; # Prevent invalid display courseware in IE 10+ with high privacy settings add_header P3P '{{ NGINX_P3P_MESSAGE }}'; - + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + # To track requests + add_header X-Request-ID $request_tracking_id; + {% endif %} + {% include "handle-tls-redirect-and-ip-disclosure.j2" %} access_log {{ nginx_log_dir }}/access.log {{ NGINX_LOG_FORMAT_NAME }}; diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 index fc2dfd4d7d0..4c6c75676f8 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms_proxy.j2 @@ -2,10 +2,16 @@ proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Port $server_port; proxy_set_header X-Forwarded-For $remote_addr; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% else %} proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; proxy_set_header X-Forwarded-Port $http_x_forwarded_port; proxy_set_header X-Forwarded-For $http_x_forwarded_for; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + proxy_set_header X-Request-ID $request_tracking_id; + {% endif %} {% endif %} # newrelic-specific header records the time when nginx handles a request. diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index ccec62f53a7..1c3494e5434 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -35,6 +35,20 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + # Setting variables based on HTTP headers to track and differentiate requests across components: the first creates a new + # variable called trace_id based on X-REQUEST-ID or Cloudflare CF-ray headers, while the second creates a variable named + # request_tracking_id based on whether trace_id is set or not. + map $http_x_request_id $trace_id { + "" "${http_cf_ray}"; + default "${http_x_request_id}"; + } + map $trace_id $request_tracking_id { + "" "${request_id}"; + default $trace_id; + } + {% endif %} + ## # Logging Settings ## @@ -52,7 +66,13 @@ http { 'referer=$http_referer user_agent="$http_user_agent" upstream_addr=$upstream_addr upstream_status=$upstream_status ' 'request_time=$request_time request_id=$request_id upstream_response_time=$upstream_response_time ' 'upstream_connect_time=$upstream_connect_time upstream_header_time=$upstream_header_time'; - + + {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} + log_format cf_custom '$http_x_forwarded_for - $remote_addr - $remote_user $http_x_forwarded_proto [$time_local] $request_tracking_id ' + '"$request" $status $body_bytes_sent $request_time ' + '"$http_referer" "$http_user_agent"'; + {% endif %} + log_format json_analytics escape=json '{' '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution '"connection": "$connection", ' # connection serial number From 059289c45ec8ef90a014a6d0625732f5de683870 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 13 Mar 2023 14:53:33 -0400 Subject: [PATCH 404/664] fix: pull in ssh key in all code paths where it may be necessary JIRA:PSRE-2282 --- util/jenkins/ansible-provision.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index b3c76b550e6..a3f3cda9d3a 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -873,6 +873,8 @@ if [[ $enable_newrelic == "true" ]]; then fi if [[ $edx_exams == 'true' ]]; then + set +x + app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" app_hostname="edx-exams" app_service_name="edx_exams" @@ -894,6 +896,8 @@ if [[ $edx_exams == 'true' ]]; then fi if [[ $subscriptions == 'true' ]]; then + set +x + app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" app_hostname="subscriptions" app_service_name="subscriptions" From df545132d0fd22ef1697539e86f1032d4e00632f Mon Sep 17 00:00:00 2001 From: Bianca Severino Date: Mon, 13 Mar 2023 19:23:22 -0400 Subject: [PATCH 405/664] fix: temporarily redirect chatgpt course to search --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index a5d49f31115..c495df7386c 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -157,5 +157,8 @@ server { } {% endfor %} + + # [WS-3807] Temporary redirect + rewrite ^/course/introduction-to-chatgpt$ "/search?skills.skill=Artificial+Intelligence" redirect; } From e60af79a38d2fd4921dda1f4054076cc9974ea42 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Wed, 15 Mar 2023 15:25:16 -0400 Subject: [PATCH 406/664] fix: add subscriptions to playbook to add dns name for sandbox (#6896) --- playbooks/roles/launch_ec2/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index 63c0d3d2814..09bafc6e3ff 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,7 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator', 'edx-exams'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator', 'edx-exams', 'subscriptions'] - name: Add new instance to host group local_action: From d988fab56878ea17933ffb9bcb95712146cd3eb7 Mon Sep 17 00:00:00 2001 From: roybrown-trilogyed <49158971+roybrown-trilogyed@users.noreply.github.com> Date: Wed, 22 Mar 2023 10:01:27 -0400 Subject: [PATCH 407/664] feat: Add GATSBY_ZENDESK_KEY CUSTOPS-5681: ADD GATSBY_ZENDESK_KEY Added `GATSBY_ZENDESK_KEY` to enable Zendesk chat web widget (classic) on edx.org Marketplace. --- CHANGELOG.md | 4 ++++ playbooks/roles/prospectus/templates/.env.environment.j2 | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52744394921..b43a6414159 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2023-03-22 + - Role: prospectus + - Added `GATSBY_ZENDESK_KEY` to enable Zendesk chat web widget (classic) on edx.org Marketplace. + - 2023-02-27 - Role: edx_django_service - Enable Celery workers in supervisor when diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 4dda25acc96..5b47d2d01b7 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -11,3 +11,4 @@ GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} +GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} \ No newline at end of file From 5e086513bd2a74a76475abd2bb6670d930f820e4 Mon Sep 17 00:00:00 2001 From: Kaustav Banerjee Date: Mon, 27 Mar 2023 23:47:34 +0530 Subject: [PATCH 408/664] fix: remove dummy order history url (#6820) --- CHANGELOG.md | 5 +++++ playbooks/roles/mfe/defaults/main.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b43a6414159..099bb67bfd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-03-27 + - Changed default value of `MFE_ORDER_HISTORY_URL` to empty string `""` to + prevent MFE header dropdown from rendering `Order History` option when + Order History URL is not configured. + - 2023-03-22 - Role: prospectus - Added `GATSBY_ZENDESK_KEY` to enable Zendesk chat web widget (classic) on edx.org Marketplace. diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 7d99dc5a94e..7e1002a30a1 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -88,7 +88,7 @@ MFE_ENTERPRISE_MARKETING_UTM_SOURCE: "" MFE_ENTERPRISE_MARKETING_UTM_CAMPAIGN: "" MFE_ENTERPRISE_MARKETING_FOOTER_UTM_MEDIUM: "" -MFE_ORDER_HISTORY_URL: https://order-history.example.com +MFE_ORDER_HISTORY_URL: "" MFE_USER_INFO_COOKIE_NAME: edx-user-info MFE_NEW_RELIC_APP_ID: 'fake_app' From 5415e25ea7265a5523ee423a4264d30c6ac561e7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 29 Mar 2023 15:46:29 +0500 Subject: [PATCH 409/664] chore: add x-request-id header in app-common.j2 nginx template --- .../templates/edx/app/nginx/sites-available/app.j2 | 4 ---- .../edx/app/nginx/sites-available/concerns/app-common.j2 | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 index 7dc6897a949..d051e6adc62 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/app.j2 @@ -28,10 +28,6 @@ server { ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}; ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }}; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; - {% if NGINX_ENABLE_REQUEST_TRACKING_ID -%} - # To Track requests - add_header X-Request-ID $request_tracking_id; - {% endif %} {% include "concerns/app-common.j2" %} } diff --git a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/app-common.j2 b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/app-common.j2 index 83e503dab64..d605cca610d 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/app-common.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/nginx/sites-available/concerns/app-common.j2 @@ -8,3 +8,8 @@ {% if edx_django_service_nginx_read_timeout %} proxy_read_timeout {{ edx_django_service_nginx_read_timeout }}; {% endif %} +{% if NGINX_ENABLE_REQUEST_TRACKING_ID %} + + # To Track requests + add_header X-Request-ID $request_tracking_id; +{% endif %} From d473002f0292cd7e4b80a702c6f2df5397c23354 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 29 Mar 2023 17:02:16 -0400 Subject: [PATCH 410/664] fix: try to work around nodejs.org multipart download flakiness --- playbooks/roles/edx_django_service/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index c471cc0f97d..fc901545677 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -147,6 +147,9 @@ creates: "{{ edx_django_service_nodeenv_dir }}" become_user: "{{ edx_django_service_user }}" when: not edx_django_service_enable_experimental_docker_shim + register: node_download_result + until: node_download_result is succeeded + retries: 5 environment: "{{ edx_django_service_environment }}" tags: - install From 41cea8c475a7472e1b11564dd56a60c9de9878e6 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 31 Mar 2023 19:13:50 +0500 Subject: [PATCH 411/664] chore: add insightvm agent role --- playbooks/insightvm_agent.yml | 11 +++++ .../roles/insightvm_agent/defaults/main.yml | 7 +++ .../roles/insightvm_agent/tasks/main.yml | 47 +++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 playbooks/insightvm_agent.yml create mode 100644 playbooks/roles/insightvm_agent/defaults/main.yml create mode 100644 playbooks/roles/insightvm_agent/tasks/main.yml diff --git a/playbooks/insightvm_agent.yml b/playbooks/insightvm_agent.yml new file mode 100644 index 00000000000..f5fe003b3d4 --- /dev/null +++ b/playbooks/insightvm_agent.yml @@ -0,0 +1,11 @@ +- name: Deploy insightvm_agent + hosts: all + become: True + gather_facts: True + vars: + serial_count: 1 + serial: "{{ serial_count }}" + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - insightvm_agent diff --git a/playbooks/roles/insightvm_agent/defaults/main.yml b/playbooks/roles/insightvm_agent/defaults/main.yml new file mode 100644 index 00000000000..48036da7bcd --- /dev/null +++ b/playbooks/roles/insightvm_agent/defaults/main.yml @@ -0,0 +1,7 @@ +--- + +r7_installer_location: /tmp/rapid7_agent_installer.sh +R7_TOKEN: "SET-ME-PLEASE" +R7_IAM_USER_ACCESS_KEY: "SET-ME-PLEASE" +R7_IAM_USER_SECRET_KEY: "SET-ME-PLEASE" +R7_BUCKET: "SET-ME-PLEASE (ex. bucket-name)" diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml new file mode 100644 index 00000000000..242075154b0 --- /dev/null +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -0,0 +1,47 @@ +--- +# insightvm_agent +# +# Example play: +# +# roles: +# - insightvm_agent + +- name: Check if Rapid7 Agent Currently Installed + stat: + path: /etc/systemd/system/ir_agent.service + register: r7_service + tags: + - manage_rapid7_check_agent + +- name: Pull Rapid7 Agent Installer from S3 + aws_s3: + aws_access_key: "{{ R7_IAM_USER_ACCESS_KEY }}" + aws_secret_key: "{{ R7_IAM_USER_SECRET_KEY }}" + bucket: "{{ R7_BUCKET }}" + object: rapid7/rapid7_agent_installer.sh + dest: "{{ r7_installer_location }}" + mode: get + overwrite: different + ignore_nonexistent_bucket: true + tags: + - manage_rapid7_pull_installer + when: not r7_service.stat.exists|bool + +- name: Ensure File Permissions are set + file: + path: "{{ r7_installer_location }}" + mode: "0755" + owner: root + group: root + ignore_errors: "{{ ansible_check_mode }}" + tags: + - manage_rapid7_file_perms + when: not r7_service.stat.exists|bool + +- name: Agent Install + command: "/tmp/rapid7_agent_installer.sh install_start --token {{ R7_TOKEN }}" + tags: + - manage_rapid7_agent_install + ignore_errors: "{{ ansible_check_mode }}" + args: + creates: /etc/systemd/system/ir_agent.service From b4da080f4cce3ff318611a386668558ebab44ebf Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 3 Apr 2023 11:30:21 +0500 Subject: [PATCH 412/664] chore: insightvm agent installation to xqueue --- playbooks/xqueue.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/xqueue.yml b/playbooks/xqueue.yml index 86c3213dd1d..f90c7ece45e 100644 --- a/playbooks/xqueue.yml +++ b/playbooks/xqueue.yml @@ -21,4 +21,5 @@ - role: hermes HERMES_TARGET_SERVICE: 'xqueue' when: XQUEUE_HERMES_ENABLED - + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT From 285ca296e12d4d7581b94e4305dff3488ff81dc9 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 3 Apr 2023 12:45:30 +0500 Subject: [PATCH 413/664] chore: stop ansible from showing insightvm-agnet token --- playbooks/roles/insightvm_agent/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index 242075154b0..8179bf48801 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -40,6 +40,7 @@ - name: Agent Install command: "/tmp/rapid7_agent_installer.sh install_start --token {{ R7_TOKEN }}" + no_log: true tags: - manage_rapid7_agent_install ignore_errors: "{{ ansible_check_mode }}" From 6c9b0881686e20870ce0f604fe9194821f8bf369 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 3 Apr 2023 18:00:41 +0500 Subject: [PATCH 414/664] chore: add InsightVM agent into the palybooks --- playbooks/analyticsapi.yml | 2 ++ playbooks/conductor.yml | 2 ++ playbooks/discovery.yml | 2 ++ playbooks/ecommerce.yml | 3 ++- playbooks/ecomworker.yml | 2 ++ playbooks/edxapp.yml | 2 ++ playbooks/enterprise_catalog.yml | 2 ++ playbooks/forum.yml | 3 ++- playbooks/insights.yml | 3 ++- playbooks/mongo_4_2.yml | 2 ++ playbooks/mongo_mms.yml | 2 ++ playbooks/prospectus.yml | 2 ++ playbooks/tools_jenkins.yml | 2 ++ playbooks/xqwatcher.yml | 2 ++ 14 files changed, 28 insertions(+), 3 deletions(-) diff --git a/playbooks/analyticsapi.yml b/playbooks/analyticsapi.yml index 5551301b698..a8d1f07c6f7 100644 --- a/playbooks/analyticsapi.yml +++ b/playbooks/analyticsapi.yml @@ -24,3 +24,5 @@ - role: hermes HERMES_TARGET_SERVICE: 'analytics_api' when: ANALYTICS_API_HERMES_ENABLED + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/conductor.yml b/playbooks/conductor.yml index 11522c137fa..d02facbe0bb 100644 --- a/playbooks/conductor.yml +++ b/playbooks/conductor.yml @@ -23,3 +23,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/discovery.yml b/playbooks/discovery.yml index eb0717c0823..68cfcf42496 100644 --- a/playbooks/discovery.yml +++ b/playbooks/discovery.yml @@ -24,4 +24,6 @@ - role: hermes HERMES_TARGET_SERVICE: 'discovery' when: DISCOVERY_HERMES_ENABLED + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/ecommerce.yml b/playbooks/ecommerce.yml index 1fd78ca8692..27cc9be3a92 100644 --- a/playbooks/ecommerce.yml +++ b/playbooks/ecommerce.yml @@ -24,4 +24,5 @@ - role: hermes HERMES_TARGET_SERVICE: 'ecommerce' when: ECOMMERCE_HERMES_ENABLED - + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/ecomworker.yml b/playbooks/ecomworker.yml index f7c288770bb..5a9fe7de6a9 100644 --- a/playbooks/ecomworker.yml +++ b/playbooks/ecomworker.yml @@ -20,4 +20,6 @@ - role: hermes HERMES_TARGET_SERVICE: 'ecomworker' when: ECOMMERCE_WORKER_HERMES_ENABLED + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/edxapp.yml b/playbooks/edxapp.yml index 2364f0f585a..ea509022c26 100644 --- a/playbooks/edxapp.yml +++ b/playbooks/edxapp.yml @@ -50,3 +50,5 @@ filename: '{{ hermes_download_dir }}/studio.yml' command: "sudo /bin/cp {{ hermes_download_dir }}/studio.yml {{ COMMON_CFG_DIR }}/studio.yml && sudo /edx/app/edxapp/reload_cms_config.sh" secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','^(.*)$','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/enterprise_catalog.yml b/playbooks/enterprise_catalog.yml index c773022b4dd..f708fadeb7c 100644 --- a/playbooks/enterprise_catalog.yml +++ b/playbooks/enterprise_catalog.yml @@ -19,3 +19,5 @@ - role: hermes HERMES_TARGET_SERVICE: 'enterprise_catalog' when: ENTERPRISE_CATALOG_HERMES_ENABLED + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/forum.yml b/playbooks/forum.yml index 8d356cfa336..22df4f5f10e 100644 --- a/playbooks/forum.yml +++ b/playbooks/forum.yml @@ -21,4 +21,5 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG - + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/insights.yml b/playbooks/insights.yml index 1add0ec6555..ecc32becc90 100644 --- a/playbooks/insights.yml +++ b/playbooks/insights.yml @@ -24,4 +24,5 @@ - role: hermes HERMES_TARGET_SERVICE: 'insights' when: INSIGHTS_HERMES_ENABLED - + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/mongo_4_2.yml b/playbooks/mongo_4_2.yml index 50e75177f24..7f0c02c072e 100644 --- a/playbooks/mongo_4_2.yml +++ b/playbooks/mongo_4_2.yml @@ -27,3 +27,5 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/mongo_mms.yml b/playbooks/mongo_mms.yml index f22c031ebbe..89ad585bc32 100644 --- a/playbooks/mongo_mms.yml +++ b/playbooks/mongo_mms.yml @@ -13,3 +13,5 @@ when: COMMON_ENABLE_DATADOG - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/prospectus.yml b/playbooks/prospectus.yml index ce0a0b51d28..2c95039870c 100644 --- a/playbooks/prospectus.yml +++ b/playbooks/prospectus.yml @@ -29,3 +29,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/tools_jenkins.yml b/playbooks/tools_jenkins.yml index 57848d95a7d..b4fc1808a50 100644 --- a/playbooks/tools_jenkins.yml +++ b/playbooks/tools_jenkins.yml @@ -37,3 +37,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/xqwatcher.yml b/playbooks/xqwatcher.yml index 8798dda77cc..42f052d913f 100644 --- a/playbooks/xqwatcher.yml +++ b/playbooks/xqwatcher.yml @@ -19,3 +19,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG + - role: insightvm_agent + when: COMMON_ENABLE_INSIGHTVM_AGENT From b2c80d1ae958f0a20d494bb0047f02d29ee1a2fc Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 3 Apr 2023 21:09:02 +0500 Subject: [PATCH 415/664] fix: set default value for COMMON_ENABLE_INSIGHTVM_AGENT --- playbooks/roles/common_vars/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 7ed87040a50..db161b70c92 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -107,6 +107,7 @@ COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: False # as well COMMON_ENABLE_NEWRELIC_APP: False COMMON_ENABLE_MINOS: False +COMMON_ENABLE_INSIGHTVM_AGENT: False COMMON_TAG_EC2_INSTANCE: False COMMON_BOTO_VERSION: '2.48.0' COMMON_NODE_VERSION: '12.13.0' From 4495bfc0e25cc7c1c33d9fa742440648e3b0110f Mon Sep 17 00:00:00 2001 From: Marek Wrobel Date: Wed, 5 Apr 2023 15:25:23 -0400 Subject: [PATCH 416/664] Revert "WS-3807: temporarily redirect chatgpt course to search" --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index c495df7386c..a5d49f31115 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -157,8 +157,5 @@ server { } {% endfor %} - - # [WS-3807] Temporary redirect - rewrite ^/course/introduction-to-chatgpt$ "/search?skills.skill=Artificial+Intelligence" redirect; } From bcddf7fa85f8c37b5e1392d86cb45c1679338dfb Mon Sep 17 00:00:00 2001 From: Kaustav Banerjee Date: Mon, 10 Apr 2023 14:11:41 +0530 Subject: [PATCH 417/664] fix: change default mongodb replica set to null (#6831) --- CHANGELOG.md | 4 ++++ playbooks/roles/edxapp/defaults/main.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 099bb67bfd1..e030f33756a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-04-07 + - Changed default value of `EDXAPP_MONGO_REPLICA_SET` to `null` from existing + empty string `""`, to make it compatible with pymongo >= 3.11 in Nutmeg and above. + - 2023-03-27 - Changed default value of `MFE_ORDER_HISTORY_URL` to empty string `""` to prevent MFE header dropdown from rendering `Order History` option when diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 1dfe19c3f92..45b02f25fdf 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -74,7 +74,7 @@ EDXAPP_MONGO_PORT: 27017 EDXAPP_MONGO_USER: 'edxapp' EDXAPP_MONGO_DB_NAME: 'edxapp' EDXAPP_MONGO_USE_SSL: False -EDXAPP_MONGO_REPLICA_SET: '' +EDXAPP_MONGO_REPLICA_SET: null EDXAPP_MONGO_AUTH_DB: '' # Used only if EDXAPP_MONGO_REPLICA_SET is provided. EDXAPP_MONGO_CMS_READ_PREFERENCE: 'PRIMARY' From 6fbec8900dbeacf29165146de370190fab55830e Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 12 Apr 2023 10:46:30 +0500 Subject: [PATCH 418/664] fix: start hermes after supervisor started --- playbooks/roles/hermes/templates/hermes-systemd.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/hermes/templates/hermes-systemd.service.j2 b/playbooks/roles/hermes/templates/hermes-systemd.service.j2 index f9c2b7eb316..7816b136469 100644 --- a/playbooks/roles/hermes/templates/hermes-systemd.service.j2 +++ b/playbooks/roles/hermes/templates/hermes-systemd.service.j2 @@ -1,7 +1,7 @@ [Unit] Description=Hermes - Hermes is the messenger/bureaucrat of the edx stack. It fetches documents and files them on a server's filesystem on a regular basis Documentation=https://github.com/edx/hermes/blob/master/README.md -After=network.target +After=network.target supervisor.service [Service] User={{ hermes_user }} From 34af926cd23fe1493a01f5a1b818dc944ff96b43 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 13 Apr 2023 15:11:08 +0500 Subject: [PATCH 419/664] chore: retain cache and public folders --- playbooks/roles/prospectus/defaults/main.yml | 2 ++ playbooks/roles/prospectus/tasks/main.yml | 31 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index bda2ef045f8..39e6b80d399 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -58,3 +58,5 @@ prospectus_git_identity: "{{ prospectus_app_dir }}/prospectus-git-identity" prospectus_code_dir: "{{ prospectus_app_dir }}/prospectus" prospectus_ssl_nginx_port: 443 prospectus_use_python3: true + +PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS: False diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 75caf9f3a76..853fc36d872 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -5,6 +5,20 @@ state: absent when: PROSPECTUS_GIT_IDENTITY != "none" +- name: check if cache dir exists + stat: + path: "{{ prospectus_code_dir }}/.cache" + register: register_cache_dir + +- name: check if data dir exists + stat: + path: "{{ PROSPECTUS_DATA_DIR }}" + register: register_data_dir + +- name: move cache dir to temp + command: mv {{ prospectus_code_dir }}/.cache /tmp/ + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + - name: Remove old git repo file: state: absent @@ -19,6 +33,7 @@ file: state: absent path: "{{ PROSPECTUS_DATA_DIR }}" + when: not PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS - name: Create prospectus app folder file: @@ -59,6 +74,22 @@ register: prospectus_checkout when: PROSPECTUS_GIT_IDENTITY == "none" +- name: move cache dir to {{ prospectus_code_dir }} + command: mv /tmp/.cache "{{ prospectus_code_dir }}/" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + +- name: create prospectus public folder + file: + path: "{{ prospectus_code_dir }}/public" + state: directory + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_data_dir.stat.exists + +- name: move data dir to {{ prospectus_code_dir }}/public + shell: "mv {{ PROSPECTUS_DATA_DIR }}/* {{ prospectus_code_dir }}/public/" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_data_dir.stat.exists + - name: install python3.8 apt: pkg: From e8182bf41e1ad9f41ee04bb9fb7779a9d3955c47 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 14 Apr 2023 09:59:44 +0500 Subject: [PATCH 420/664] chore: create cache-data folder in tmp --- playbooks/roles/prospectus/tasks/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 853fc36d872..36076b5330e 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -15,8 +15,14 @@ path: "{{ PROSPECTUS_DATA_DIR }}" register: register_data_dir +- name: create prospectus temp cache folder + file: + path: "/tmp/cache-data" + state: directory + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + - name: move cache dir to temp - command: mv {{ prospectus_code_dir }}/.cache /tmp/ + command: mv {{ prospectus_code_dir }}/.cache /tmp/cache-data/ when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists - name: Remove old git repo @@ -75,7 +81,7 @@ when: PROSPECTUS_GIT_IDENTITY == "none" - name: move cache dir to {{ prospectus_code_dir }} - command: mv /tmp/.cache "{{ prospectus_code_dir }}/" + command: mv /tmp/cache-data/.cache "{{ prospectus_code_dir }}/" when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists - name: create prospectus public folder From 72436a7ed7a9d130a8abbc82b352e1ef6df1d753 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 18 Apr 2023 20:15:29 +0500 Subject: [PATCH 421/664] fix: convert PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS to boolean --- playbooks/roles/prospectus/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 36076b5330e..3bc3b7bcce3 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -19,11 +19,11 @@ file: path: "/tmp/cache-data" state: directory - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - name: move cache dir to temp command: mv {{ prospectus_code_dir }}/.cache /tmp/cache-data/ - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - name: Remove old git repo file: @@ -39,7 +39,7 @@ file: state: absent path: "{{ PROSPECTUS_DATA_DIR }}" - when: not PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS + when: not PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool - name: Create prospectus app folder file: @@ -82,7 +82,7 @@ - name: move cache dir to {{ prospectus_code_dir }} command: mv /tmp/cache-data/.cache "{{ prospectus_code_dir }}/" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_cache_dir.stat.exists + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - name: create prospectus public folder file: @@ -90,11 +90,11 @@ state: directory owner: "{{ prospectus_user }}" group: "{{ prospectus_user }}" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_data_dir.stat.exists + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists - name: move data dir to {{ prospectus_code_dir }}/public shell: "mv {{ PROSPECTUS_DATA_DIR }}/* {{ prospectus_code_dir }}/public/" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS and register_data_dir.stat.exists + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists - name: install python3.8 apt: From 4b627d219f1267c44926b79c16326017a2328058 Mon Sep 17 00:00:00 2001 From: k8 <63807774+katebygrace@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:06:28 -0400 Subject: [PATCH 422/664] chore: add fail2ban Add fail2ban for instances --- playbooks/roles/edx_ansible/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edx_ansible/defaults/main.yml b/playbooks/roles/edx_ansible/defaults/main.yml index db0c31aef7f..2c69b5fbf2c 100644 --- a/playbooks/roles/edx_ansible/defaults/main.yml +++ b/playbooks/roles/edx_ansible/defaults/main.yml @@ -19,6 +19,8 @@ # included in the play. EDX_ANSIBLE_DUMP_VARS: false +edx_ansible_debian_running_services: + - fail2ban edx_ansible_debian_pkgs_default: - python-apt @@ -44,7 +46,7 @@ edx_ansible_release_specific_debian_pkgs: focal: - python3-dev -edx_ansible_debian_pkgs: "{{ edx_ansible_debian_pkgs_default + edx_ansible_release_specific_debian_pkgs[ansible_distribution_release] }}" +edx_ansible_debian_pkgs: "{{ edx_ansible_debian_running_services + edx_ansible_debian_pkgs_default + edx_ansible_release_specific_debian_pkgs[ansible_distribution_release] }}" edx_ansible_app_dir: "{{ COMMON_APP_DIR }}/edx_ansible" edx_ansible_code_dir: "{{ edx_ansible_app_dir }}/edx_ansible" From f2b624e500c8c371558a42959eb09f1c1ebd5d39 Mon Sep 17 00:00:00 2001 From: k8 <63807774+katebygrace@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:11:13 -0400 Subject: [PATCH 423/664] chore: add fail2ban Add fail2ban for instances --- playbooks/roles/edx_ansible/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml index ad8265dddbf..011dacfcf14 100644 --- a/playbooks/roles/edx_ansible/tasks/main.yml +++ b/playbooks/roles/edx_ansible/tasks/main.yml @@ -53,6 +53,14 @@ tags: - install:system-requirements +- name: Start and enable running services + ansible.builtin.systemd: + state: started + enabled: true + name: "{{ edx_ansible_debian_running_services }}" + tags: + - install:system-requirements + - include: deploy.yml tags: - deploy From bc773b875be331d77f62b7f3286abfac9ff7c8dc Mon Sep 17 00:00:00 2001 From: k8 <63807774+katebygrace@users.noreply.github.com> Date: Wed, 19 Apr 2023 17:20:04 -0400 Subject: [PATCH 424/664] chore: add fail2ban add fail2ban --- playbooks/roles/edx_ansible/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml index 011dacfcf14..65bfbf414e7 100644 --- a/playbooks/roles/edx_ansible/tasks/main.yml +++ b/playbooks/roles/edx_ansible/tasks/main.yml @@ -53,6 +53,10 @@ tags: - install:system-requirements +- include: deploy.yml + tags: + - deploy + - name: Start and enable running services ansible.builtin.systemd: state: started @@ -60,7 +64,3 @@ name: "{{ edx_ansible_debian_running_services }}" tags: - install:system-requirements - -- include: deploy.yml - tags: - - deploy From c4aae71cbadfc6d1c19789cdc905b2a022131809 Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Fri, 21 Apr 2023 11:31:19 -0400 Subject: [PATCH 425/664] fix: update demjson package --- .github/workflows/playbook-test.yml | 2 +- .github/workflows/syntax-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playbook-test.yml b/.github/workflows/playbook-test.yml index d1d102c45d3..8bb8315feb6 100644 --- a/.github/workflows/playbook-test.yml +++ b/.github/workflows/playbook-test.yml @@ -23,7 +23,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | - pip install demjson + pip install demjson3 pip install -r requirements.txt - name: Run Make test.syntax run: | diff --git a/.github/workflows/syntax-test.yml b/.github/workflows/syntax-test.yml index 2d56edb212d..03693045e0a 100644 --- a/.github/workflows/syntax-test.yml +++ b/.github/workflows/syntax-test.yml @@ -23,7 +23,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | - pip install demjson + pip install demjson3 pip install -r requirements.txt - name: Run Make test.syntax run: | From cc2149a7875acee020230bad026894ce514510f2 Mon Sep 17 00:00:00 2001 From: k8 <63807774+katebygrace@users.noreply.github.com> Date: Fri, 21 Apr 2023 11:49:00 -0400 Subject: [PATCH 426/664] fix: fix fail2ban JIRA:PSRE-2404 --- playbooks/roles/edx_ansible/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml index 65bfbf414e7..3033cd1afe9 100644 --- a/playbooks/roles/edx_ansible/tasks/main.yml +++ b/playbooks/roles/edx_ansible/tasks/main.yml @@ -62,5 +62,6 @@ state: started enabled: true name: "{{ edx_ansible_debian_running_services }}" + with_items: "{{ edx_ansible_debian_running_services }}" tags: - install:system-requirements From 54f0ee5ba02f3c5da71e93f5de03a078d1d3420a Mon Sep 17 00:00:00 2001 From: k8 <63807774+katebygrace@users.noreply.github.com> Date: Fri, 21 Apr 2023 11:51:07 -0400 Subject: [PATCH 427/664] fix: fix fail2ban JIRA:PSRE-2404 Co-authored-by: Adam Blackwell --- playbooks/roles/edx_ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edx_ansible/tasks/main.yml b/playbooks/roles/edx_ansible/tasks/main.yml index 3033cd1afe9..6755493c372 100644 --- a/playbooks/roles/edx_ansible/tasks/main.yml +++ b/playbooks/roles/edx_ansible/tasks/main.yml @@ -61,7 +61,7 @@ ansible.builtin.systemd: state: started enabled: true - name: "{{ edx_ansible_debian_running_services }}" + name: "{{ item }}" with_items: "{{ edx_ansible_debian_running_services }}" tags: - install:system-requirements From 628a5dbe20c69d9d4aa84bf9374dfbb0b230dba1 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Mon, 24 Apr 2023 11:59:54 -0400 Subject: [PATCH 428/664] fix: remove JWT_DECODE_HANDLER override (#6921) Remove the override of JWT_DECODE_HANDLER in the settings files, because it already matches the default set in base.py, and when we wish to remove this custom jwt_decode_handler, it is possible that this setting will unnecessarily cause an error in the future. See https://github.com/openedx/ecommerce/blob/b5fa2e55e3d4f5f8224d381317177cf4086b35a6/ecommerce/settings/base.py#L449 The JWT_AUTH dictionary should not be replaced as a whole, but should only replace keys that are declared, according to this: https://github.com/openedx/ecommerce/blob/b5fa2e55e3d4f5f8224d381317177cf4086b35a6/ecommerce/settings/production.py#L51-L53 --- docker/build/ecommerce/ecommerce.yml | 1 - playbooks/roles/ecommerce/defaults/main.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/docker/build/ecommerce/ecommerce.yml b/docker/build/ecommerce/ecommerce.yml index baecec8a922..84ab0d84f41 100644 --- a/docker/build/ecommerce/ecommerce.yml +++ b/docker/build/ecommerce/ecommerce.yml @@ -55,7 +55,6 @@ JWT_AUTH: JWT_ALGORITHM: HS256 JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature - JWT_DECODE_HANDLER: ecommerce.extensions.api.handlers.jwt_decode_handler JWT_ISSUERS: - AUDIENCE: lms-key ISSUER: http://localhost:18000/oauth2 diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 0e3decffe41..25d95a0ee1b 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -63,7 +63,6 @@ ECOMMERCE_LOGOUT_URL: '{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}/logout/' ECOMMERCE_LMS_URL_ROOT: 'http://127.0.0.1:8000' ECOMMERCE_JWT_ALGORITHM: 'HS256' ECOMMERCE_JWT_VERIFY_EXPIRATION: true -ECOMMERCE_JWT_DECODE_HANDLER: 'ecommerce.extensions.api.handlers.jwt_decode_handler' ECOMMERCE_JWT_ISSUERS: - ISSUER: "{{ COMMON_JWT_ISSUER }}" AUDIENCE: "{{ COMMON_JWT_AUDIENCE }}" @@ -223,7 +222,6 @@ ecommerce_config: JWT_ALGORITHM: '{{ ECOMMERCE_JWT_ALGORITHM }}' JWT_VERIFY_EXPIRATION: '{{ ECOMMERCE_JWT_VERIFY_EXPIRATION }}' JWT_LEEWAY: '{{ ECOMMERCE_JWT_LEEWAY }}' - JWT_DECODE_HANDLER: '{{ ECOMMERCE_JWT_DECODE_HANDLER }}' JWT_ISSUERS: '{{ ECOMMERCE_JWT_ISSUERS }}' JWT_PUBLIC_SIGNING_JWK_SET: '{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}' JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' From 05e056e3eb2d2d70265425a5c43015b71cbbea86 Mon Sep 17 00:00:00 2001 From: srwang Date: Mon, 24 Apr 2023 12:44:05 -0400 Subject: [PATCH 429/664] feat: upgrade prospectus node --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 39e6b80d399..f1675d4f3ba 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '18.13.0' +PROSPECTUS_NODE_VERSION: '19.3.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From 5bb4ee51fe1ea320b8442264414bbba0a9d1c448 Mon Sep 17 00:00:00 2001 From: srwang <12386424+srwang@users.noreply.github.com> Date: Mon, 24 Apr 2023 15:46:51 -0400 Subject: [PATCH 430/664] Revert "feat: upgrade prospectus node" --- playbooks/roles/prospectus/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index f1675d4f3ba..39e6b80d399 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -37,7 +37,7 @@ PROSPECTUS_GIT_IDENTITY: "none" prospectus_repo: 'ssh://git@github.com/edx/prospectus.git' PROSPECTUS_VERSION: 'master' edx_django_service_use_python3: false -PROSPECTUS_NODE_VERSION: '19.3.0' +PROSPECTUS_NODE_VERSION: '18.13.0' prospectus_service_name: 'prospectus' prospectus_home: '{{ COMMON_APP_DIR }}/{{ prospectus_service_name }}' prospectus_venv_dir: '{{ prospectus_home }}/venvs/{{ prospectus_service_name }}' From e13361a7a04868c67f15f3a0015b9afe77f628f1 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 28 Apr 2023 16:39:37 +0500 Subject: [PATCH 431/664] feat: add ansible play to create postgresql dbs and users --- playbooks/create_pgsql_db_and_users.yml | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 playbooks/create_pgsql_db_and_users.yml diff --git a/playbooks/create_pgsql_db_and_users.yml b/playbooks/create_pgsql_db_and_users.yml new file mode 100644 index 00000000000..10a0ee780be --- /dev/null +++ b/playbooks/create_pgsql_db_and_users.yml @@ -0,0 +1,65 @@ +# +# This play will create databases and user for an application. +# It can be run like so: +# +# ansible-playbook -c local -i 'localhost,' create_pgsql_db_and_users.yml -e@./db.yml +# +# If running ansible from a python virtualenv you will need a command like the following +# +# ansible-playbook -c local -i 'localhost,' create_pgsql_db_and_users.yml -e@./db.yml -e "ansible_python_interpreter=$(which python)" +# +# the content of db.yml contains the following dictionaries + +# database_connection: &default_connection +# login_host: "pgsql.example.org" +# login_user: "root" +# login_password: "super-secure-password" + +# databases: +# - name: "example" +# state: "present" +# <<: *default_connection + +# database_users: +# - name: "example" +# db: "example" +# state: "present" +# password: "user-password" +# privileges: "ALL" +# <<: *default_connection + +- name: Create databases and users + hosts: all + gather_facts: False + tasks: + # Install required library, currently this needs to be available + # to system python. + - name: install PostgreSQL client + pip: name={{item}} state=present + with_items: + - psycopg2-binary + + - name: create PostgreSQL databases + postgresql_db: + db: "{{ item.name}}" + state: "{{ item.state }}" + login_host: "{{ item.login_host }}" + login_user: "{{ item.login_user }}" + login_password: "{{ item.login_password }}" + with_items: "{{ databases }}" + tags: + - dbs + + - name: create PostgreSQL users and assign privileges + postgresql_user: + db: "{{ item.db }}" + name: "{{ item.name }}" + state: "{{ item.state | default('present') }}" + priv: "{{ item.privileges }}" + password: "{{ item.password }}" + login_host: "{{ item.login_host }}" + login_user: "{{ item.login_user }}" + login_password: "{{ item.login_password }}" + with_items: "{{ database_users }}" + tags: + - users From 1d5e1e21c707da01fcd0ab764351f0f70115f183 Mon Sep 17 00:00:00 2001 From: RG Date: Wed, 3 May 2023 12:38:09 +0300 Subject: [PATCH 432/664] feat: Add settings for Verifiable Credentials This changes allow to configure Verifiable Credentials settings for the "credentials" service. Related tickets: * [OeX_Cred-426](https://youtrack.raccoongang.com/issue/OeX_Cred-426) --- playbooks/roles/credentials/defaults/main.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/playbooks/roles/credentials/defaults/main.yml b/playbooks/roles/credentials/defaults/main.yml index 937f3797dbc..d19f1f78e14 100644 --- a/playbooks/roles/credentials/defaults/main.yml +++ b/playbooks/roles/credentials/defaults/main.yml @@ -99,6 +99,7 @@ CREDENTIALS_FILE_STORAGE_BACKEND: {} CREDENTIALS_CORS_ORIGIN_ALLOW_ALL: false CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT: - '{{ CREDENTIALS_DOMAIN }}' + - '{{ CREDENTIALS_LEARNER_RECORD_MFE_HOSTNAME }}' CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA: [] CREDENTIALS_CORS_ORIGIN_WHITELIST: '{{ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT + CREDENTIALS_CORS_ORIGIN_WHITELIST_EXTRA }}' @@ -124,8 +125,12 @@ credentials_service_config_overrides: CREDENTIALS_SERVICE_USER: '{{ CREDENTIALS_SERVICE_USER }}' FILE_STORAGE_BACKEND: '{{ CREDENTIALS_FILE_STORAGE_BACKEND }}' LANGUAGE_COOKIE_NAME: '{{ CREDENTIALS_LANGUAGE_COOKIE_NAME }}' + USE_LEARNER_RECORD_MFE: '{{ CREDENTIALS_USE_LEARNER_RECORD_MFE }}' + LEARNER_RECORD_MFE_RECORDS_PAGE_URL: "https://{{ CREDENTIALS_LEARNER_RECORD_MFE_HOSTNAME }}/" CSRF_COOKIE_SECURE: "{{ CREDENTIALS_CSRF_COOKIE_SECURE }}" + CSRF_TRUSTED_ORIGINS: "{{ CREDENTIALS_CSRF_TRUSTED_ORIGINS }}" USERNAME_REPLACEMENT_WORKER: "{{ CREDENTIALS_USERNAME_REPLACEMENT_WORKER }}" + VERIFIABLE_CREDENTIALS: "{{ CREDENTIALS_VERIFIABLE_CREDENTIALS }}" # See edx_django_service_automated_users for an example of what this should be CREDENTIALS_AUTOMATED_USERS: {} @@ -137,6 +142,8 @@ CREDENTIALS_LMS_URL_ROOT: !!null CREDENTIALS_DISCOVERY_API_URL: !!null CREDENTIALS_CSRF_COOKIE_SECURE: false +CREDENTIALS_CSRF_TRUSTED_ORIGINS: + - "{{ CREDENTIALS_LEARNER_RECORD_MFE_HOSTNAME }}" CREDENTIALS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING: false @@ -155,3 +162,26 @@ CREDENTIALS_COPY_CONFIG_ENABLED: "{{ COMMON_COPY_CONFIG_ENABLED }}" CREDENTIALS_ENABLE_ADMIN_URLS_RESTRICTION: false CREDENTIALS_ADMIN_URLS: - admin + +CREDENTIALS_USE_LEARNER_RECORD_MFE: false +CREDENTIALS_LEARNER_RECORD_MFE_HOSTNAME: "learner-record.mfe.CHANGE-ME" + +# TODO: Link to VC documentation +# +# TODO: Comment about adding wallet python module to CREDENTIALS_EXTRA_REQUIREMENTS variable +# +CREDENTIALS_VERIFIABLE_CREDENTIALS: + DEFAULT_DATA_MODELS: + - "credentials.apps.verifiable_credentials.composition.verifiable_credentials.VerifiableCredentialsDataModel" + - "credentials.apps.verifiable_credentials.composition.open_badges.OpenBadgesDataModel" + DEFAULT_STORAGES: + - "credentials.apps.verifiable_credentials.storages.learner_credential_wallet.LCWallet" + DEFAULT_ISSUER: + NAME: "Default (system-wide)" + ID: "generate-me-with-didkit-lib" + KEY: "generate-me-with-didkit-lib" + DEFAULT_ISSUANCE_REQUEST_SERIALIZER: "credentials.apps.verifiable_credentials.issuance.serializers.IssuanceLineSerializer" + DEFAULT_RENDERER: "credentials.apps.verifiable_credentials.issuance.renderers.JSONLDRenderer" + STATUS_LIST_STORAGE: "credentials.apps.verifiable_credentials.storages.status_list.StatusList2021" + STATUS_LIST_DATA_MODEL: "credentials.apps.verifiable_credentials.composition.status_list.StatusListDataModel" + STATUS_LIST_LENGTH: 10000 From ecf034dece27f6612d71f0de75dc74264c8d42ec Mon Sep 17 00:00:00 2001 From: Kyrylo Kholodenko Date: Wed, 10 May 2023 19:40:39 +0300 Subject: [PATCH 433/664] docs: add reference to VC documentation * feat: [OeX_Cred-790] add comment about verifiable credentials documentation * docs: remove resolved TODO --------- Co-authored-by: Glib Glugovskiy --- playbooks/roles/credentials/defaults/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/credentials/defaults/main.yml b/playbooks/roles/credentials/defaults/main.yml index d19f1f78e14..e965d975e03 100644 --- a/playbooks/roles/credentials/defaults/main.yml +++ b/playbooks/roles/credentials/defaults/main.yml @@ -166,10 +166,9 @@ CREDENTIALS_ADMIN_URLS: CREDENTIALS_USE_LEARNER_RECORD_MFE: false CREDENTIALS_LEARNER_RECORD_MFE_HOSTNAME: "learner-record.mfe.CHANGE-ME" -# TODO: Link to VC documentation -# -# TODO: Comment about adding wallet python module to CREDENTIALS_EXTRA_REQUIREMENTS variable -# +# NOTE: Optional Verifiable Credentials feature +# Documentation can be found at +# https://edx-credentials.readthedocs.io/en/latest/verifiable_credentials/overview.html CREDENTIALS_VERIFIABLE_CREDENTIALS: DEFAULT_DATA_MODELS: - "credentials.apps.verifiable_credentials.composition.verifiable_credentials.VerifiableCredentialsDataModel" From 01ccf5017345d5d71f010f87f431acd227cfa67d Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Thu, 11 May 2023 14:17:31 -0400 Subject: [PATCH 434/664] feat!: Remove obsolete Jenkins and load testing configuration (#6929) There was a large amount of code in this repository that configured a now-decommissioned Jenkins server and no-longer-used load testing environments. Also removed some Android build configuration that was only ever used on that Jenkins server (this is now done differently in GitHub Actions). --- docker/build/jenkins_build/Dockerfile | 28 -- .../build/jenkins_build/ansible_overrides.yml | 76 ---- docker/build/jenkins_worker/Dockerfile | 39 -- .../jenkins_worker/ansible_overrides.yml | 29 -- docker/plays/jenkins_build.yml | 10 - docker/plays/jenkins_worker.yml | 35 -- playbooks/android_sdk.yml | 10 - playbooks/jenkins_build.yml | 23 - playbooks/jenkins_build_bastion.yml | 25 -- playbooks/jenkins_testeng_master.yml | 65 --- playbooks/jenkins_worker.yml | 35 -- playbooks/jenkins_worker_android.yml | 20 - playbooks/jenkins_worker_codejail.yml | 21 - playbooks/jenkins_worker_user_retire.yml | 21 - playbooks/locust.yml | 9 - playbooks/roles/android_sdk/defaults/main.yml | 42 -- playbooks/roles/android_sdk/tasks/main.yml | 98 ----- .../roles/jenkins_build/defaults/main.yml | 398 ------------------ .../jenkins_build/files/splunk/splunk.groovy | 9 - playbooks/roles/jenkins_build/meta/main.yml | 34 -- .../roles/jenkins_worker/defaults/main.yml | 38 -- playbooks/roles/jenkins_worker/meta/main.yml | 46 -- .../tasks/loadtest_driver_worker.yml | 36 -- playbooks/roles/jenkins_worker/tasks/main.yml | 26 -- playbooks/roles/jenkins_worker/tasks/node.yml | 16 - .../roles/jenkins_worker/tasks/packer.yml | 8 - .../roles/jenkins_worker/tasks/python.yml | 73 ---- .../tasks/python_platform_worker.yml | 90 ---- .../roles/jenkins_worker/tasks/system.yml | 50 --- playbooks/roles/jenkins_worker/tasks/test.yml | 11 - .../tasks/test_android_worker.yml | 88 ---- .../tasks/test_codejail_worker.yml | 0 .../tasks/test_platform_worker.yml | 42 -- .../jenkins_worker/templates/jenkins_env.j2 | 5 - .../templates/jenkins_nodeenv.j2 | 8 - .../roles/loadtest_driver/defaults/main.yml | 14 - playbooks/roles/loadtest_driver/meta/main.yml | 15 - .../roles/loadtest_driver/tasks/main.yml | 11 - playbooks/roles/locust/defaults/main.yml | 63 --- playbooks/roles/locust/meta/main.yml | 25 -- playbooks/roles/locust/tasks/main.yml | 77 ---- .../locust/templates/etc/motd.tail.locust.j2 | 14 - .../templates/locust_home/bash_aliases.j2 | 2 - playbooks/roles/rbenv/tasks/main.yml | 2 - util/maintenance.sh | 4 +- util/packer/README.rst | 21 - util/packer/jenkins_worker.json | 80 ---- util/packer/jenkins_worker_android.json | 69 --- util/packer/jenkins_worker_codejail.json | 69 --- util/packer/jenkins_worker_simple.json | 70 --- util/packer/jenkins_worker_user_retire.json | 68 --- util/packer/stop-automatic-updates.sh | 23 - util/packer/webpagetest.json | 26 -- util/parsefiles_config.yml | 2 - 54 files changed, 2 insertions(+), 2217 deletions(-) delete mode 100644 docker/build/jenkins_build/Dockerfile delete mode 100644 docker/build/jenkins_build/ansible_overrides.yml delete mode 100644 docker/build/jenkins_worker/Dockerfile delete mode 100644 docker/build/jenkins_worker/ansible_overrides.yml delete mode 100644 docker/plays/jenkins_build.yml delete mode 100644 docker/plays/jenkins_worker.yml delete mode 100644 playbooks/android_sdk.yml delete mode 100644 playbooks/jenkins_build.yml delete mode 100644 playbooks/jenkins_build_bastion.yml delete mode 100644 playbooks/jenkins_testeng_master.yml delete mode 100644 playbooks/jenkins_worker.yml delete mode 100644 playbooks/jenkins_worker_android.yml delete mode 100644 playbooks/jenkins_worker_codejail.yml delete mode 100644 playbooks/jenkins_worker_user_retire.yml delete mode 100644 playbooks/locust.yml delete mode 100644 playbooks/roles/android_sdk/defaults/main.yml delete mode 100644 playbooks/roles/android_sdk/tasks/main.yml delete mode 100644 playbooks/roles/jenkins_build/defaults/main.yml delete mode 100644 playbooks/roles/jenkins_build/files/splunk/splunk.groovy delete mode 100644 playbooks/roles/jenkins_build/meta/main.yml delete mode 100644 playbooks/roles/jenkins_worker/defaults/main.yml delete mode 100644 playbooks/roles/jenkins_worker/meta/main.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/main.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/node.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/packer.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/python.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/system.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/test.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/test_android_worker.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/test_codejail_worker.yml delete mode 100644 playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml delete mode 100644 playbooks/roles/jenkins_worker/templates/jenkins_env.j2 delete mode 100644 playbooks/roles/jenkins_worker/templates/jenkins_nodeenv.j2 delete mode 100644 playbooks/roles/loadtest_driver/defaults/main.yml delete mode 100644 playbooks/roles/loadtest_driver/meta/main.yml delete mode 100644 playbooks/roles/loadtest_driver/tasks/main.yml delete mode 100644 playbooks/roles/locust/defaults/main.yml delete mode 100644 playbooks/roles/locust/meta/main.yml delete mode 100644 playbooks/roles/locust/tasks/main.yml delete mode 100644 playbooks/roles/locust/templates/etc/motd.tail.locust.j2 delete mode 100644 playbooks/roles/locust/templates/locust_home/bash_aliases.j2 delete mode 100644 util/packer/README.rst delete mode 100644 util/packer/jenkins_worker.json delete mode 100644 util/packer/jenkins_worker_android.json delete mode 100644 util/packer/jenkins_worker_codejail.json delete mode 100644 util/packer/jenkins_worker_simple.json delete mode 100644 util/packer/jenkins_worker_user_retire.json delete mode 100644 util/packer/stop-automatic-updates.sh delete mode 100644 util/packer/webpagetest.json diff --git a/docker/build/jenkins_build/Dockerfile b/docker/build/jenkins_build/Dockerfile deleted file mode 100644 index 016ffe7dee8..00000000000 --- a/docker/build/jenkins_build/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/jenkins_build/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/xenial-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" - -USER root -RUN apt-get update - -ADD . /edx/app/edx_ansible/edx_ansible -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays -COPY docker/build/jenkins_build/ansible_overrides.yml / -RUN PYTHONUNBUFFERED=1 /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook \ - -v jenkins_build.yml \ - -i '127.0.0.1,' \ - -c local \ - -e@/ansible_overrides.yml \ - -t 'install' \ - -vv - -CMD /bin/su -l jenkins --shell=/bin/bash -c "/usr/bin/daemon -f --name=jenkins --inherit --env=JENKINS_HOME=/edx/var/jenkins --output=/var/log/jenkins/jenkins.log --pidfile=/var/run/jenkins/jenkins.pid -- /usr/bin/java -jar /usr/share/jenkins/jenkins.war --webroot=/var/cache/jenkins/war --httpPort=8080 --ajp13Port=-1" diff --git a/docker/build/jenkins_build/ansible_overrides.yml b/docker/build/jenkins_build/ansible_overrides.yml deleted file mode 100644 index 5b49ca9470d..00000000000 --- a/docker/build/jenkins_build/ansible_overrides.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -# This docker image for build jenkins is currently intended for local -# development, or as a base for the community, so we leave out github oauth and -# add the seed job creator. -build_jenkins_configuration_scripts: - - 1addJarsToClasspath.groovy - - 2checkInstalledPlugins.groovy - - 3importCredentials.groovy - - 3installGroovy.groovy - - 3installPython.groovy - - 3mainConfiguration.groovy - - 3setGlobalProperties.groovy - - 3shutdownCLI.groovy - - 4configureEc2Plugin.groovy -# - 4configureGHOAuth.groovy # intentionally left commented out - - 4configureGHPRB.groovy - - 4configureGit.groovy - - 4configureGithub.groovy - - 4configureMailerPlugin.groovy - - 4configureMaskPasswords.groovy - - 4configureSecurity.groovy - - 5configureEmailExtension.groovy - - 5addSeedJob.groovy # added this - - 5createLoggers.groovy - - -jenkins_common_non_plugin_template_files: - - credentials - - ec2_config - - email_ext_config - - ghprb_config - - git_config - - github_config - # - github_oauth # intentionally commented out - - groovy_config - - log_config - - mailer_config - - main_config - - mask_passwords_config - - properties_config - - python_config - - security - - seed_config - -# Add the jenkins-worker label so that this jenkins master will work -# out-of-the-box for running most kinds of jobs. This makes integration -# testing easier, and is easier for the openedx community. -# Also add the android-worker label so that android testing can be done -# easily on a local dev environment. NOTE: this also requires running -# playbooks/android_sdk.yml in order to have all of the necessary Android -# compilation and testing tools, but isn't necessary for most cases and -# therefore should be omitted from normal builds of this container. -jenkins_common_main_labels: - - 'dsl-seed-runner' - - 'backup-runner' - - 'jenkins-worker' # added this - - 'android-worker' - -# We're running all our jobs on the Jenkins Master by default (one container -# only), so we need to bump up the number of executors for some jobs with -# downstream jobs to work correctly. -jenkins_common_main_num_executors: 6 - -# Add basic permissions for a non-authenticated user to be able to view -# the jenkins instance and its jobs. When used for development, an -# initial admin password is generated and will be used for job creation. -# However, in order to get to the login page, these permissions must be -# configured. -JENKINS_SECURITY_GROUPS: - - NAME: 'anonymous' - PERMISSIONS: - - 'hudson.model.Hudson.Read' - - 'hudson.model.Item.Discover' - - 'hudson.model.Item.Read' - USERS: - - 'anonymous' diff --git a/docker/build/jenkins_worker/Dockerfile b/docker/build/jenkins_worker/Dockerfile deleted file mode 100644 index daa22ff5424..00000000000 --- a/docker/build/jenkins_worker/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/jenkins_worker/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -# Run the edxapp play with custom ansible overrides -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" -USER root - -ADD . /edx/app/edx_ansible/edx_ansible -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays - -COPY docker/build/jenkins_worker/ansible_overrides.yml /jenkins_worker/ansible_overrides.yml -COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml - -ARG OPENEDX_RELEASE=master -ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} -RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook jenkins_worker.yml \ - -c local -i '127.0.0.1,' \ - --extra-vars="@/jenkins_worker/ansible_overrides.yml" \ - --extra-vars="@/devstack/ansible_overrides.yml" \ - -# Add sshd to enable jenkins master to ssh into containers -RUN apt-get update \ - && apt-get install -y openssh-server \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ARG JENKINS_WORKER_KEY_URL=https://files.edx.org/testeng/jenkins.keys -RUN mkdir /var/run/sshd \ - && curl ${JENKINS_WORKER_KEY_URL} --create-dirs -o /edx/app/edxapp/.ssh/authorized_keys - -CMD ["/usr/sbin/sshd", "-D"] -EXPOSE 22 diff --git a/docker/build/jenkins_worker/ansible_overrides.yml b/docker/build/jenkins_worker/ansible_overrides.yml deleted file mode 100644 index 5cd7adb2d96..00000000000 --- a/docker/build/jenkins_worker/ansible_overrides.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -COMMON_SECURITY_UPDATES: true -EDXAPP_INSTALL_PRIVATE_REQUIREMENTS: true -EDXAPP_LMS_BASE_SCHEME: http -EDXAPP_LMS_GUNICORN_EXTRA_CONF: 'reload = True' -EDXAPP_NO_PREREQ_INSTALL: 0 -EDXAPP_OAUTH_ENFORCE_SECURE: false -EDXAPP_PYTHON_SANDBOX: false -EDXAPP_SETTINGS: 'devstack_docker' -MONGO_AUTH: false -SECURITY_UPGRADE_ON_ANSIBLE: true - -devstack: true -edxapp_debian_pkgs_extra: - - mongodb-clients -edxapp_environment_extra: - SELENIUM_BROWSER: 'firefox' - SELENIUM_HOST: 'localhost' - SELENIUM_PORT: '4444' -edxapp_npm_production: 'no' -edxapp_requirements_files: - - "{{ testing_requirements_file }}" -edxapp_user: ubuntu -edxapp_user_createhome: 'yes' -edxapp_user_shell: '/bin/bash' -migrate_db: false -mongo_enable_journal: false -service_variants_enabled: [] -testing_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/testing.txt" diff --git a/docker/plays/jenkins_build.yml b/docker/plays/jenkins_build.yml deleted file mode 100644 index 25c9a6556a0..00000000000 --- a/docker/plays/jenkins_build.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deploy the build jenkins - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - common - - jenkins_build diff --git a/docker/plays/jenkins_worker.yml b/docker/plays/jenkins_worker.yml deleted file mode 100644 index 532ec60d878..00000000000 --- a/docker/plays/jenkins_worker.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Configure a Jenkins worker instance -# This has all the requirements to run test jobs, -# but not the Jenkins Java app. - -- name: Configure instance(s) - hosts: all - become: True - gather_facts: True - vars: - mongo_enable_journal: False - serial_count: 1 - platform_worker: True - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - MONGO_AUTH: false - SQLITE_AUTOCONF_URL: "https://www.sqlite.org/2019/sqlite-autoconf-3280000.tar.gz" - SQLITE_AUTOCONF_CREATED_PATH: "sqlite-autoconf-3280000" - SQLITE_FIX_PYTHON_PATH: "/home/jenkins/edx-venv/bin/python" - serial: "{{ serial_count }}" - vars_files: - - roles/edxapp/defaults/main.yml - - roles/xqueue/defaults/main.yml - - roles/forum/defaults/main.yml - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - docker-tools - - mysql - - edxlocal - - memcache - - mongo_3_2 - - browsers - - jenkins_worker - - sqlite_fix - - newrelic_infrastructure diff --git a/playbooks/android_sdk.yml b/playbooks/android_sdk.yml deleted file mode 100644 index 54a79f45095..00000000000 --- a/playbooks/android_sdk.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Configure a system to compile, test and sign the Android client. Use this -# play to configure local testing environments. If you need to configure a -# Jenkins worker for Android tasks, use playbooks/jenkins_worker_android.yml ---- -- name: Configure a system for building and testing the edX Android app - hosts: localhost - become: True - gather_facts: True - roles: - - android_sdk diff --git a/playbooks/jenkins_build.yml b/playbooks/jenkins_build.yml deleted file mode 100644 index dc9c07d80b6..00000000000 --- a/playbooks/jenkins_build.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Configure instance(s) - hosts: all - become: True - gather_facts: True - vars: - COMMON_ENABLE_DATADOG: False - COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: True - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - role: datadog - when: COMMON_ENABLE_DATADOG - - role: datadog-uninstall - when: not COMMON_ENABLE_DATADOG - - jenkins_build - - role: newrelic_infrastructure - when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - tags: - - newreliconly diff --git a/playbooks/jenkins_build_bastion.yml b/playbooks/jenkins_build_bastion.yml deleted file mode 100644 index 3f25b9d984b..00000000000 --- a/playbooks/jenkins_build_bastion.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Bootstrap instance(s) - hosts: all - gather_facts: no - become: True - roles: - - python - -- name: Configure instance(s) - hosts: all - become: True - gather_facts: True - vars: - COMMON_ENABLE_DATADOG: False - COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: True - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - role: newrelic_infrastructure - when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - tags: - - newreliconly diff --git a/playbooks/jenkins_testeng_master.yml b/playbooks/jenkins_testeng_master.yml deleted file mode 100644 index f69eef5642a..00000000000 --- a/playbooks/jenkins_testeng_master.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Configure a Jenkins master instance for testeng -# This has the Jenkins Java app, but none of the requirements -# to run the tests. - -- name: Configure instance(s) - hosts: jenkins_master - become: True - gather_facts: True - vars: - COMMON_DATA_DIR: "/mnt2" - COMMON_ENABLE_DATADOG: True - COMMON_ENABLE_SPLUNKFORWARDER: True - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - jenkins_jvm_args: "-Djava.awt.headless=true -Xmx8192m -XX:MaxPermSize=512m" - - SPLUNKFORWARDER_LOG_ITEMS: - - source: '/var/lib/jenkins/jobs/*/builds/*/junitResult.xml' - recursive: true - index: 'testeng' - sourcetype: junit - followSymlink: false - crcSalt: '' - - - source: '/var/lib/jenkins/jobs/*/builds/*/build.xml' - index: 'testeng' - recursive: true - sourcetype: build_result - followSymlink: false - crcSalt: '' - blacklist: '(((\.(gz))|\d)$)|(.*seed.*)' - - - source: '/var/lib/jenkins/jobs/*/builds/*/log' - index: 'testeng' - recursive: true - sourcetype: build_log - followSymlink: false - crcSalt: '' - blacklist: '(((\.(gz))|\d)$)|(.*seed.*)' - - - source: '/var/lib/jenkins/jobs/*/builds/*/archive/test_root/log/timing.*.log' - index: 'testeng' - sourcetype: 'json_timing_log' - followSymlink: false - - - source: '/var/log/jenkins/jenkins.log' - index: 'testeng' - recursive: false - followSymlink: false - - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - role: datadog - when: COMMON_ENABLE_DATADOG - - role: datadog-uninstall - when: not COMMON_ENABLE_DATADOG - - jenkins_master - - # run just the splunkforwarder role by using '--tags "splunkonly"' - # e.g. ansible-playbook jenkins_testeng_master.yml -i inventory.ini --tags "splunkonly" -vvvv - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER - tags: splunkonly - become: True diff --git a/playbooks/jenkins_worker.yml b/playbooks/jenkins_worker.yml deleted file mode 100644 index d180bbf4476..00000000000 --- a/playbooks/jenkins_worker.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Configure a Jenkins worker instance -# This has all the requirements to run test jobs, -# but not the Jenkins Java app. - -- name: Configure instance(s) - hosts: jenkins_worker - become: True - gather_facts: True - vars: - mongo_enable_journal: False - serial_count: 1 - platform_worker: True - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - MONGO_AUTH: false - SQLITE_AUTOCONF_URL: "https://www.sqlite.org/2019/sqlite-autoconf-3280000.tar.gz" - SQLITE_AUTOCONF_CREATED_PATH: "sqlite-autoconf-3280000" - SQLITE_FIX_PYTHON_PATH: "/home/jenkins/edx-venv/bin/python" - serial: "{{ serial_count }}" - vars_files: - - roles/edxapp/defaults/main.yml - - roles/xqueue/defaults/main.yml - - roles/forum/defaults/main.yml - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - docker-tools - - mysql - - edxlocal - - memcache - - mongo_3_2 - - browsers - - jenkins_worker - - sqlite_fix - - newrelic_infrastructure diff --git a/playbooks/jenkins_worker_android.yml b/playbooks/jenkins_worker_android.yml deleted file mode 100644 index 0de550bf8fa..00000000000 --- a/playbooks/jenkins_worker_android.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Configure a Jenkins worker instance -# This has all the requirements to build, sign, and test the android -# client, but does not contain the platform - -- name: Configure instance(s) - hosts: jenkins_worker - become: True - gather_facts: True - vars: - serial_count: 1 - android_worker: True - jenkins_groups: "jenkins" - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - serial: "{{ serial_count }}" - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - jenkins_worker - - newrelic_infrastructure diff --git a/playbooks/jenkins_worker_codejail.yml b/playbooks/jenkins_worker_codejail.yml deleted file mode 100644 index 7e8cc66085a..00000000000 --- a/playbooks/jenkins_worker_codejail.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Configure a Jenkins worker instance specifically for running tests for -# CodeJail, which requires specific set up with regards to python execution - -- name: Configure instance(s) - hosts: jenkins_worker - become: True - gather_facts: True - vars: - serial_count: 1 - codejail_worker: True - codejail_sandbox_caller: 'jenkins' - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - serial: "{{ serial_count }}" - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - docker-tools - - jenkins_worker - - codejail - - newrelic_infrastructure diff --git a/playbooks/jenkins_worker_user_retire.yml b/playbooks/jenkins_worker_user_retire.yml deleted file mode 100644 index 6ab180a8660..00000000000 --- a/playbooks/jenkins_worker_user_retire.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Configure a Jenkins worker instance to run the user retirement jobs. - -- name: Configure instance(s) - hosts: jenkins_worker - become: True - gather_facts: True - vars: - mongo_enable_journal: False - serial_count: 1 - COMMON_SECURITY_UPDATES: yes - SECURITY_UPGRADE_ON_ANSIBLE: true - MONGO_AUTH: false - jenkins_worker_install_python27: false - serial: "{{ serial_count }}" - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - docker-tools - - memcache - - mongo_3_2 - - jenkins_worker diff --git a/playbooks/locust.yml b/playbooks/locust.yml deleted file mode 100644 index 613f2279f80..00000000000 --- a/playbooks/locust.yml +++ /dev/null @@ -1,9 +0,0 @@ - -- name: Deploy Locust - hosts: all - become: True - gather_facts: True - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - locust diff --git a/playbooks/roles/android_sdk/defaults/main.yml b/playbooks/roles/android_sdk/defaults/main.yml deleted file mode 100644 index ae719265275..00000000000 --- a/playbooks/roles/android_sdk/defaults/main.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# User/group to manage Android SDK -android_user: "android" -android_group: "android" -# Tarball to download - -# https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip -android_download: "sdk-tools-linux-4333796.zip" - -# Checksum of Android SDK (from: https://developer.android.com/studio/index.html#downloads) -android_checksum: "8c7c28554a32318461802c1291d76fccfafde054" - -# path to installed android sdk -android_home: "/opt/android-sdk-linux" - -# The SDK version used to compile the project | 6 | Android SDK Platform 28 -android_build_targets: "\"platforms;android-28\"" - -# other android dependencies that cannot be tested via the android sdk manager. instead, stat the android_test_path to test for presence of the package -# Plateform Tools | 6 | Android SDK Platform 28 -# The BuildTools | Android SDK Build-Tools 28.0.3 -# Additional components -# extras;google;m2repository | 58 | Google Repository -# extras;android;m2repository| 47.0.0 | Android Support Repository -android_tools: - - { package: "\"platform-tools\"", android_test_path: 'platform-tools' } - - { package: "\"build-tools;28.0.3\"", android_test_path: 'build-tools/28.0.3' } - - { package: "\"extras;google;m2repository\"", android_test_path: 'extras/google/m2repository' } - - { package: "\"extras;android;m2repository\"", android_test_path: 'extras/android/m2repository' } - -# Andriod Virtual Device Image -android_sys_image: "\"system-images;android-28;google_apis;x86\"" - -# libraries needed for avd(android virtual device) emulation -android_apt_libraries: - - libglu1 - - libpulse0 - -# Lising files to get executable permissions -android_files: - - '/opt/android-sdk-linux/tools/bin/avdmanager' - - '/opt/android-sdk-linux/emulator/emulator' diff --git a/playbooks/roles/android_sdk/tasks/main.yml b/playbooks/roles/android_sdk/tasks/main.yml deleted file mode 100644 index 91bbe288530..00000000000 --- a/playbooks/roles/android_sdk/tasks/main.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# Configure a system for building the edx android application - -# Configure a user/group to own/run the android sdk -- name: Create group for the user of the sdk - group: name={{ android_group }} state=present -- name: Add the user to the group and configure shell - user: name={{ android_user }} append=yes group={{ android_group }} shell=/bin/bash - -# Download the Android SDK/tools tarball from Google's download site. -# NOTE: while it is the general policy to use repositories/ppas as much as possible -# it did not seem reliable for this work. Other avenues were explored: -# - pre-Ubuntu 16 releases do not contain the android sdk in their apt repos -# - the existing ppas containing the sdk are questionable -# - ubuntu-make did not seem reliable at the time of writing this -- name: Download the Android SDK - get_url: - url: "https://dl.google.com/android/repository/{{ android_download }}" - dest: /tmp/android-sdk.zip -- name: Verify checksum of Android SDK - shell: "sha1sum /tmp/android-sdk.zip" - register: sdk_checksum -- assert: - that: - "'{{ android_checksum }}' in sdk_checksum.stdout" -- name: Create Android Home directory - file: - path: "{{ android_home }}" - state: directory - owner: "{{ android_user }}" - group: "{{ android_group }}" - mode: 0775 -# Unzip Android SDK and pipe the output to dev/null because unzipping archive creates a large amount of logging which fails the job -- name: Unzip the Android archive - shell: "unzip /tmp/android-sdk.zip -d {{ android_home }} > /dev/null" -- name: Change perms - shell: "chown -R {{ android_user }}:{{ android_group }} {{ android_home }}" - -# Use the android sdk manager to install the build targets necessary for the edx mobile app -- name: Install Android API levels - shell: "echo 'y' | {{ android_home }}/tools/bin/sdkmanager {{ android_build_targets }}" - become: yes - become_user: "{{ android_user }}" - -# Create repository file needed for sdkmanaer -- name: Create needed repository file - file: - path: "~/.android/repositories.cfg" - state: touch - - become: yes - become_user: "{{ android_user }}" - -# Put Android package names into a list for easier installation command -- name: Gather Android packages to download into a list - util_map: - function: 'zip_to_list' - input: "{{ android_tools }}" - args: - - "package" - register: android_packages -# Use the android sdk manager to install the build tools necessary for the edx mobile app -- name: Install other Android tools - shell: "echo 'y' | {{ android_home }}/tools/bin/sdkmanager {{ android_packages.function_output | join(' ') }} > /dev/null" - become: yes - become_user: "{{ android_user }}" -# Link adb to tools, where all the rest of the android tools are -- name: Add symlink for adb - file: - src: "{{ android_home }}/platform-tools/adb" - dest: "{{ android_home }}/tools/adb" - state: link - owner: "{{ android_user }}" - group: "{{ android_group }}" -# The following libraries are only needed to run AVD emulation, not for compiling -- name: Install additional libraries used for Android emulation - apt: - name: "{{ item }}" - update_cache: yes - state: present - with_items: "{{ android_apt_libraries }}" -# Download Android System Image to run emulator -- name: Download Android System Image - shell: "echo 'y' | {{ android_home }}/tools/bin/sdkmanager {{ android_sys_image }} > /dev/null" - become: yes - become_user: "{{ android_user }}" - -- name: Adding executable permissions - file: - path: "{{ item }}" - mode: 0655 - with_items: "{{ android_files }}" - -- name: Install zip utility - apt: - name: "zip" - state: present - update_cache: yes \ No newline at end of file diff --git a/playbooks/roles/jenkins_build/defaults/main.yml b/playbooks/roles/jenkins_build/defaults/main.yml deleted file mode 100644 index 8eb05c54799..00000000000 --- a/playbooks/roles/jenkins_build/defaults/main.yml +++ /dev/null @@ -1,398 +0,0 @@ -build_jenkins_user_uid: 1002 -build_jenkins_group_gid: 1004 -BUILD_JENKINS_VERSION: jenkins_2.319.3 -build_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16384m -DsessionTimeout=60' - -build_jenkins_python_versions: - - python3.5 - - python3.8 - -build_jenkins_configuration_scripts: - - 1addJarsToClasspath.groovy - - 3importCredentials.groovy - - 3installGroovy.groovy - - 3mainConfiguration.groovy - - 3setGlobalProperties.groovy - - 4configureEc2Plugin.groovy - - 4configureGHOAuth.groovy - - 4configureGHPRB.groovy - - 4configureGit.groovy - - 4configureGithub.groovy - - 4configureMailerPlugin.groovy - - 4configureMaskPasswords.groovy - - 4configureSecurity.groovy - - 4configureSlack.groovy - - 4configureSplunk.groovy - - 4configureTimestamper.groovy - - 5configureEmailExtension.groovy - - 5createLoggers.groovy - -# plugins -build_jenkins_plugins_list: - - name: 'analysis-core' - version: '1.96' - group: 'org.jvnet.hudson.plugins' - - name: 'ansicolor' - version: '1.0.0' - group: 'org.jenkins-ci.plugins' - - name: 'ant' - version: '1.12' - group: 'org.jenkins-ci.plugins' - - name: 'antisamy-markup-formatter' - version: '2.1' - group: 'org.jenkins-ci.plugins' - - name: 'aws-credentials' - version: '1.28.1' - group: 'org.jenkins-ci.plugins' - - name: 'aws-java-sdk' - version: '1.12.101-300.vc09c7be9cb57' - group: 'org.jenkins-ci.plugins' - - name: 'badge' - version: '1.9' - group: 'org.jenkins-ci.plugins' - - name: 'bootstrap5-api' - version: '5.1.1-1' - group: 'io.jenkins.plugins' - - name: 'bouncycastle-api' - version: '2.25' - group: 'org.jenkins-ci.plugins' - - name: 'build-name-setter' - version: '2.1.0' - group: 'org.jenkins-ci.plugins' - - name: 'build-timeout' - version: '1.20' - group: 'org.jenkins-ci.plugins' - - name: 'build-user-vars-plugin' - version: '1.5' - group: 'org.jenkins-ci.plugins' - - name: 'checks-api' - version: '1.7.2' - group: 'io.jenkins.plugins' - - name: 'cloudbees-folder' - version: '6.16' - group: 'org.jenkins-ci.plugins' - - name: 'cobertura' - version: '1.17' - group: 'org.jenkins-ci.plugins' - - name: 'code-coverage-api' - version: '2.0.2' - group: 'io.jenkins.plugins' - - name: 'copyartifact' - version: '1.46.2' - group: 'org.jenkins-ci.plugins' - - name: 'credentials' - version: '2.6.1' - group: 'org.jenkins-ci.plugins' - - name: 'credentials-binding' - version: '1.27' - group: 'org.jenkins-ci.plugins' - - name: 'cvs' - version: '2.19' - group: 'org.jenkins-ci.plugins' - - name: 'data-tables-api' - version: '1.11.3-1' - group: 'io.jenkins.plugins' - - name: 'display-url-api' - version: '2.3.5' - group: 'org.jenkins-ci.plugins' - - name: 'docker-commons' - version: '1.14' - group: 'org.jenkins-ci.plugins' - - name: 'ec2' - version: '1.50.3' - group: 'org.jenkins-ci.plugins' - - name: 'email-ext' - version: '2.66' - group: 'org.jenkins-ci.plugins' - - name: 'envinject' - version: '2.3.0' - group: 'org.jenkins-ci.plugins' - - name: 'echarts-api' - version: '5.2.1-1' - group: 'io.jenkins.plugins' - - name: 'exclusive-execution' - version: '0.8' - group: 'org.jenkins-ci.plugins' - - name: 'external-monitor-job' - version: '1.7' - group: 'org.jenkins-ci.plugins' - - name: 'flexible-publish' - version: '0.16.1' - group: 'org.jenkins-ci.plugins' - - name: 'font-awesome-api' - version: '5.15.4-1' - group: 'io.jenkins.plugins' - - name: 'forensics-api' - version: '1.5.0' - group: 'io.jenkins.plugins' - - name: 'ghprb' - version: '1.42.2' - group: 'org.jenkins-ci.plugins' - - name: 'git' - version: '4.2.2' - group: 'org.jenkins-ci.plugins' - - name: 'git-client' - version: '3.0.0' - group: 'org.jenkins-ci.plugins' - - name: 'github' - version: '1.29.2' - group: 'com.coravy.hudson.plugins.github' - - name: 'github-api' - version: '1.133' - group: 'org.jenkins-ci.plugins' - - name: 'github-branch-source' - version: '2.9.9' - group: 'org.jenkins-ci.plugins' - - name: 'github-oauth' - version: '0.33' - group: 'org.jenkins-ci.plugins' - - name: 'gradle' - version: '1.29' - group: 'org.jenkins-ci.plugins' - - name: 'groovy' - version: '2.4' - group: 'org.jenkins-ci.plugins' - - name: 'groovy-postbuild' - version: '2.5' - group: 'org.jvnet.hudson.plugins' - - name: 'htmlpublisher' - version: '1.25' - group: 'org.jenkins-ci.plugins' - - name: 'jackson2-api' - version: '2.13.0' - group: 'org.jenkins-ci.plugins' - - name: 'javadoc' - version: '1.6' - group: 'org.jenkins-ci.plugins' - - name: 'jobConfigHistory' - version: '2.19' - group: 'org.jenkins-ci.plugins' - - name: 'jdk-tool' - version: '1.5' - group: 'org.jenkins-ci.plugins' - - name: 'job-dsl' - version: '1.77' - group: 'org.jenkins-ci.plugins' - - name: 'jquery3-api' - version: '3.6.0-2' - group: 'io.jenkins.plugins' - - name: 'junit' - version: '1.53' - group: 'org.jenkins-ci.plugins' - - name: 'ldap' - version: '1.26' - group: 'org.jenkins-ci.plugins' - - name: 'lockable-resources' - version: '2.10' - group: 'org.6wind.jenkins' - - name: 'mailer' - version: '1.23' - group: 'org.jenkins-ci.plugins' - - name: 'mask-passwords' - version: '2.13' - group: 'org.jenkins-ci.plugins' - - name: 'matrix-auth' - version: '2.6.8' - group: 'org.jenkins-ci.plugins' - - name: 'matrix-project' - version: '1.18' - group: 'org.jenkins-ci.plugins' - - name: 'maven-plugin' - version: '3.4' - group: 'org.jenkins-ci.main' - - name: 'monitoring' - version: '1.88.0' - group: 'org.jvnet.hudson.plugins' - - name: 'multiple-scms' - version: '0.6' - group: 'org.jenkins-ci.plugins' - - name: 'nodelabelparameter' - version: '1.9.2' - group: 'org.jenkins-ci.plugins' - - name: 'pam-auth' - version: '1.6.1' - group: 'org.jenkins-ci.plugins' - - name: 'parameterized-trigger' - version: '2.39' - group: 'org.jenkins-ci.plugins' - - name: 'pipeline-model-definition' - version: '1.3.4.1' - group: 'org.jenkinsci.plugins' - - name: 'pipeline-utility-steps' - version: '2.10.0' - group: 'org.jenkins-ci.plugins' - - name: 'plugin-util-api' - version: '2.5.0' - group: 'io.jenkins.plugins' - - name: 'popper2-api' - version: '2.10.2-1' - group: 'io.jenkins.plugins' - - name: 'PrioritySorter' - version: '2.9' - group: 'org.jenkins-ci.plugins' - - name: 'promoted-builds' - version: '3.10' - group: 'org.jenkins-ci.plugins' - - name: 'pyenv-pipeline' - version: '2.1.2' - group: 'org.jenkins-ci.plugins' - - name: 'rebuild' - version: '1.32' - group: 'com.sonyericsson.hudson.plugins.rebuild' - - name: 'resource-disposer' - version: '0.16' - group: 'org.jenkins-ci.plugins' - - name: 'run-condition' - version: '1.5' - group: 'org.jenkins-ci.plugins' - - name: 'script-security' - version: '1.78' - group: 'org.jenkins-ci.plugins' - - name: 'slack' - version: '2.21' - group: 'org.jenkins-ci.plugins' - - name: 'snakeyaml-api' - version: '1.29.1' - group: 'io.jenkins.plugins' - - name: 'splunk-devops' - version: '1.9.7' - group: 'com.splunk.splunkins' - - name: 'splunk-devops-extend' - version: '1.9.7' - group: 'com.splunk.splunkins' - - name: 'ssh-agent' - version: '1.22' - group: 'org.jenkins-ci.plugins' - - name: 'ssh-credentials' - version: '1.18.1' - group: 'org.jenkins-ci.plugins' - - name: 'ssh-slaves' - version: '1.28.1' - group: 'org.jenkins-ci.plugins' - - name: 'structs' - version: '1.23' - group: 'org.jenkins-ci.plugins' - - name: 'subversion' - version: '2.15.1' - group: 'org.jenkins-ci.plugins' - - name: 'text-finder' - version: '1.17' - group: 'org.jenkins-ci.plugins' - - name: 'throttle-concurrents' - version: '2.0.1' - group: 'org.jenkins-ci.plugins' - - name: 'timestamper' - version: '1.14' - group: 'org.jenkins-ci.plugins' - - name: 'token-macro' - version: '267.vcdaea6462991' - group: 'org.jenkins-ci.plugins' - - name: 'translation' - version: '1.16' - group: 'org.jenkins-ci.plugins' - - name: 'trilead-api' - version: '1.0.13' - group: 'org.jenkins-ci.plugins' - - name: 'variant' - version: '1.4' - group: 'org.jenkins-ci.plugins' - - name: 'violations' - version: '0.7.11' - group: 'org.jenkins-ci.plugins' - - name: 'warnings-ng' - version: '9.5.1' - group: 'io.jenkins.plugins' - - name: 'workflow-aggregator' - version: '2.6' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-api' - version: '2.47' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'windows-slaves' - version: '1.8' - group: 'org.jenkins-ci.plugins' - - name: 'workflow-cps' - version: '2633.v6baeedc13805' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-cps-global-lib' - version: '2.15' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-job' - version: '2.42' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-multibranch' - version: '2.24' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-scm-step' - version: '2.13' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'workflow-support' - version: '3.8' - group: 'org.jenkins-ci.plugins.workflow' - - name: 'ws-cleanup' - version: '0.39' - group: 'org.jenkins-ci.plugins' - - name: 'xunit' - version: '1.93' - group: 'org.jenkins-ci.plugins' - -# ghprb -build_jenkins_ghprb_white_list_phrase: '.*[Aa]dd\W+to\W+whitelist.*' -build_jenkins_ghprb_ok_phrase: '.*ok\W+to\W+test.*' -build_jenkins_ghprb_retest_phrase: '.*jenkins\W+run\W+all.*' -build_jenkins_ghprb_skip_phrase: '.*\[[Ss]kip\W+ci\].*' -build_jenkins_ghprb_cron_schedule: 'H/5 * * * *' - -# github -JENKINS_GITHUB_CONFIG: '' - -# ec2 -build_jenkins_instance_cap: '500' - -# seed -build_jenkins_seed_name: 'manually_seed_one_job' - -# logs -build_jenkins_log_list: - - LOG_RECORDER: 'Ghprb' - LOGGERS: - - name: 'org.jenkinsci.plugins.ghprb.GhprbPullRequest' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.GhprbRootAction' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.GhprbRepository' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.GhprbGitHub' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.Ghprb' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.GhprbTrigger' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.ghprb.GhprbBuilds' - log_level: 'ALL' - - LOG_RECORDER: 'GithubPushLogs' - LOGGERS: - - name: 'com.cloudbees.jenkins.GitHubPushTrigger' - log_level: 'ALL' - - name: 'org.jenkinsci.plugins.github.webhook.WebhookManager' - log_level: 'ALL' - - name: 'com.cloudbees.jenkins.GitHubWebHook' - log_level: 'ALL' - - name: 'hudson.plugins.git.GitSCM' - log_level: 'ALL' - -# splunk -build_jenkins_splunk_ignored_jobs: '(^((?!edx-(platform|e2e)|gather|ironwood).)*$)|.*private.*' -build_jenkins_splunk_script_type: 'inline' -build_jenkins_splunk_script_path: 'splunk/splunk.groovy' -build_jenkins_splunk_file_path: '{{ role_path }}/../jenkins_build/files/splunk/splunk.groovy' -build_jenkins_splunk_metadata: [] - -# timestamper -build_jenkins_timestamper_system_time: "''HH:mm:ss' '" -build_jenkins_timestamper_elapsed: "''HH:mm:ss.S' '" -build_jenkins_timestamper_pipelines_enabled: true - -BUILD_JENKINS_SPLUNK_ENABLED: false -build_jenkins_splunk_master_hostname: "" -build_jenkins_splunk_event_source: "" diff --git a/playbooks/roles/jenkins_build/files/splunk/splunk.groovy b/playbooks/roles/jenkins_build/files/splunk/splunk.groovy deleted file mode 100644 index a0e48ce2b83..00000000000 --- a/playbooks/roles/jenkins_build/files/splunk/splunk.groovy +++ /dev/null @@ -1,9 +0,0 @@ -//send job metadata and junit reports with page size set to 50 (each event contains max 50 test cases) -splunkins.sendTestReport(50) - - -// Send paver timing logs to Splunk -splunkins.archive("**/timing*.log", null, false, "10MB") - -// Index the codecov metrics generated by gather-codecov-metrics -splunkins.archive("**/codecov_metrics.json", null, false, "1MB") diff --git a/playbooks/roles/jenkins_build/meta/main.yml b/playbooks/roles/jenkins_build/meta/main.yml deleted file mode 100644 index 06c17ec6312..00000000000 --- a/playbooks/roles/jenkins_build/meta/main.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -dependencies: - - common - - role: jenkins_common - JENKINS_COMMON_VERSION: '{{ BUILD_JENKINS_VERSION }}' - jenkins_common_user_uid: '{{ build_jenkins_user_uid }}' - jenkins_common_group_gid: '{{ build_jenkins_group_gid }}' - jenkins_common_jvm_args: '{{ build_jenkins_jvm_args }}' - jenkins_common_configuration_scripts: '{{ build_jenkins_configuration_scripts }}' - jenkins_common_template_files: '{{ build_jenkins_template_files }}' - jenkins_common_plugins_list: '{{ build_jenkins_plugins_list }}' - jenkins_common_ghprb_white_list_phrase: '{{ build_jenkins_ghprb_white_list_phrase }}' - jenkins_common_ghprb_ok_phrase: '{{ build_jenkins_ghprb_ok_phrase }}' - jenkins_common_ghprb_retest_phrase: '{{ build_jenkins_ghprb_retest_phrase }}' - jenkins_common_ghprb_skip_phrase: '{{ build_jenkins_ghprb_skip_phrase }}' - jenkins_common_ghprb_cron_schedule: '{{ build_jenkins_ghprb_cron_schedule }}' - jenkins_common_github_configs: '{{ JENKINS_GITHUB_CONFIG }}' - jenkins_common_instance_cap: '{{ build_jenkins_instance_cap }}' - jenkins_common_seed_name: '{{ build_jenkins_seed_name }}' - jenkins_common_log_list: '{{ build_jenkins_log_list }}' - jenkins_common_server_name: '{{ JENKINS_SERVER_NAME }}' - jenkins_common_splunk_ignored_jobs: '{{ build_jenkins_splunk_ignored_jobs }}' - jenkins_common_splunk_script_type: '{{ build_jenkins_splunk_script_type }}' - jenkins_common_splunk_script_path: '{{ build_jenkins_splunk_script_path }}' - jenkins_common_splunk_metadata: '{{ build_jenkins_splunk_metadata }}' - jenkins_common_splunk_master_hostname: '{{ build_jenkins_splunk_master_hostname }}' - jenkins_common_splunk_event_source: '{{ build_jenkins_splunk_event_source }}' - jenkins_common_splunk_enabled: '{{ BUILD_JENKINS_SPLUNK_ENABLED }}' - jenkins_common_splunk_file_path: '{{ build_jenkins_splunk_file_path }}' - jenkins_common_email_replyto: '{{ JENKINS_MAILER_REPLY_TO_ADDRESS }}' - jenkins_common_python_versions: '{{ build_jenkins_python_versions }}' - jenkins_common_timestamper_system_clock_format: '{{ build_jenkins_timestamper_system_time }}' - jenkins_common_timestamper_elapsed_time_format: '{{ build_jenkins_timestamper_elapsed }}' - jenkins_common_timestamper_enabled_on_pipelines: '{{ build_jenkins_timestamper_pipelines_enabled }}' diff --git a/playbooks/roles/jenkins_worker/defaults/main.yml b/playbooks/roles/jenkins_worker/defaults/main.yml deleted file mode 100644 index 1d950c4f4c7..00000000000 --- a/playbooks/roles/jenkins_worker/defaults/main.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -jenkins_user: "jenkins" -# the main group for the jenkins user -jenkins_group: "jenkins" -# the groups that the jenkins user must belong to -jenkins_groups: "jenkins,docker" -jenkins_home: /home/jenkins - -JENKINS_EDX_PLATFORM_VERSION: master - -# System packages -jenkins_debian_pkgs: - - pkg-config - - libffi-dev - - python3.8-dev - - libsqlite3-dev - - libfreetype6-dev - - libpq-dev - -# packer direct download URL -packer_url: "https://releases.hashicorp.com/packer/1.4.4/packer_1.4.4_linux_amd64.zip" - -JENKINS_NODE_VERSION: "12" -ansible_distribution_release: "xenial" - -# Flag which will override the '2.7' entry below when false. -jenkins_worker_install_python27: false - -jenkins_worker_python_versions: - - 3.8 - -# The packaging for Python 3.7 and above split distutils out into a separate package; -# needed for virtualenv creation -jenkins_worker_distutils_versions: - - 3.8 - -edx_platform_python_versions: - - 3.8 diff --git a/playbooks/roles/jenkins_worker/meta/main.yml b/playbooks/roles/jenkins_worker/meta/main.yml deleted file mode 100644 index 84ede1ee097..00000000000 --- a/playbooks/roles/jenkins_worker/meta/main.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -dependencies: - - common - - role: jscover - when: platform_worker is defined - - role: oraclejdk - - # dependencies for edx-app jenkins worker: - - role: edxapp_common - when: platform_worker is defined - - # dependencies for android worker - - role: android_sdk - when: android_worker is defined - # User/group to manage Android SDK - android_user: "android" - android_group: "android" - # Tarball to download - # old path "android-sdk_r24.4.1-linux.tgz" replaced - android_download: "sdk-tools-linux-4333796.zip" - - # Checksum of Android SDK (from: https://developer.android.com/studio/index.html#downloads) - # Old Checksum "725bb360f0f7d04eaccff5a2d57abdd49061326d" - replaced - android_checksum: "8c7c28554a32318461802c1291d76fccfafde054" - - # path to installed android sdk - android_home: "/opt/android-sdk-linux" - - # The SDK version used to compile the project | 6 | Android SDK Platform 28 - android_build_targets: "\"platforms;android-28\"" - - # other android dependencies that cannot be tested via the android sdk manager. instead, stat the android_test_path to test for presence of the package - # Plateform Tools | 6 | Android SDK Platform 28 - # The BuildTools | Android SDK Build-Tools 28.0.3 - # Additional components - # extras;google;m2repository | 58 | Google Repository - # extras;android;m2repository| 47.0.0 | Android Support Repository - android_tools: - - { package: "\"platform-tools\"", android_test_path: 'platform-tools' } - - { package: "\"build-tools;28.0.3\"", android_test_path: 'build-tools/28.0.3' } - - { package: "\"extras;google;m2repository\"", android_test_path: 'extras/google/m2repository' } - - { package: "\"extras;android;m2repository\"", android_test_path: 'extras/android/m2repository' } - - # dependencies for loadtest driver worker - - role: loadtest_driver - when: loadtest_driver_worker is defined diff --git a/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml b/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml deleted file mode 100644 index c5f8ba0da6a..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/loadtest_driver_worker.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Create a virtualenv for edx-load-tests by installing the requirements and -# packaging the virtualenv. - -- name: Create shallow clone of edx-load-tests - git: - repo: https://github.com/edx-unsupported/edx-load-tests.git - dest: "{{ jenkins_home }}/shallow-clone" - version: "master" - depth: 1 - become_user: "{{ jenkins_user }}" - -- name: Install most edx-load-tests requirements using pip - pip: - requirements: "{{ jenkins_home }}/shallow-clone/requirements/{{ item }}" - extra_args: "--exists-action=w" - virtualenv: "{{ jenkins_home }}/edx-venv" - virtualenv_command: virtualenv - with_items: - - base.txt - become_user: "{{ jenkins_user }}" - -# Archive the current state of the virtualenv as a starting point for new -# builds. The edx-venv directory is deleted and then recreated cleanly from -# the archive by the jenkins build scripts. -# -# TODO: after we migrate to ansible 2.3+, change this task to use the archive -# module. http://docs.ansible.com/ansible/archive_module.html -- name: Create a clean virtualenv archive - command: "tar -cpzf edx-venv_clean.tar.gz edx-venv" - args: - chdir: "{{ jenkins_home }}" - become_user: "{{ jenkins_user }}" - -- name: Remove the shallow-clone - file: path={{ jenkins_home }}/shallow-clone state=absent diff --git a/playbooks/roles/jenkins_worker/tasks/main.yml b/playbooks/roles/jenkins_worker/tasks/main.yml deleted file mode 100644 index 76a94f9feb4..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# jenkins -# -# Provision a Jenkins worker instance. -# - When platform_worker is set, the resulting instance can run edx-platform tests - -# All jenkins workers -- include: packer.yml -- include: system.yml -- include: python.yml -- include: node.yml - -# only platform workers -- include: python_platform_worker.yml - when: platform_worker is defined - -# only loadtest driver workers -- include: loadtest_driver_worker.yml - when: loadtest_driver_worker is defined - -# Run appropriate tests -- include: test.yml -- include: test_platform_worker.yml - when: platform_worker is defined -- include: test_android_worker.yml - when: android_worker is defined diff --git a/playbooks/roles/jenkins_worker/tasks/node.yml b/playbooks/roles/jenkins_worker/tasks/node.yml deleted file mode 100644 index 53571e5f0db..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/node.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Install nodejs - -- name: Install the gpg key for nodejs LTS - apt_key: - url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" - state: present -- name: Install the nodejs LTS repos - apt_repository: - repo: "deb https://deb.nodesource.com/node_{{ JENKINS_NODE_VERSION }}.x {{ ansible_distribution_release }} main" - state: present - update_cache: yes -- name: Install the nodejs - apt: - name: nodejs - state: present diff --git a/playbooks/roles/jenkins_worker/tasks/packer.yml b/playbooks/roles/jenkins_worker/tasks/packer.yml deleted file mode 100644 index 40c91ed9ed5..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/packer.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Download packer - shell: "curl -L {{ packer_url }} -o /var/tmp/packer.zip" - args: - creates: /var/tmp/packer.zip - -- name: Unzip packer - unarchive: src=/var/tmp/packer.zip dest=/usr/local/bin copy=no diff --git a/playbooks/roles/jenkins_worker/tasks/python.yml b/playbooks/roles/jenkins_worker/tasks/python.yml deleted file mode 100644 index a48d40e85e8..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/python.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- -# Versions of Python newer than 3.5 are not available in the default -# package index for Ubuntu 16.04. Add the deadsnakes PPA for anything -# newer -- name: add deadsnakes PPA for newer Python versions - apt_repository: - repo: "ppa:deadsnakes/ppa" - update_cache: yes - when: ansible_distribution_release == 'xenial' - -# Install newer versions of python for testing, but do not set them -# as the default version -- name: Install python versions - apt: - name: 'python{{ item }}' - state: present - update_cache: yes - with_items: '{{ jenkins_worker_python_versions }}' - -# Install 'dev' packages for each version of python that is installed -- name: Install python dev packages - apt: - name: 'python{{ item }}-dev' - state: present - update_cache: yes - with_items: '{{ jenkins_worker_python_versions }}' - -# Install 'distutils' packages for each installed version of python which has one -- name: Install python distutils packages - apt: - name: 'python{{ item }}-distutils' - state: present - update_cache: yes - with_items: '{{ jenkins_worker_distutils_versions }}' - -# For Python version 3.5, install pip via Ubuntu apt-get, as get-pip.py can no longer be used. -# get-pip.py also doesn't support Python2.7 - that version's install is broken at the moment. -- name: Install distro pip for Python3 to accomodate Python 3.5. - apt: - name: python3-pip - state: present - update_cache: yes - -- name: Upgrade pip for installed python version 3.5. - shell: - cmd: "python{{ item }} -m pip install --upgrade pip==20.3.4" - when: item == 3.5 - with_items: '{{ jenkins_worker_python_versions }}' - register: python_versions - -- name: Fetch get-pip.py from pypa - get_url: - url: "https://bootstrap.pypa.io/get-pip.py" - dest: "/tmp/get-pip.py" - -- name: Install most recent 'pip' for installed python versions 3.6 and greater. - shell: - cmd: "python{{ item.item }} /tmp/get-pip.py" - with_items: '{{ python_versions.results }}' - when: - - item | skipped - - item.item != 2.7 or (item.item == 2.7 and jenkins_worker_install_python27) - -# Requests library is required for the github status script. -- name: Install requests Python library - pip: - name: "{{ common_pip_pkgs + ['requests'] }}" - state: present - executable: 'pip{{ item }}' - with_items: "{{ jenkins_worker_python_versions }}" - when: - - item != 2.7 or (item == 2.7 and jenkins_worker_install_python27) - diff --git a/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml b/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml deleted file mode 100644 index 35c03fd1047..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/python_platform_worker.yml +++ /dev/null @@ -1,90 +0,0 @@ ---- - -# Create a virtualenv for edx-platform by installing the requirements -# and packaging the virtualenv. -# A shallow clone is created off of master. The depth setting -# refers to the --depth-setting of git clone. A value of 1 -# will truncate all history prior to the last revision. -- name: Create shallow clone of edx-platform - git: - repo: https://github.com/openedx/edx-platform.git - dest: "{{ jenkins_home }}/shallow-clone" - version: "{{ JENKINS_EDX_PLATFORM_VERSION }}" - depth: 1 - become_user: "{{ jenkins_user }}" - -# In order to create multiple virtualenvs with the same name, -# put them into separate directories -- name: Create directories for virtualenvs to avoid naming collisions - file: - path: "{{ jenkins_home }}/edx-venv-{{ item }}" - state: directory - with_items: "{{ jenkins_worker_python_versions }}" - become_user: "{{ jenkins_user }}" - -# Create the virtualenvs and install the correct pip version in each one. -- name: Create virtualenvs and install desired pip version - pip: - chdir: "{{ jenkins_home }}/shallow-clone" - name: "{{ common_pip_pkgs }}" - virtualenv: "{{ jenkins_home }}/edx-venv-{{ item }}/edx-venv" - virtualenv_command: virtualenv - virtualenv_python: "python{{ item }}" - with_items: "{{ edx_platform_python_versions }}" - become_user: "{{ jenkins_user }}" - -# Combine testing and django requirements files for single virtualenv invocation -- name: Combine requirements files - shell: "cat {{ jenkins_home }}/shallow-clone/requirements/edx/testing.txt {{ jenkins_home }}/shallow-clone/requirements/edx/django.txt > {{ jenkins_home }}/shallow-clone/requirements/edx/jenkins.txt" - become_user: "{{ jenkins_user }}" - -# Install the platform requirements using pip. -- name: Install edx-platform requirements using pip - pip: - chdir: "{{ jenkins_home }}/shallow-clone" - requirements: "{{ jenkins_home }}/shallow-clone/requirements/edx/jenkins.txt" - extra_args: "--exists-action=w" - virtualenv: "{{ jenkins_home }}/edx-venv-{{ item }}/edx-venv" - with_items: "{{ edx_platform_python_versions }}" - become_user: "{{ jenkins_user }}" - -# Archive the current state of each of the virtualenvs -# as a starting point for new builds. -# The edx-venv-x directory is deleted and then recreated -# cleanly from the archive by the jenkins build scripts. -- name: Create a clean virtualenv archive - command: "tar -C edx-venv-{{ item }} -cpzf edx-venv_clean-{{ item }}.tar.gz edx-venv" - args: - chdir: "{{ jenkins_home }}" - with_items: "{{ edx_platform_python_versions }}" - become_user: "{{ jenkins_user }}" - -- name: Add script to set install node packages - template: - src: jenkins_nodeenv.j2 - dest: "{{ jenkins_home }}/jenkins_nodeenv" - owner: "{{ jenkins_user }}" - group: "{{ jenkins_group }}" - mode: "0500" - when: platform_worker is defined - -# Install node packages for platform. -- name: Install node packages for caching - command: "bash {{ jenkins_home }}/jenkins_nodeenv" - become_user: "{{ jenkins_user }}" - -# Archive the current state of the npm cache as a starting -# point for new builds. Most builds don't introduce package -# changes, so npm won't need to fetch anything from a remote -# registry. This should reduce the frequency of hanging -# npm installs (see https://openedx.atlassian.net/browse/TE-2732) -- name: Create a clean npm cache archive - command: "tar -cpzf edx-npm-cache_clean.tar.gz .npm" - args: - chdir: "{{ jenkins_home }}" - become_user: "{{ jenkins_user }}" - -# Remove the shallow-clone directory now that we are -# done with it -- name: Remove shallow-clone - file: path={{ jenkins_home }}/shallow-clone state=absent diff --git a/playbooks/roles/jenkins_worker/tasks/system.yml b/playbooks/roles/jenkins_worker/tasks/system.yml deleted file mode 100644 index 7a1a375a7a9..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/system.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -- name: Create jenkins group - group: name={{ jenkins_group }} state=present - -# The Jenkins account needs a login shell because Jenkins uses scp -- name: Add the jenkins user to the group and configure shell - user: - name: '{{ jenkins_user }}' - groups: '{{ jenkins_groups }}' - append: yes - shell: /bin/bash - -# Because of a bug in the latest release of the EC2 plugin -# we need to use a key generated by Amazon (not imported) -# To satisfy this, we allow users to log in as Jenkins -# using the same keypair the instance was started with. -- name: Create .ssh directory - file: - path={{ jenkins_home }}/.ssh state=directory - owner={{ jenkins_user }} group={{ jenkins_group }} - ignore_errors: yes - -- name: Get the authorized key that should be used for this machine. - shell: "curl {{ jenkins_worker_key_url }} -o {{ jenkins_home }}/.ssh/authorized_keys" - when: jenkins_worker_key_url is defined - -- name: Set key permissions - file: - path={{ jenkins_home }}/.ssh/authorized_keys - owner={{ jenkins_user }} group={{ jenkins_group }} mode=400 - -- name: Install system packages - apt: pkg={{','.join(jenkins_debian_pkgs)}} - state=present update_cache=yes - -- name: Add script to set up environment variables - template: - src=jenkins_env.j2 dest={{ jenkins_home }}/jenkins_env - owner={{ jenkins_user }} group={{ jenkins_group }} mode=0500 - when: platform_worker is defined - -# Need to add Github to known_hosts to avoid -# being prompted when using git through ssh -- name: Add github.com to known_hosts if it does not exist - shell: "ssh-keygen -f {{ jenkins_home }}/.ssh/known_hosts -H -F github.com | grep -q found || ssh-keyscan -H github.com > {{ jenkins_home }}/.ssh/known_hosts" - -# Edit the /etc/hosts file so that the Preview button will work in Studio -- name: add preview.localhost to /etc/hosts - shell: sed -i -r 's/^127.0.0.1\s+.*$/127.0.0.1 localhost preview.localhost/' /etc/hosts - become: yes diff --git a/playbooks/roles/jenkins_worker/tasks/test.yml b/playbooks/roles/jenkins_worker/tasks/test.yml deleted file mode 100644 index f67cdb5f517..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/test.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Tests for this role - - -### Tests ### -- name: Verify java cmd is using v 1.8 - shell: java -version - register: java_version -- assert: - that: - - "'1.8.0' in java_version.stderr" diff --git a/playbooks/roles/jenkins_worker/tasks/test_android_worker.yml b/playbooks/roles/jenkins_worker/tasks/test_android_worker.yml deleted file mode 100644 index db378bfaf60..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/test_android_worker.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# Verify Android Platform -- name: Verify that Android Platform is installed - shell: "stat {{ android_home }}/platforms/android-28" - register: android_plateform -- assert: - that: - - "android_plateform.rc == 0" - -# Verify Android Platform Tools -- name: Verify that Android Platform Tools are installed - shell: "stat {{ android_home }}/platform-tools" - register: android_plateform_tools -- assert: - that: - - "android_plateform_tools.rc == 0" - -# Verify Android Build Tools -- name: Verify that Android Build Tools are installed - shell: "stat {{ android_home }}/build-tools" - register: android_build_tools -- assert: - that: - - "android_build_tools.rc == 0" - -# Verify Android Google Repository -- name: Verify that Android Google Repository is installed - shell: "stat {{ android_home }}/extras/google/m2repository" - register: android_google_repo -- assert: - that: - - "android_google_repo.rc == 0" - -# Verify Android Repository -- name: Verify that Android Repository is installed - shell: "stat {{ android_home }}/extras/android/m2repository" - register: android_repo -- assert: - that: - - "android_repo.rc == 0" - -# Verify system image -- name: Verify that Android sys image is installed - shell: "stat {{ android_home }}/system-images/android-28" - register: android_27_image -- assert: - that: - - "android_27_image.rc == 0" - -# Verify Android tool -- name: Verify Android tool is installed - shell: "stat {{ android_home }}/tools/android" - register: android_version -- assert: - that: - - "android_version.rc == 0" - -# Verify Emulator tool -- name: Verify Emulator tool is installed - shell: "stat {{ android_home }}/tools/emulator" - register: emulator_version -- assert: - that: - - "emulator_version.rc == 0" - -# Verify AVD Manager tool -- name: Verify AVD Manager tool is installed - shell: "stat {{ android_home }}/tools/bin/avdmanager" - register: avdmanager_version -- assert: - that: - - "avdmanager_version.rc == 0" - -# Verify SDK Manager tool -- name: Verify SDK Manager tool is installed - shell: "stat {{ android_home }}/tools/bin/sdkmanager" - register: sdkmanager_version -- assert: - that: - - "sdkmanager_version.rc == 0" - -# Verify adb tool -- name: Verify adb tool is installed - shell: "stat {{ android_home }}/platform-tools/adb" - register: adb_version -- assert: - that: - - "adb_version.rc == 0" \ No newline at end of file diff --git a/playbooks/roles/jenkins_worker/tasks/test_codejail_worker.yml b/playbooks/roles/jenkins_worker/tasks/test_codejail_worker.yml deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml b/playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml deleted file mode 100644 index bba982fb2c1..00000000000 --- a/playbooks/roles/jenkins_worker/tasks/test_platform_worker.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -# Tests for this role - - -# Set up # - -# To get a baseline comparison for timestamp comparisons -# create a testfile and register its stat info -- name: Create test file - file: path=testfile state=touch -- name: Stat test file - stat: path=testfile - register: testfile - - -### Tests ### - -# Firefox has a specific version, not the latest. This test also ensures it was not -# pulled in via dependency or misuse/clobbering. -- name: Verify firefox version - shell: firefox --version - register: FIREFOX_VERSION -- assert: - that: - - "'61' in FIREFOX_VERSION.stdout" - -# Verify the virtualenv tar is newly-built -- name: Get info on virtualenv tar - stat: path={{ jenkins_home }}/edx-venv_clean-3.8.tar.gz - register: edxvenv -- assert: - that: -# Assert that it was modified at least within the hour - - "{{ testfile.stat.mtime }} - {{ edxvenv.stat.mtime }} < 3600" - -# Verify that postfix (mail server) is listening -- wait_for: host={{ inventory_hostname }} port=25 - delegate_to: localhost - -# Tear Down # -- name: Remove test file - file: path=testfile state=absent diff --git a/playbooks/roles/jenkins_worker/templates/jenkins_env.j2 b/playbooks/roles/jenkins_worker/templates/jenkins_env.j2 deleted file mode 100644 index bdadfed50be..00000000000 --- a/playbooks/roles/jenkins_worker/templates/jenkins_env.j2 +++ /dev/null @@ -1,5 +0,0 @@ -# Configure JavaScript coverage -export JSCOVER_JAR=/usr/local/bin/JSCover-all-{{ JSCOVER_VERSION }}.jar - -# Set the display to the virtual frame buffer (Xvfb) -export DISPLAY=:1 diff --git a/playbooks/roles/jenkins_worker/templates/jenkins_nodeenv.j2 b/playbooks/roles/jenkins_worker/templates/jenkins_nodeenv.j2 deleted file mode 100644 index ceaa116a96d..00000000000 --- a/playbooks/roles/jenkins_worker/templates/jenkins_nodeenv.j2 +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Install the version of Node used in the platform tests on Jenkins -# and then install the Node packages specified in the platform. These -# installs will be cached and used as a baseline for installations -# in future test runs. -cd {{ jenkins_home }}/shallow-clone -npm install diff --git a/playbooks/roles/loadtest_driver/defaults/main.yml b/playbooks/roles/loadtest_driver/defaults/main.yml deleted file mode 100644 index 5a9fb5510ac..00000000000 --- a/playbooks/roles/loadtest_driver/defaults/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- - -# ulimit variables -ulimit_config: - - domain: '*' - type: soft - item: nofile - value: 4096 - - domain: '*' - type: hard - item: nofile - value: 4096 - -ulimit_conf_file: "/etc/security/limits.conf" diff --git a/playbooks/roles/loadtest_driver/meta/main.yml b/playbooks/roles/loadtest_driver/meta/main.yml deleted file mode 100644 index d11985cdb9a..00000000000 --- a/playbooks/roles/loadtest_driver/meta/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for role loadtest_driver -# - -dependencies: - - common diff --git a/playbooks/roles/loadtest_driver/tasks/main.yml b/playbooks/roles/loadtest_driver/tasks/main.yml deleted file mode 100644 index be0e0eb2592..00000000000 --- a/playbooks/roles/loadtest_driver/tasks/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# configure a machine for driving loadtests. - -# Specifically, we want to allow as many open connections as possible, to -# simulate more locust clients. -- name: Increase file descriptor limit of the system - lineinfile: - dest: "{{ ulimit_conf_file }}" - line: "{{ item.domain }} {{ item.type }} {{ item.item }} {{ item.value }}" - with_items: "{{ ulimit_config }}" - diff --git a/playbooks/roles/locust/defaults/main.yml b/playbooks/roles/locust/defaults/main.yml deleted file mode 100644 index a2c16efe7bc..00000000000 --- a/playbooks/roles/locust/defaults/main.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for role locust -# - -# -# vars are namespace with the module name. -# -locust_service_name: "locust" -locust_home: "{{ COMMON_APP_DIR }}/{{ locust_service_name }}" -locust_user: "locust" -locust_code_dir: "{{ locust_home }}/edx-load-tests" -locust_requirements_base: "{{ locust_code_dir }}/requirements" - -LOCUST_GIT_IDENTITY: !!null - -LOCUST_LOADTEST_REPO: 'edx-load-tests' -LOCUST_LOADTEST_REPO_VERSION: 'master' - -LOCUST_SERVICE_CONFIG: '' - -LOCUST_REPOS: - - PROTOCOL: "ssh" - DOMAIN: "{{ COMMON_GIT_MIRROR }}" - PATH: "{{ COMMON_GIT_PATH }}" - REPO: "{{ LOCUST_LOADTEST_REPO }}" - VERSION: "{{ LOCUST_LOADTEST_REPO_VERSION }}" - DESTINATION: "{{ locust_code_dir }}" - SSH_KEY: "{{ LOCUST_GIT_IDENTITY }}" -# -# OS packages -# - -locust_debian_pkgs: - # Needed by our fork of locust :-( because it needs matplotlib. - - libfreetype6-dev - - libpng12-dev - # Needed for opaque-keys - - liblapack-dev - - gfortran - -locust_redhat_pkgs: [] - -# ulimit variables -ulimit_config: - - domain: '*' - type: soft - item: nofile - value: 4096 - - domain: '*' - type: hard - item: nofile - value: 4096 - -ulimit_conf_file: "/etc/security/limits.conf" diff --git a/playbooks/roles/locust/meta/main.yml b/playbooks/roles/locust/meta/main.yml deleted file mode 100644 index 499eda3a3e1..00000000000 --- a/playbooks/roles/locust/meta/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for role locust -# -dependencies: - - common - - loadtest_driver - - role: edx_service_with_rendered_config - edx_service_with_rendered_config_service_name: "{{ locust_service_name }}" - edx_service_with_rendered_config_service_config: "{{ LOCUST_SERVICE_CONFIG }}" - edx_service_with_rendered_config_repos: "{{ LOCUST_REPOS }}" - edx_service_with_rendered_config_user: "{{ locust_user }}" - edx_service_with_rendered_config_home: "{{ locust_home }}" - edx_service_with_rendered_config_packages: - debian: "{{ locust_debian_pkgs }}" - redhat: "{{ locust_redhat_pkgs }}" - diff --git a/playbooks/roles/locust/tasks/main.yml b/playbooks/roles/locust/tasks/main.yml deleted file mode 100644 index 1517f65014d..00000000000 --- a/playbooks/roles/locust/tasks/main.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# -# -# Tasks for role locust -# -# Overview: -# - Expects an edx-load-tests repo that declares locust.io as a requirement -# - Installs edx-load-tests repo using standard edx service roles -# - Makes the locust user suitable for running an interactive shell -# - Adds a MOTD message for self-help resources related to loadtesting -# -# Dependencies: -# - edx-service role -# - loadtest_driver role -# - load tests repo with locust tests in it. -# -# Example play: -# -# # To run: ansible-playbook locust.yml -i "locustdriver.sandbox.edx.org," -e "@/Users/derf/workspace/sandbox-secure/ansible/vars/developer-sandbox.yml" -# - name: Deploy Locust -# hosts: all -# become: True -# gather_facts: True -# roles: -# - locust - -- name: Install base requirements used by all load tests - pip: - requirements: "{{ locust_requirements_base }}/base.txt" - virtualenv: "{{ locust_home }}/venvs/{{ locust_service_name }}" - state: present - become_user: "{{ locust_user }}" - -- name: Configure locust user with an interactive shell - user: - name: "{{ locust_user }}" - shell: /bin/bash - -# This is a hack because we're not using .bash_aliases for its stated purpose. -# As of this writing, the script will activate a preinstalled virtualenv and -# change to the edx-load-tests directory. -- name: Hack .bash_aliases to make it more convenient to start loadtests - template: - src: locust_home/bash_aliases.j2 - dest: "{{ locust_home }}/.bash_aliases" - owner: "{{ locust_user }}" - group: root - mode: '644' - -- name: Setup a loadtest-specific MOTD - template: - src: etc/motd.tail.locust.j2 - dest: /etc/motd.tail.locust - owner: root - group: root - mode: '644' - # the directory /etc/update-motd.d is a xenial-ism only - when: ansible_distribution_release == 'xenial' - -- name: Add motd.tail.locust to update-motd.d - copy: - dest: "/etc/update-motd.d/76-motd-tail-locust" - content: "#!/bin/sh\necho\ncat /etc/motd.tail.locust\n" - force: true - owner: root - group: root - mode: "0755" - # the directory /etc/update-motd.d is a xenial-ism only - when: ansible_distribution_release == 'xenial' diff --git a/playbooks/roles/locust/templates/etc/motd.tail.locust.j2 b/playbooks/roles/locust/templates/etc/motd.tail.locust.j2 deleted file mode 100644 index 106eb17ce89..00000000000 --- a/playbooks/roles/locust/templates/etc/motd.tail.locust.j2 +++ /dev/null @@ -1,14 +0,0 @@ -******************************************************************* -* Message regarding loadtests * -* * -* Start your loadtesting journey by switching to the locust user: * -* * -* sudo su locust * -* * -* Please consult the loadtest environment queue before running * -* load tests against our shared loadtest environment: * -* https://openedx.atlassian.net/wiki/x/B4M3AQ * -* * -* For troubleshooting or other help, see our documentation: * -* https://openedx.atlassian.net/wiki/x/-QEsAQ * -******************************************************************* diff --git a/playbooks/roles/locust/templates/locust_home/bash_aliases.j2 b/playbooks/roles/locust/templates/locust_home/bash_aliases.j2 deleted file mode 100644 index 2faa4d9737d..00000000000 --- a/playbooks/roles/locust/templates/locust_home/bash_aliases.j2 +++ /dev/null @@ -1,2 +0,0 @@ -cd {{ locust_code_dir }} -source {{ locust_home }}/venvs/{{ locust_service_name }}/bin/activate diff --git a/playbooks/roles/rbenv/tasks/main.yml b/playbooks/roles/rbenv/tasks/main.yml index da9a8f98765..23589323f96 100644 --- a/playbooks/roles/rbenv/tasks/main.yml +++ b/playbooks/roles/rbenv/tasks/main.yml @@ -181,7 +181,6 @@ - name: remove rbenv version of rake file: path="{{ rbenv_dir }}/.rbenv/versions/{{ rbenv_ruby_version }}/bin/rake" state=absent - when: jenkins_worker is not defined or not jenkins_worker tags: - install - install:base @@ -190,7 +189,6 @@ shell: "gem install rake -v {{ RBENV_RAKE_VERSION }}" become_user: "{{ rbenv_user }}" environment: "{{ rbenv_environment }}" - when: jenkins_worker is not defined or not jenkins_worker tags: - install - install:base diff --git a/util/maintenance.sh b/util/maintenance.sh index ffe40a1bef2..097cd1c230d 100755 --- a/util/maintenance.sh +++ b/util/maintenance.sh @@ -13,10 +13,10 @@ ED=$1 ENABLE_ARG=$2 case $ED in - loadtest-edx|stage-edx|prod-edx|prod-edge) + stage-edx|prod-edx|prod-edge) ;; *) - echo "ERROR: environment-deploy must be one of loadtest-edx, stage-edx, prod-edx or prod-edge" + echo "ERROR: environment-deploy must be one of stage-edx, prod-edx or prod-edge" echo usage ;; diff --git a/util/packer/README.rst b/util/packer/README.rst deleted file mode 100644 index 496ca4f0a7f..00000000000 --- a/util/packer/README.rst +++ /dev/null @@ -1,21 +0,0 @@ -Packer -======= - -``jenkins_worker.json`` is the `packer configuration template`_ that tells packer how to build the image. - -- `template variable`_ pattern ``"foo": "{{env `BAR`}}"`` - - - What this does is take the value of the environment variable BAR which was set in the shell that kicks off the ``packer build jenkins_worker.json`` command (for example in a jenkins job) and pass it through to the user variable "foo". - - This the user variable "foo" will now be available globally within the template. - - If the environment variable is not set in the shell that kicks off the packer build command, the user variable value will be the empty string. - -- Regarding the `ansible-playbook command`_ that is used to run the jenkins_worker role's -e (--extra-vars) option - - - `playbook variable`_ pattern ``-e 'bar={{ user `foo` }}'`` - - Packer has determined the value of the template user variable "foo" from the local environment variable (see above) - - Ansible will use this as the value for the playbook variable "bar" when running the play. - -.. _packer configuration template: http://www.packer.io/docs/templates/introduction.html -.. _template variable: http://www.packer.io/docs/templates/user-variables.html -.. _ansible-playbook command: http://docs.ansible.com/playbooks_intro.html#executing-a-playbook -.. _playbook variable: http://docs.ansible.com/playbooks_variables.html#passing-variables-on-the-command-line diff --git a/util/packer/jenkins_worker.json b/util/packer/jenkins_worker.json deleted file mode 100644 index 5735d77254e..00000000000 --- a/util/packer/jenkins_worker.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "new_relic_infrastructure_license_key": "{{env `NEWRELIC_INFRASTRUCTURE_LICENSE_KEY`}}", - "playbook_remote_dir": "/tmp/packer-edx-playbooks", - "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible", - "ami": "{{env `JENKINS_WORKER_AMI`}}", - "test_platform_version": "{{env `TEST_PLATFORM_VERSION`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}", - "remote_branch": "{{env `REMOTE_BRANCH`}}", - "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}", - "instance_type": "m3.large", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "jenkins worker", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}", - "tags": { - "delete_or_keep": "{{user `delete_or_keep`}}" - }, - "launch_block_device_mappings": [{ - "delete_on_termination": true, - "device_name": "/dev/sda1", - "volume_size": "40", - "volume_type": "gp2" - }] - }], - "provisioners": [{ - "type": "shell", - "inline": ["rm -rf {{user `playbook_remote_dir`}}", - "mkdir {{user `playbook_remote_dir`}}"] - }, { - "type": "file", - "source": "stop-automatic-updates.sh", - "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh" - }, { - "type": "file", - "source": "../../util/install/ansible-bootstrap.sh", - "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh" - }, { - "type": "shell", - "inline": ["cd {{user `playbook_remote_dir`}}", - "export CONFIGURATION_VERSION='{{user `remote_branch`}}'", - "sudo bash ./stop-automatic-updates.sh", - "sudo bash ./ansible-bootstrap.sh" - ] - }, { - "type": "shell-local", - "command": "rm ../../playbooks/edx-east" - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/jenkins_worker.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"JENKINS_EDX_PLATFORM_VERSION={{user `test_platform_version`}} NEWRELIC_INFRASTRUCTURE_LICENSE_KEY={{user `new_relic_infrastructure_license_key`}} initialize_replica_set=false mongo_configure_replica_set=false jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}' ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/run_role.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"role=test_build_server TEST_EDX_PLATFORM_VERSION={{user `test_platform_version`}} ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }] -} diff --git a/util/packer/jenkins_worker_android.json b/util/packer/jenkins_worker_android.json deleted file mode 100644 index 3073542c12f..00000000000 --- a/util/packer/jenkins_worker_android.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "new_relic_infrastructure_license_key": "{{env `NEWRELIC_INFRASTRUCTURE_LICENSE_KEY`}}", - "playbook_remote_dir": "/tmp/packer-edx-playbooks", - "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible", - "ami": "{{env `JENKINS_WORKER_AMI`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}", - "remote_branch": "{{env `REMOTE_BRANCH`}}", - "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "jenkins_worker_android-{{isotime | clean_ami_name}}", - "instance_type": "m3.large", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "jenkins worker android", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}", - "tags": { - "delete_or_keep": "{{user `delete_or_keep`}}" - }, - "launch_block_device_mappings": [{ - "delete_on_termination": true, - "device_name": "/dev/sda1", - "volume_size": "40", - "volume_type": "gp2" - }] - }], - "provisioners": [{ - "type": "shell", - "inline": ["rm -rf {{user `playbook_remote_dir`}}", - "mkdir {{user `playbook_remote_dir`}}"] - }, { - "type": "file", - "source": "stop-automatic-updates.sh", - "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh" - }, { - "type": "file", - "source": "../../util/install/ansible-bootstrap.sh", - "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh" - }, { - "type": "shell", - "inline": ["cd {{user `playbook_remote_dir`}}", - "export CONFIGURATION_VERSION='{{user `remote_branch`}}'", - "sudo bash ./stop-automatic-updates.sh", - "sudo bash ./ansible-bootstrap.sh" - ] - }, { - "type": "shell-local", - "command": "rm ../../playbooks/edx-east" - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/jenkins_worker_android.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"NEWRELIC_INFRASTRUCTURE_LICENSE_KEY={{user `new_relic_infrastructure_license_key`}} jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}' ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }] -} diff --git a/util/packer/jenkins_worker_codejail.json b/util/packer/jenkins_worker_codejail.json deleted file mode 100644 index 269bcfd3809..00000000000 --- a/util/packer/jenkins_worker_codejail.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "new_relic_infrastructure_license_key": "{{env `NEWRELIC_INFRASTRUCTURE_LICENSE_KEY`}}", - "playbook_remote_dir": "/tmp/packer-edx-playbooks", - "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible", - "ami": "{{env `JENKINS_WORKER_AMI`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}", - "remote_branch": "{{env `REMOTE_BRANCH`}}", - "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "jenkins_worker_codejail-{{isotime | clean_ami_name}}", - "instance_type": "m3.medium", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "jenkins worker codejail", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}", - "tags": { - "delete_or_keep": "{{user `delete_or_keep`}}" - }, - "launch_block_device_mappings": [{ - "delete_on_termination": true, - "device_name": "/dev/sda1", - "volume_size": "40", - "volume_type": "gp2" - }] - }], - "provisioners": [{ - "type": "shell", - "inline": ["rm -rf {{user `playbook_remote_dir`}}", - "mkdir {{user `playbook_remote_dir`}}"] - }, { - "type": "file", - "source": "stop-automatic-updates.sh", - "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh" - }, { - "type": "file", - "source": "../../util/install/ansible-bootstrap.sh", - "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh" - }, { - "type": "shell", - "inline": ["cd {{user `playbook_remote_dir`}}", - "export CONFIGURATION_VERSION='{{user `remote_branch`}}'", - "sudo bash ./stop-automatic-updates.sh", - "sudo bash ./ansible-bootstrap.sh" - ] - }, { - "type": "shell-local", - "command": "rm ../../playbooks/edx-east" - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/jenkins_worker_codejail.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"NEWRELIC_INFRASTRUCTURE_LICENSE_KEY={{user `new_relic_infrastructure_license_key`}} jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}' ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }] -} diff --git a/util/packer/jenkins_worker_simple.json b/util/packer/jenkins_worker_simple.json deleted file mode 100644 index 6ff5ed7a9e7..00000000000 --- a/util/packer/jenkins_worker_simple.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "new_relic_infrastructure_license_key": "{{env `NEWRELIC_INFRASTRUCTURE_LICENSE_KEY`}}", - "playbook_remote_dir": "/tmp/packer-edx-playbooks", - "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible", - "ami": "{{env `JENKINS_WORKER_AMI`}}", - "test_platform_version": "{{env `TEST_PLATFORM_VERSION`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}", - "remote_branch": "{{env `REMOTE_BRANCH`}}", - "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}", - "instance_type": "m3.large", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "jenkins worker", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}", - "tags": { - "delete_or_keep": "{{user `delete_or_keep`}}" - }, - "launch_block_device_mappings": [{ - "delete_on_termination": true, - "device_name": "/dev/sda1", - "volume_size": "40", - "volume_type": "gp2" - }] - }], - "provisioners": [{ - "type": "shell", - "inline": ["rm -rf {{user `playbook_remote_dir`}}", - "mkdir {{user `playbook_remote_dir`}}"] - }, { - "type": "file", - "source": "stop-automatic-updates.sh", - "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh" - }, { - "type": "file", - "source": "../../util/install/ansible-bootstrap.sh", - "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh" - }, { - "type": "shell", - "inline": ["cd {{user `playbook_remote_dir`}}", - "export CONFIGURATION_VERSION='{{user `remote_branch`}}'", - "sudo bash ./stop-automatic-updates.sh", - "sudo bash ./ansible-bootstrap.sh" - ] - }, { - "type": "shell-local", - "command": "rm ../../playbooks/edx-east" - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/jenkins_worker.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"JENKINS_EDX_PLATFORM_VERSION={{user `test_platform_version`}} NEWRELIC_INFRASTRUCTURE_LICENSE_KEY={{user `new_relic_infrastructure_license_key`}} initialize_replica_set=false mongo_configure_replica_set=false jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}' ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }] -} diff --git a/util/packer/jenkins_worker_user_retire.json b/util/packer/jenkins_worker_user_retire.json deleted file mode 100644 index 194f28c6d23..00000000000 --- a/util/packer/jenkins_worker_user_retire.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "playbook_remote_dir": "/tmp/packer-edx-playbooks", - "venv_dir": "/edx/app/edx_ansible/venvs/edx_ansible", - "ami": "{{env `JENKINS_WORKER_AMI`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}", - "remote_branch": "{{env `REMOTE_BRANCH`}}", - "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "jenkins_worker-{{isotime | clean_ami_name}}", - "instance_type": "m3.large", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "jenkins worker for user retirement", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}", - "tags": { - "delete_or_keep": "{{user `delete_or_keep`}}" - }, - "launch_block_device_mappings": [{ - "delete_on_termination": true, - "device_name": "/dev/sda1", - "volume_size": "40", - "volume_type": "gp2" - }] - }], - "provisioners": [{ - "type": "shell", - "inline": ["rm -rf {{user `playbook_remote_dir`}}", - "mkdir {{user `playbook_remote_dir`}}"] - }, { - "type": "file", - "source": "stop-automatic-updates.sh", - "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh" - }, { - "type": "file", - "source": "../../util/install/ansible-bootstrap.sh", - "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh" - }, { - "type": "shell", - "inline": ["cd {{user `playbook_remote_dir`}}", - "export CONFIGURATION_VERSION='{{user `remote_branch`}}'", - "sudo bash ./stop-automatic-updates.sh", - "sudo bash ./ansible-bootstrap.sh" - ] - }, { - "type": "shell-local", - "command": "rm ../../playbooks/edx-east" - }, { - "type": "ansible-local", - "playbook_file": "../../playbooks/jenkins_worker_user_retire.yml", - "playbook_dir": "../../playbooks", - "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook", - "inventory_groups": "jenkins_worker", - "extra_arguments": [ - "-e \"initialize_replica_set=false mongo_configure_replica_set=false jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}' ansible_python_interpreter=/usr/bin/python3\"", - "-vvv" - ] - }] -} diff --git a/util/packer/stop-automatic-updates.sh b/util/packer/stop-automatic-updates.sh deleted file mode 100644 index 610cb391c51..00000000000 --- a/util/packer/stop-automatic-updates.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Kill the apt services holding a dpkg lock, so that the ansible-bootstrap -# script can run without conflicts. - -# NOTE: this is a temporary fix. Instead, we should be doing what SRE does, -# and first run the security+common roles on a vanilla AMI, which will disable -# unattended-updates and set up users. Then we can feel free to run the -# ansible bootstrap without any problems. - -set -xe - -if grep -q 'Focal Fossa' /etc/os-release; then - systemctl stop apt-daily.service - systemctl kill --kill-who=all apt-daily.service - # Our jenkins job for building AMIs will timeout, even if the lock is - # never released. - while lsof |grep -q /var/lib/dpkg/lock; do - echo "Waiting for apt to release the dpkg lock..." - sleep 5 - done -fi - diff --git a/util/packer/webpagetest.json b/util/packer/webpagetest.json deleted file mode 100644 index aec6e9ae628..00000000000 --- a/util/packer/webpagetest.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "security_group": "{{env `AWS_SECURITY_GROUP`}}", - "ami": "{{env `WEBPAGETEST_BASE_AMI`}}" - }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "ami_name": "wpt-{{isotime | clean_ami_name}}", - "instance_type": "m3.medium", - "region": "us-east-1", - "source_ami": "{{user `ami`}}", - "ssh_username": "ubuntu", - "ami_description": "webpagetest", - "iam_instance_profile": "jenkins-worker", - "security_group_id": "{{user `security_group`}}" - }], - "provisioners": [{ - "type": "shell", - "inline": ["echo 'THESE ARE THE WEBPAGETEST SETTINGS:'", - "cat /var/www/webpagetest/www/settings/settings.ini"] - }] -} diff --git a/util/parsefiles_config.yml b/util/parsefiles_config.yml index 6e7854fedc3..7b83943242d 100644 --- a/util/parsefiles_config.yml +++ b/util/parsefiles_config.yml @@ -33,8 +33,6 @@ weights: - notes: 2 - mongo: 1 - devpi: 1 - - jenkins_build: 8 - - jenkins_worker: 23 - analytics_pipeline: 8 - analytics_pipeline_hadoop_datanode: 2 - analytics_pipeline_hadoop_namenode: 3 From dacaa18b7dfd998576ea42ffeee5d404443b5b8b Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Thu, 18 May 2023 14:53:38 -0400 Subject: [PATCH 435/664] feat: enable distributed tracing by default in newrelic.ini.j2 (#6930) This repo is being DEPRed, and at some point these configs will be easier to config per service. For now, we just need to set a global default to enable this. --- playbooks/roles/edxapp/templates/newrelic.ini.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/edxapp/templates/newrelic.ini.j2 b/playbooks/roles/edxapp/templates/newrelic.ini.j2 index 26bb86bf2a3..69c3b3deea4 100644 --- a/playbooks/roles/edxapp/templates/newrelic.ini.j2 +++ b/playbooks/roles/edxapp/templates/newrelic.ini.j2 @@ -27,3 +27,5 @@ # `course_id`. # browser_monitoring.attributes.enabled=true + +distributed_tracing.enabled=true From a9708a41080d1beaa15bf27e742e63bd435a0d82 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 19 May 2023 11:58:58 +0500 Subject: [PATCH 436/664] feat: update github actions runner --- docker/build/github-actions-runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build/github-actions-runner/Dockerfile b/docker/build/github-actions-runner/Dockerfile index 61e6ef5e9cb..b4d6a5a65e9 100644 --- a/docker/build/github-actions-runner/Dockerfile +++ b/docker/build/github-actions-runner/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:focal # Set the github runner version -ARG RUNNER_VERSION="2.279.0" +ARG RUNNER_VERSION="2.304.0" ENV GITHUB_ORGANIZATION="" ENV GITHUB_ACCESS_TOKEN="" From aa553803a488b3d41041b47c69a5b2a6682291e8 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 19 May 2023 17:18:58 -0400 Subject: [PATCH 437/664] fix: Don't run celery on web frontend servers Fixes a bug that probably only affects edx.org where enabling the celery worker at build time causes it to always run, even on a frontend web worker node. At build time edx.org sets disable_edx_services to true. We have a different script that enables specific services at boot time based on EC2 instance tags. --- playbooks/roles/edx_django_service/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index fc901545677..b71fe5f979e 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -351,7 +351,7 @@ - install:configuration - name: enable celery worker supervisor script - when: edx_django_service_enable_celery_workers + when: edx_django_service_enable_celery_workers and not disable_edx_services file: src: "{{ supervisor_available_dir }}/{{ edx_django_service_workers_supervisor_conf }}" dest: "{{ supervisor_cfg_dir }}/{{ edx_django_service_workers_supervisor_conf }}" From f2eaec97b2d4574edf269ea6465e4297731d69c7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 24 May 2023 11:31:20 +0500 Subject: [PATCH 438/664] chore: add env var for APPLE_ROOT_CA --- playbooks/roles/ecommerce/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 25d95a0ee1b..f297d61e230 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -299,6 +299,7 @@ ecommerce_gunicorn_port: "8130" ecommerce_environment: ECOMMERCE_CFG: "{{ COMMON_CFG_DIR }}/{{ ecommerce_service_name }}.yml" + APPLE_ROOT_CA: "{{ COMMON_APP_DIR }}/ecommerce/ecommerce/ecommerce/extensions/iap/AppleRootCA-G3.cer" ecommerce_create_demo_data: false From 4e09fa13a69ca15e92c4900c48f6f86ec608033a Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Tue, 30 May 2023 14:11:56 -0400 Subject: [PATCH 439/664] Revert "feat: enable distributed tracing by default in newrelic.ini.j2 (#6930)" (#6934) This reverts commit dacaa18b7dfd998576ea42ffeee5d404443b5b8b. --- playbooks/roles/edxapp/templates/newrelic.ini.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/playbooks/roles/edxapp/templates/newrelic.ini.j2 b/playbooks/roles/edxapp/templates/newrelic.ini.j2 index 69c3b3deea4..26bb86bf2a3 100644 --- a/playbooks/roles/edxapp/templates/newrelic.ini.j2 +++ b/playbooks/roles/edxapp/templates/newrelic.ini.j2 @@ -27,5 +27,3 @@ # `course_id`. # browser_monitoring.attributes.enabled=true - -distributed_tracing.enabled=true From 41052e2dfcacf2f9d371cfd3408b09eb6ac2877f Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Fri, 2 Jun 2023 12:01:55 +0500 Subject: [PATCH 440/664] chore: use private dockerfile for installing private requirements (#6935) --- util/jenkins/ansible-provision.sh | 20 ++++++++++++++++++++ util/jenkins/app-container-provisioner.sh | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index a3f3cda9d3a..326875c064a 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -692,9 +692,29 @@ EOF sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/lms.yml sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/cms.yml + # Remove exiting private requirements if found + if [[ -f "$WORKSPACE/dockerfiles-internal/edx-platform-private/private_requirements.txt" ]] ; then + rm -f $WORKSPACE/dockerfiles-internal/edx-platform-private/private_requirements.txt + fi + + # Extract private requirements for sandbox + readarray app_private_requirements < <(cat $WORKSPACE/configuration/playbooks/roles/edxapp/defaults/main.yml | $WORKSPACE/yq e -o=j -I=0 '.EDXAPP_PRIVATE_REQUIREMENTS[]') + for app_private_requirement in "${app_private_requirements[@]}"; do + if ! $(echo ${app_private_requirement} | $WORKSPACE/yq '. | has("extra_args")' -) ; then + req_name=$(echo "${app_private_requirement}" | $WORKSPACE/yq -e '.name' -) + echo -e "${req_name}" >> $WORKSPACE/dockerfiles-internal/edx-platform-private/private_requirements.txt + else + req_name=$(echo "${app_private_requirement}" | $WORKSPACE/yq -e '.name' -) + req_extra_args=$(echo "${app_private_requirement}" | $WORKSPACE/yq -e '.extra_args' -) + echo -e "${req_extra_args} ${req_name}" >> $WORKSPACE/dockerfiles-internal/edx-platform-private/private_requirements.txt + fi + done + # copy app config file ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/lms.yml dest=/var/tmp/lms.yml" -u ubuntu -b ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/cms.yml dest=/var/tmp/cms.yml" -u ubuntu -b + # copy private Dockerfile and requirements file + ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/dockerfiles-internal/edx-platform-private dest=/var/tmp/" -u ubuntu -b set +x app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" diff --git a/util/jenkins/app-container-provisioner.sh b/util/jenkins/app-container-provisioner.sh index 03d19f0ed52..1542b360d53 100644 --- a/util/jenkins/app-container-provisioner.sh +++ b/util/jenkins/app-container-provisioner.sh @@ -62,7 +62,9 @@ if ! $(docker image inspect ${app_image_name} >/dev/null 2>&1 && echo true || ec cd /edx/app/${app_name}/${app_repo} export DOCKER_BUILDKIT=1 if [[ ${app_service_name} == 'lms' || ${app_service_name} == 'cms' ]]; then - docker build . -t ${app_repo}:latest --target base + docker build . -t ${app_repo}:base --target base + cd /var/tmp/edx-platform-private + docker build . --build-arg BASE_IMAGE=${app_repo} --build-arg BASE_TAG=base -t ${app_repo}:latest else docker build . -t ${app_repo}:latest fi From 280c7b7b8311e0f6631d6abf744f9c7674151fd9 Mon Sep 17 00:00:00 2001 From: Agrendalath Date: Mon, 5 Jun 2023 09:48:53 +0200 Subject: [PATCH 441/664] chore: update Problem Builder XBlock --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 45b02f25fdf..1ffcf07ffc9 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -542,7 +542,7 @@ EDXAPP_EXTRA_REQUIREMENTS: [] # - name: git+https://git.myproject.org/MyProject#egg=MyProject EDXAPP_PRIVATE_REQUIREMENTS: # For Harvard courses: - - name: xblock-problem-builder==5.1.0 + - name: xblock-problem-builder==5.1.3 # Oppia XBlock - name: git+https://github.com/oppia/xblock.git@1030adb3590ad2d32c93443cc8690db0985d76b6#egg=oppia-xblock extra_args: -e From 51c978c2fa58eb1456389169bae3435c732d7a7e Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 12 Jun 2023 11:57:03 -0400 Subject: [PATCH 442/664] build: fix make upgrade by versioning pip consistent with piptools --- Makefile | 6 +++--- pre-requirements.txt | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 pre-requirements.txt diff --git a/Makefile b/Makefile index f3918dc8869..8142015e4e4 100755 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ main.help: @echo '' requirements: - pip install -qr pre-requirements.txt --exists-action w + pip install -qr requirements/pip.txt --exists-action w pip install -qr requirements.txt --exists-action w COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt @@ -28,7 +28,7 @@ $(COMMON_CONSTRAINTS_TXT): upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade upgrade: $(COMMON_CONSTRAINTS_TXT) ## update the pip requirements files to use the latest releases satisfying our constraints - pip install -qr pre-requirements.txt --exists-action w + pip install -qr requirements/pip.txt pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in @@ -44,7 +44,7 @@ upgrade: $(COMMON_CONSTRAINTS_TXT) pip-compile --upgrade -o util/jenkins/requirements.txt requirements/jenkins.in # Post process all of the files generated above to work around open pip-tools issues util/post-pip-compile.sh \ - requirements/pip-tools.txt \ + requirements/pip-tools.txt \ requirements.txt \ playbooks/roles/aws/templates/requirements.txt.j2 \ util/elasticsearch/requirements.txt \ diff --git a/pre-requirements.txt b/pre-requirements.txt deleted file mode 100644 index ed2bcefab31..00000000000 --- a/pre-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pip==21.2.1 From de8c8988a106e6fdbf0d78c8463923f904befb28 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Mon, 12 Jun 2023 11:59:02 -0400 Subject: [PATCH 443/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 18 +++---- requirements.txt | 30 +++++------ requirements/common_constraints.txt | 13 +++-- requirements/pip-tools.txt | 18 +++---- requirements/pip.txt | 10 ++-- util/elasticsearch/requirements.txt | 8 +-- util/jenkins/requirements-cloudflare.txt | 14 ++--- util/jenkins/requirements.txt | 53 +++++++++++-------- util/pingdom/requirements.txt | 14 ++--- util/vpc-tools/requirements.txt | 14 ++--- 10 files changed, 99 insertions(+), 93 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 3f3e985e5a5..a9a76cd4346 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -1,16 +1,16 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -awscli==1.25.72 +awscli==1.27.151 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.24.71 +boto3==1.26.151 # via -r requirements/aws.in -botocore==1.27.71 +botocore==1.29.151 # via # awscli # boto3 @@ -23,7 +23,7 @@ jmespath==1.0.1 # via # boto3 # botocore -pyasn1==0.4.8 +pyasn1==0.5.0 # via rsa python-dateutil==2.8.2 # via @@ -37,13 +37,13 @@ pyyaml==5.3.1 # awscli rsa==4.7.2 # via awscli -s3cmd==2.2.0 +s3cmd==2.3.0 # via -r requirements/aws.in -s3transfer==0.6.0 +s3transfer==0.6.1 # via # awscli # boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.12 +urllib3==1.26.16 # via botocore diff --git a/requirements.txt b/requirements.txt index ea7312aeb17..58a64305c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # @@ -8,7 +8,7 @@ ansible==2.10.7 # via -r requirements/base.in ansible-base==2.10.17 # via ansible -awscli==1.25.72 +awscli==1.27.151 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -16,25 +16,25 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.24.71 +boto3==1.26.151 # via -r requirements/base.in -botocore==1.27.71 +botocore==1.29.151 # via # awscli # boto3 # s3transfer -certifi==2022.6.15.1 +certifi==2023.5.7 # via requests cffi==1.15.1 # via # bcrypt # cryptography # pynacl -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests colorama==0.4.4 # via awscli -cryptography==38.0.1 +cryptography==41.0.1 # via # ansible-base # paramiko @@ -50,7 +50,7 @@ docutils==0.16 # via awscli ecdsa==0.13.3 # via -r requirements/base.in -idna==3.3 +idna==3.4 # via requests jinja2==2.8 # via @@ -68,7 +68,7 @@ mysqlclient==1.4.6 # via -r requirements/base.in networkx==1.11 # via -r requirements/base.in -packaging==21.3 +packaging==23.1 # via ansible-base paramiko==2.4.2 # via -r requirements/base.in @@ -76,7 +76,7 @@ pathlib2==2.3.0 # via -r requirements/base.in prettytable==0.7.2 # via -r requirements/base.in -pyasn1==0.4.8 +pyasn1==0.5.0 # via # paramiko # rsa @@ -88,8 +88,6 @@ pymongo==3.9.0 # via -r requirements/base.in pynacl==1.5.0 # via paramiko -pyparsing==3.0.9 - # via packaging python-dateutil==2.8.2 # via botocore pyyaml==5.4.1 @@ -97,13 +95,13 @@ pyyaml==5.4.1 # -r requirements/base.in # ansible-base # awscli -requests==2.28.1 +requests==2.31.0 # via # -r requirements/base.in # datadog rsa==4.7.2 # via awscli -s3transfer==0.6.0 +s3transfer==0.6.1 # via # awscli # boto3 @@ -112,7 +110,7 @@ six==1.16.0 # bcrypt # pathlib2 # python-dateutil -urllib3==1.26.12 +urllib3==1.26.16 # via # botocore # requests diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index b1dfdf0176d..7e39123ff04 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -19,9 +19,14 @@ Django<4.0 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html elasticsearch<7.14.0 -# setuptools==60.0 had breaking changes and busted several service's pipeline. -# Details can be found here: https://github.com/pypa/setuptools/issues/2940 -setuptools<60 - # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected django-simple-history==3.0.0 + +# tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. +# Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 +tox<4.0.0 + +# edx-sphinx-theme is not compatible with latest Sphinx==6.0.0 version +# Pinning Sphinx version unless the compatibility issue gets resolved +# For details, see issue https://github.com/openedx/edx-sphinx-theme/issues/197 +sphinx<6.0.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ebb8aa6a30f..8620fabd6b5 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,23 +4,19 @@ # # make upgrade # -build==0.8.0 +build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools -packaging==21.3 +packaging==23.1 # via build -pep517==0.13.0 - # via build -pip-tools==6.8.0 +pip-tools==6.13.0 # via -r requirements/pip-tools.in -pyparsing==3.0.9 - # via packaging +pyproject-hooks==1.0.0 + # via build tomli==2.0.1 - # via - # build - # pep517 -wheel==0.37.1 + # via build +wheel==0.40.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 35b87891c26..c9cbf009061 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,13 +4,11 @@ # # make upgrade # -wheel==0.37.1 +wheel==0.40.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==22.2.2 +pip==23.1.2 + # via -r requirements/pip.in +setuptools==67.8.0 # via -r requirements/pip.in -setuptools==59.8.0 - # via - # -c requirements/common_constraints.txt - # -r requirements/pip.in diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 254db4a8416..188701ce669 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # @@ -8,7 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -jsonpickle==2.2.0 +jsonpickle==3.0.1 # via deepdiff -urllib3==1.26.12 +urllib3==1.26.16 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 9ad045870f2..2fbd1f0b7c6 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -1,18 +1,18 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -certifi==2022.6.15.1 +certifi==2023.5.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via -r requirements/cloudflare.in -idna==3.3 +idna==3.4 # via requests -requests==2.28.1 +requests==2.31.0 # via -r requirements/cloudflare.in -urllib3==1.26.12 +urllib3==2.0.3 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 4b610fd138e..20b4a05fd7c 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -1,33 +1,37 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # amqp==5.1.1 # via kombu -awscli==1.25.72 +awscli==1.27.151 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in -billiard==3.6.4.0 +backports-zoneinfo[tzdata]==0.2.1 + # via + # celery + # kombu +billiard==4.1.0 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.24.71 +boto3==1.26.151 # via -r requirements/jenkins.in -botocore==1.27.71 +botocore==1.29.151 # via # awscli # boto3 # s3transfer -celery==5.2.7 +celery==5.3.0 # via -r requirements/jenkins.in -certifi==2022.6.15.1 +certifi==2023.5.7 # via # opsgenie-sdk # requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via @@ -46,48 +50,47 @@ colorama==0.4.4 # via awscli docutils==0.16 # via awscli -idna==3.3 +idna==3.4 # via requests jmespath==1.0.1 # via # boto3 # botocore -kombu==5.2.4 +kombu==5.3.0 # via celery opsgenie-sdk==0.3.1 # via -r requirements/jenkins.in -prompt-toolkit==3.0.31 +prompt-toolkit==3.0.38 # via click-repl -pyasn1==0.4.8 +pyasn1==0.5.0 # via rsa pymysql==0.9.3 # via -r requirements/jenkins.in python-dateutil==2.8.2 # via # botocore + # celery # opsgenie-sdk # s3cmd python-gnupg==0.5.0 # via -r requirements/jenkins.in python-magic==0.4.27 # via s3cmd -pytz==2022.2.1 - # via - # celery - # opsgenie-sdk +pytz==2023.3 + # via opsgenie-sdk pyyaml==5.4.1 # via # -r requirements/jenkins.in # awscli redis==2.10.6 # via -r requirements/jenkins.in -requests==2.28.1 +requests==2.31.0 # via opsgenie-sdk rsa==4.7.2 # via awscli -s3cmd==2.2.0 +s3cmd==2.3.0 # via -r requirements/jenkins.in -s3transfer==0.6.0 +s3transfer==0.6.1 # via # awscli # boto3 @@ -98,7 +101,13 @@ six==1.16.0 # python-dateutil splunk-sdk==1.6.16 # via -r requirements/jenkins.in -urllib3==1.26.12 +typing-extensions==4.6.3 + # via kombu +tzdata==2023.3 + # via + # backports-zoneinfo + # celery +urllib3==1.26.16 # via # botocore # opsgenie-sdk @@ -108,7 +117,7 @@ vine==5.0.0 # amqp # celery # kombu -wcwidth==0.2.5 +wcwidth==0.2.6 # via prompt-toolkit # The following packages are considered to be unsafe in a requirements file: diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 04183c2c24a..9ee7f1503b8 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -1,22 +1,22 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -certifi==2022.6.15.1 +certifi==2023.5.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests click==6.7 # via -r requirements/pingdom.in -idna==3.3 +idna==3.4 # via requests pyyaml==6.0 # via -r requirements/pingdom.in -requests==2.28.1 +requests==2.31.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==1.26.12 +urllib3==2.0.3 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 635cb6e3dcc..edbfba21f4c 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2022.6.15.1 +certifi==2023.5.7 # via requests -charset-normalizer==2.1.1 +charset-normalizer==3.1.0 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==3.3 +idna==3.4 # via requests -requests==2.28.1 +requests==2.31.0 # via -r requirements/vpc-tools.in -urllib3==1.26.12 +urllib3==2.0.3 # via requests From de73ae24741b45ed3d0d15f714e37baa1fa78bd5 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 13 Jun 2023 11:00:10 -0400 Subject: [PATCH 444/664] fix: following https://github.com/openedx/configuration/pull/6938 --- playbooks/roles/edx_ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edx_ansible/defaults/main.yml b/playbooks/roles/edx_ansible/defaults/main.yml index 2c69b5fbf2c..467953700b3 100644 --- a/playbooks/roles/edx_ansible/defaults/main.yml +++ b/playbooks/roles/edx_ansible/defaults/main.yml @@ -58,7 +58,7 @@ edx_ansible_user: "edx-ansible" edx_ansible_source_repo: https://github.com/openedx/configuration.git edx_ansible_requirements_file: "{{ edx_ansible_code_dir }}/requirements.txt" edx_ansible_requirements_files: - - "{{ edx_ansible_code_dir }}/pre-requirements.txt" + - "{{ edx_ansible_code_dir }}/requirements/pip.txt" - "{{ edx_ansible_code_dir }}/requirements.txt" # edX configuration repo From 35715cc078e6f8c553ddb258597f720d953f16e4 Mon Sep 17 00:00:00 2001 From: roybrown-trilogyed <49158971+roybrown-trilogyed@users.noreply.github.com> Date: Tue, 20 Jun 2023 09:12:46 -0400 Subject: [PATCH 445/664] feat: XPRT-65 Add Xpert API URL env variables --- CHANGELOG.md | 5 +++++ playbooks/roles/prospectus/templates/.env.environment.j2 | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e030f33756a..8cbb42c8c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-06-20 + - Role: prospectus + - Added `GATSBY_XPERT_STG_API_URL`, `GATSBY_XPERT_UAT_API_URL` and `GATSBY_XPERT_PROD_API_URL` + to enable Prospectus to communicate with the Xpert API + - 2023-04-07 - Changed default value of `EDXAPP_MONGO_REPLICA_SET` to `null` from existing empty string `""`, to make it compatible with pymongo >= 3.11 in Nutmeg and above. diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 5b47d2d01b7..337cec0f01f 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -11,4 +11,7 @@ GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} -GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} \ No newline at end of file +GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} +GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} +GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} +GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} \ No newline at end of file From ed74f00867a8bd7e3e44b108f6cb300ad0721a86 Mon Sep 17 00:00:00 2001 From: roybrown-trilogyed <49158971+roybrown-trilogyed@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:28:08 -0400 Subject: [PATCH 446/664] feat: REVERT - XPRT-65 Add Xpert API URL env variables This reverts commit 35715cc078e6f8c553ddb258597f720d953f16e4. --- CHANGELOG.md | 5 ----- playbooks/roles/prospectus/templates/.env.environment.j2 | 5 +---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cbb42c8c0a..e030f33756a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,6 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). - - 2023-06-20 - - Role: prospectus - - Added `GATSBY_XPERT_STG_API_URL`, `GATSBY_XPERT_UAT_API_URL` and `GATSBY_XPERT_PROD_API_URL` - to enable Prospectus to communicate with the Xpert API - - 2023-04-07 - Changed default value of `EDXAPP_MONGO_REPLICA_SET` to `null` from existing empty string `""`, to make it compatible with pymongo >= 3.11 in Nutmeg and above. diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 337cec0f01f..5b47d2d01b7 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -11,7 +11,4 @@ GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} -GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} -GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} -GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} -GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} \ No newline at end of file +GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} \ No newline at end of file From 730536629b94fa4c1dfb0943489b953156e6d42f Mon Sep 17 00:00:00 2001 From: roybrown-trilogyed <49158971+roybrown-trilogyed@users.noreply.github.com> Date: Tue, 20 Jun 2023 11:41:53 -0400 Subject: [PATCH 447/664] feat: XPRT-65 Add Xpert API URL env variables This reverts commit ed74f00867a8bd7e3e44b108f6cb300ad0721a86. --- CHANGELOG.md | 5 +++++ playbooks/roles/prospectus/templates/.env.environment.j2 | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e030f33756a..8cbb42c8c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). + - 2023-06-20 + - Role: prospectus + - Added `GATSBY_XPERT_STG_API_URL`, `GATSBY_XPERT_UAT_API_URL` and `GATSBY_XPERT_PROD_API_URL` + to enable Prospectus to communicate with the Xpert API + - 2023-04-07 - Changed default value of `EDXAPP_MONGO_REPLICA_SET` to `null` from existing empty string `""`, to make it compatible with pymongo >= 3.11 in Nutmeg and above. diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 5b47d2d01b7..337cec0f01f 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -11,4 +11,7 @@ GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} -GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} \ No newline at end of file +GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} +GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} +GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} +GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} \ No newline at end of file From 33f7c133188d874553ef59c6ac6ab4bc09323474 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 21 Jun 2023 12:35:08 +0500 Subject: [PATCH 448/664] feat: install mysql 8.0 on sandbox --- playbooks/roles/mysql/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mysql/defaults/main.yml b/playbooks/roles/mysql/defaults/main.yml index 92cdf4f00ca..de977aa4ecf 100644 --- a/playbooks/roles/mysql/defaults/main.yml +++ b/playbooks/roles/mysql/defaults/main.yml @@ -21,7 +21,7 @@ mysql_dir: /etc/mysql mysql_socket: /var/run/mysqld/mysqld.sock -mysql_8_0_install: false +mysql_8_0_install: true mysql_server_8_0_pkgs: - mysql-client-8.0 From 631e81271c71b8e95c6cb3905ec273e54e137c14 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 1 Jun 2023 18:05:03 +0500 Subject: [PATCH 449/664] chore: include the Nginx X-Robots-Tag header in the Prospectus Nginx configuration --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 6 ++++++ playbooks/roles/prospectus/defaults/main.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index a5d49f31115..2d9db5f1292 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -11,6 +11,12 @@ server { add_header X-Frame-Options DENY; + {% if NGINX_PROSPECTUS_DISABLE_INDEXING %} + + add_header X-Robots-Tag "noindex, nofollow" always; + + {% endif %} + {% if NGINX_ENABLE_SSL %} listen {{ prospectus_ssl_nginx_port }} ssl; diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 39e6b80d399..29158e55c17 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -29,6 +29,7 @@ PROSPECTUS_ALGOLIA_ADMIN_KEY: 'fake-key' # nginx vars PROSPECTUS_DATA_DIR: '/edx/var/prospectus' NGINX_PROSPECTUS_PROXY_INTERCEPT_ERRORS: true +NGINX_PROSPECTUS_DISABLE_INDEXING: false PROSPECTUS_STATIC_SITES: [] PROSPECTUS_TEMPORARY_REDIRECTS: [] From b534c89dd0eb59eecbaadceecb0e79ad25302ac7 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 26 Jun 2023 16:32:09 -0400 Subject: [PATCH 450/664] feat: Start uploading prospectus stg builds to s3 ISRE-1999 --- playbooks/roles/prospectus/defaults/main.yml | 4 ++ playbooks/roles/prospectus/tasks/main.yml | 52 ++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 29158e55c17..7310464e451 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -61,3 +61,7 @@ prospectus_ssl_nginx_port: 443 prospectus_use_python3: true PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS: False +PROSPECTUS_S3_HOSTING_ENABLED: False +PROSPECTUS_S3_HOSTING_BUCKET: !!null +PROSPECTUS_S3_HOSTING_PREFIX: !!null +PROSPECTUS_S3_UPLOAD_MAX_CONCCURENCT_REQUESTS: 50 diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 3bc3b7bcce3..0b3587a419e 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -59,7 +59,7 @@ mode: "0600" when: PROSPECTUS_GIT_IDENTITY != "none" -- name: "Checkout prospectus repo into {{ prospectus_code_dir }}" +- name: "Checkout prospectus repo into {{ prospectus_code_dir }} with key" git: dest: "{{ prospectus_code_dir }}" repo: "{{ prospectus_repo }}" @@ -67,17 +67,17 @@ accept_hostkey: yes key_file: "{{ prospectus_git_identity }}" become_user: "{{ prospectus_user }}" - register: prospectus_checkout + register: prospectus_checkout_with_key when: PROSPECTUS_GIT_IDENTITY != "none" -- name: Checkout prospectus repo into {{ prospectus_code_dir }} +- name: Checkout prospectus repo into {{ prospectus_code_dir }} without key git: dest: "{{ prospectus_code_dir }}" repo: "{{ prospectus_repo }}" version: "{{ PROSPECTUS_VERSION }}" accept_hostkey: yes become_user: "{{ prospectus_user }}" - register: prospectus_checkout + register: prospectus_checkout_without_key when: PROSPECTUS_GIT_IDENTITY == "none" - name: move cache dir to {{ prospectus_code_dir }} @@ -224,3 +224,47 @@ - name: Move prospectus public folder to var folder shell: "mv {{ prospectus_code_dir }}/public/* {{ PROSPECTUS_DATA_DIR }}" + +- name: Set prospectus_s3_prefix to prospectus_checkout_without_key.after + ignore_errors: true + set_fact: + prospectus_s3_prefix: "{{ prospectus_checkout_without_key.after }}" + when: PROSPECTUS_GIT_IDENTITY == "none" + +- name: Set prospectus_s3_prefix to prospectus_checkout_with_key.after + ignore_errors: true + set_fact: + prospectus_s3_prefix: "{{ prospectus_checkout_with_key.after }}-{{ ansible_date_time.epoch }}" + when: PROSPECTUS_GIT_IDENTITY != "none" + +- name: Set prospectus_s3_prefix to PROSPECTUS_S3_HOSTING_PREFIX + ignore_errors: true + set_fact: + prospectus_s3_prefix: "{{ PROSPECTUS_S3_HOSTING_PREFIX }}" + when: PROSPECTUS_S3_HOSTING_PREFIX != None + +- name: Set Prospectus S3 max concurrency + ignore_errors: true + become_user: "{{ prospectus_user }}" + shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCCURENCT_REQUESTS }}' + when: PROSPECTUS_S3_HOSTING_ENABLED|bool + tags: + - install + - install:system-requirements + +- name: Upload prospectus to S3 + ignore_errors: true + become_user: "{{ prospectus_user }}" + shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ prospectus_s3_prefix }}' + when: PROSPECTUS_S3_HOSTING_ENABLED|bool + tags: + - install + - install:system-requirements + +- name: Download prospectus redirects to GoCD + ignore_errors: true + when: PROSPECTUS_S3_HOSTING_ENABLED|bool + fetch: + src: "{{ prospectus_redirect_file }}" + dest: "{{ artifact_path }}/prospectus-redirects.conf" + flat: yes From dfc25ecd514fdb13ebf513c08863b2d99b97a601 Mon Sep 17 00:00:00 2001 From: John Nagro Date: Wed, 28 Jun 2023 17:49:41 +0000 Subject: [PATCH 451/664] fix: pkg-config for mysql --- .../roles/ansible-role-django-ida/templates/defaults/main.yml.j2 | 1 + playbooks/roles/edx_django_service/defaults/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2 b/playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2 index 1eefaf5c425..5acc8e57c14 100644 --- a/playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2 +++ b/playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2 @@ -142,5 +142,6 @@ nginx_{{ role_name }}_gunicorn_hosts: {{ role_name }}_debian_pkgs: - libmysqlclient-dev - libssl-dev + - pkg-config {{ role_name }}_redhat_pkgs: [] diff --git a/playbooks/roles/edx_django_service/defaults/main.yml b/playbooks/roles/edx_django_service/defaults/main.yml index cde52c7581f..818cee92921 100644 --- a/playbooks/roles/edx_django_service/defaults/main.yml +++ b/playbooks/roles/edx_django_service/defaults/main.yml @@ -54,6 +54,7 @@ edx_django_service_debian_pkgs_default: - libmemcached-dev - libmysqlclient-dev - libssl-dev + - pkg-config edx_django_service_debian_pkgs_extra: [] edx_django_service_debian_pkgs: '{{ edx_django_service_debian_pkgs_default + edx_django_service_debian_pkgs_extra }}' From a791a9f7857ed0c9515a5399f4b7d0d239080e17 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Wed, 28 Jun 2023 18:44:38 -0400 Subject: [PATCH 452/664] feat: enterprise-catalog - bump up worker concurrency to 4 --- playbooks/roles/enterprise_catalog/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/enterprise_catalog/defaults/main.yml b/playbooks/roles/enterprise_catalog/defaults/main.yml index eb595040554..939a79ff96b 100644 --- a/playbooks/roles/enterprise_catalog/defaults/main.yml +++ b/playbooks/roles/enterprise_catalog/defaults/main.yml @@ -162,6 +162,6 @@ ENTERPRISE_CATALOG_ADMIN_URLS: worker_django_settings_module: "{{ ENTERPRISE_CATALOG_DJANGO_SETTINGS_MODULE }}" ENTERPRISE_CATALOG_CELERY_WORKERS: - queue: '{{ enterprise_catalog_celery_default_queue }}' - concurrency: 1 + concurrency: 4 monitor: True enterprise_catalog_workers: "{{ ENTERPRISE_CATALOG_CELERY_WORKERS }}" From 6bd1195a4723f8ba7e7243f5e96d78bf527da90a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 6 Jul 2023 13:52:01 +0500 Subject: [PATCH 453/664] chore: implementing a conditional build for MongoDB versions 4.2 and 4.4 --- playbooks/edx_continuous_integration.yml | 5 ++++- util/jenkins/ansible-provision.sh | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 4d6b532d7d7..62d56e2feea 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,7 +25,10 @@ - role: edxlocal tags: edxlocal - memcache - - mongo_4_2 + - role: mongo_4_2 + when: MONGO_4_2_ENABLED + - role: mongo_4_4 + when: MONGO_4_4_ENABLED - role: redis - { role: "edxapp", celery_worker: True, when: edxapp_containerized is defined and not edxapp_containerized } - { role: "edxapp", when: edxapp_containerized is defined and not edxapp_containerized } diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 326875c064a..a528eff95e6 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -468,6 +468,19 @@ EOF_AUTH fi +if [[ $mongo_version == "4.2" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_4_2_ENABLED: True +MONGO_4_4_ENABLED: False +MONGO_VERSION +fi +if [[ $mongo_version == "4.4" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_4_2_ENABLED: False +MONGO_4_4_ENABLED: True +MONGO_VERSION +fi + if [[ -n $nginx_users ]]; then cat << EOF_AUTH >> $extra_vars_file NGINX_USERS: $nginx_users @@ -671,7 +684,7 @@ edxapp_containerized: true CAN_GENERATE_NEW_JWT_SIGNATURE: false EOF ansible -i "${deploy_host}," $deploy_host -m include_role -a "name=memcache" -u ubuntu -b - for playbook in redis mongo_4_2; do + for playbook in redis $mongo_version; do run_ansible $playbook.yml -i "${deploy_host}," $extra_var_arg --user ubuntu done run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu --tags "edxlocal" From 4a03e296199ba726a68bb8874ecfc9ca6a6028a5 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 6 Jul 2023 18:03:31 -0400 Subject: [PATCH 454/664] feat: Add prospectus redirects upload ISRE-2008 Upload prospectus redirects file from GoCD to nginx EC2 build instance. --- .../prospectus_download_redirects.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 playbooks/continuous_delivery/prospectus_download_redirects.yml diff --git a/playbooks/continuous_delivery/prospectus_download_redirects.yml b/playbooks/continuous_delivery/prospectus_download_redirects.yml new file mode 100644 index 00000000000..e6b13dbb38d --- /dev/null +++ b/playbooks/continuous_delivery/prospectus_download_redirects.yml @@ -0,0 +1,29 @@ +- name: Download edX Prospectus Service nginx redirects file + hosts: all + become: True + gather_facts: True + vars: + ENABLE_DATADOG: False + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'prospectus' + PROSPECTUS_DATA_DIR: "/edx/var/prospectus" + NGINX_OVERRIDE_DEFAULT_MAP_HASH_SIZE: True + NGINX_MAP_HASH_MAX_SIZE: 4096 + NGINX_MAP_HASH_BUCKET_SIZE: 128 + PROSPECTUS_ENABLED: True + PROSPECTUS_SANDBOX_BUILD: FALSE + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + tasks: + - name: Create redirects config directory + tags: + - download_prospectus_redirects + file: + path: "{{ prospectus_redirect_file | dirname }}" + state: directory + - name: Upload prospectus redirects from GoCD + when: PROSPECTUS_S3_HOSTING_ENABLED|bool + copy: + src: "{{ artifact_path }}/prospectus-redirects.conf" + dest: "{{ prospectus_redirect_file }}" From d98cea79efb5f14d1bbe83b00e733b7a51e057d0 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 10 Jul 2023 13:50:06 -0400 Subject: [PATCH 455/664] chore: Remove unneeded code --- .../continuous_delivery/prospectus_download_redirects.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/continuous_delivery/prospectus_download_redirects.yml b/playbooks/continuous_delivery/prospectus_download_redirects.yml index e6b13dbb38d..34d03109641 100644 --- a/playbooks/continuous_delivery/prospectus_download_redirects.yml +++ b/playbooks/continuous_delivery/prospectus_download_redirects.yml @@ -17,13 +17,10 @@ when: COMMON_ENABLE_AWS_ROLE tasks: - name: Create redirects config directory - tags: - - download_prospectus_redirects file: path: "{{ prospectus_redirect_file | dirname }}" state: directory - name: Upload prospectus redirects from GoCD - when: PROSPECTUS_S3_HOSTING_ENABLED|bool copy: src: "{{ artifact_path }}/prospectus-redirects.conf" dest: "{{ prospectus_redirect_file }}" From 3562e3e1582ff69e0230e774111d2cc74fb5bb9c Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 10 Jul 2023 13:49:34 -0400 Subject: [PATCH 456/664] feat: Add prospectus toggles ISRE-2008 --- playbooks/prospectus.yml | 2 ++ playbooks/roles/prospectus/defaults/main.yml | 8 +++++--- playbooks/roles/prospectus/meta/main.yml | 5 +++-- playbooks/roles/prospectus/tasks/main.yml | 8 ++++---- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/playbooks/prospectus.yml b/playbooks/prospectus.yml index 2c95039870c..61e5054cf10 100644 --- a/playbooks/prospectus.yml +++ b/playbooks/prospectus.yml @@ -18,7 +18,9 @@ - role: automated AUTOMATED_USERS: "{{ PROSPECTUS_AUTOMATED_USERS | default({}) }}" - role: prospectus + when: PROSPECTUS_ENABLE_BUILD|bool - role: nginx + when: PROSPECTUS_ENABLE_NGINX|bool nginx_app_dir: "/etc/nginx" nginx_sites: - prospectus diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 7310464e451..26d765dc9f0 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -61,7 +61,9 @@ prospectus_ssl_nginx_port: 443 prospectus_use_python3: true PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS: False -PROSPECTUS_S3_HOSTING_ENABLED: False +PROSPECTUS_ENABLE_NGINX: True +PROSPECTUS_ENABLE_BUILD: True +PROSPECTUS_S3_UPLOAD_ENABLED: False +PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 PROSPECTUS_S3_HOSTING_BUCKET: !!null -PROSPECTUS_S3_HOSTING_PREFIX: !!null -PROSPECTUS_S3_UPLOAD_MAX_CONCCURENCT_REQUESTS: 50 +PROSPECTUS_S3_HOSTING_BUCKET_URL: !!null diff --git a/playbooks/roles/prospectus/meta/main.yml b/playbooks/roles/prospectus/meta/main.yml index 3d12d718ea7..43bbc4a1460 100644 --- a/playbooks/roles/prospectus/meta/main.yml +++ b/playbooks/roles/prospectus/meta/main.yml @@ -1,4 +1,5 @@ --- dependencies: - - common - - nginx + - role: common + - role: nginx + when: PROSPECTUS_ENABLE_NGINX|bool diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 0b3587a419e..98c6200f490 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -246,8 +246,8 @@ - name: Set Prospectus S3 max concurrency ignore_errors: true become_user: "{{ prospectus_user }}" - shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCCURENCT_REQUESTS }}' - when: PROSPECTUS_S3_HOSTING_ENABLED|bool + shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS }}' + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool tags: - install - install:system-requirements @@ -256,14 +256,14 @@ ignore_errors: true become_user: "{{ prospectus_user }}" shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ prospectus_s3_prefix }}' - when: PROSPECTUS_S3_HOSTING_ENABLED|bool + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool tags: - install - install:system-requirements - name: Download prospectus redirects to GoCD ignore_errors: true - when: PROSPECTUS_S3_HOSTING_ENABLED|bool + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool fetch: src: "{{ prospectus_redirect_file }}" dest: "{{ artifact_path }}/prospectus-redirects.conf" From 921bd4c9da42fce79b6eb8b0aca598f9c00da4a8 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 10 Jul 2023 16:08:14 -0400 Subject: [PATCH 457/664] feat: Add prospectus s3 proxy nginx ISRE-2008 --- .../app/nginx/sites-available/prospectus.j2 | 58 ++++++++++++++++++- playbooks/roles/prospectus/defaults/main.yml | 1 + 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 2d9db5f1292..7b340c68def 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -11,6 +11,10 @@ server { add_header X-Frame-Options DENY; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + resolver 127.0.0.53; + {% endif %} + {% if NGINX_PROSPECTUS_DISABLE_INDEXING %} add_header X-Robots-Tag "noindex, nofollow" always; @@ -50,42 +54,82 @@ server { location = /404.html { root {{ PROSPECTUS_DATA_DIR }}; add_header Cache-Control "no-store, max-age=0" always; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + {% endif %} } location /event { # this page is designed to be injected into other pages via an iframe add_header X-Frame-Options ''; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + {% endif %} } # Cache js/css for a long time at the edge, they are versioned in their names location ~ \.(js|css)$ { add_header 'Cache-Control' 'public, max-age=31536000, immutable'; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + {% endif %} } # images sometimes change, we want to cache them for an hour at the edge to reduce bandwidth. location /images/ { add_header 'Cache-Control' 'public, max-age=3600'; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + {% endif %} } # favicon is requested a lot. cache it at the edge. location /favicon.ico { add_header 'Cache-Control' 'public, max-age=86400'; + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + {% endif %} } # Ignore the rollout group headers for the health check endpoint. location /HealthCheck { + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/HealthCheck/index.html; + # proxy_redirect ensures redirects from s3 are rewritten + # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ + # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive + proxy_redirect "/{{ PROSPECTUS_S3_HOSTING_PREFIX }}" " "; + {% else %} try_files $uri $uri/index.html; + {% endif %} } - location /es/bio/ { + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html break; + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html; + # proxy_redirect ensures redirects from s3 are rewritten + # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ + # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive + proxy_redirect "/{{ PROSPECTUS_S3_HOSTING_PREFIX }}" " "; + {% else %} try_files $uri $uri/ /es/bio/index.html; + {% endif %} } location /bio/ { + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html break; + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html; + # proxy_redirect ensures redirects from s3 are rewritten + # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ + # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive + proxy_redirect "/{{ PROSPECTUS_S3_HOSTING_PREFIX }}" " "; + {% else %} try_files $uri $uri/ /bio/index.html; + {% endif %} } # preview pages are not cached, and have basic auth @@ -142,6 +186,16 @@ server { add_header 'Cache-Control' 'public, max-age=1800'; try_files $uri $uri/index.html =404; + location / { + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # proxy_redirect ensures redirects from s3 are rewritten + # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ + # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive + proxy_redirect "/{{ PROSPECTUS_S3_HOSTING_PREFIX }}" " "; + {% endif %} + } + # PROSPECTUS_STATIC_SITES will be a list of dictionaries which have a: # - router_path: The path you will go to on the router to access the content # - proxied_path: The path to proxy the requests to @@ -162,6 +216,6 @@ server { proxy_pass {{ static_site.proxied_path }}/$1/index.html; } - {% endfor %} + {% endfor -%} } diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 26d765dc9f0..286ac46992a 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -63,6 +63,7 @@ prospectus_use_python3: true PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS: False PROSPECTUS_ENABLE_NGINX: True PROSPECTUS_ENABLE_BUILD: True +PROSPECTUS_S3_HOSTING_PROXY_ENABLED: False PROSPECTUS_S3_UPLOAD_ENABLED: False PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 PROSPECTUS_S3_HOSTING_BUCKET: !!null From e13b87354687c92f85fb6a3d821801a6a9417547 Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Fri, 21 Jul 2023 14:01:08 -0400 Subject: [PATCH 458/664] chore: Add retry logic to prospectus npm build --- playbooks/roles/prospectus/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 98c6200f490..ab847d32ab4 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -192,6 +192,10 @@ chdir: "{{ prospectus_code_dir }}" environment: "{{ prospectus_env_vars }}" become_user: "{{ prospectus_user }}" + register: result + until: "result is not failed" + retries: 2 + delay: 10 tags: - install - install:app-requirements From 420bebddb2c887b13937b5a41f8004f9f5295bba Mon Sep 17 00:00:00 2001 From: SaadYousaf Date: Mon, 10 Jul 2023 12:51:30 +0500 Subject: [PATCH 459/664] feat: update Ruby version from 2.5.7 to 3.0.4 for cs_comment_service --- playbooks/roles/forum/defaults/main.yml | 4 +++- playbooks/roles/forum/tasks/deploy.yml | 11 ++++++++++- playbooks/roles/rbenv/defaults/main.yml | 8 +++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index d9ad4c19ac7..f377c9fcde9 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -7,6 +7,7 @@ forum_rbenv_dir: "{{ forum_app_dir }}" forum_rbenv_root: "{{ forum_app_dir }}/.rbenv" forum_rbenv_shims: "{{ forum_rbenv_root }}/shims" forum_rbenv_bin: "{{ forum_rbenv_root }}/bin" +forum_gemfile: 'Gemfile3' forum_supervisor_wrapper: "{{ forum_app_dir }}/forum-supervisor.sh" forum_gem_root: "{{ forum_rbenv_dir }}/.gem" forum_gem_bin: "{{ forum_gem_root }}/bin" @@ -63,6 +64,7 @@ FORUM_REBUILD_INDEX: false FORUM_ENABLE_MONGODB_INDEXES: false forum_base_env: &forum_base_env + BUNDLE_GEMFILE: "{{ forum_gemfile }}" RBENV_ROOT: "{{ forum_rbenv_root }}" GEM_HOME: "{{ forum_gem_root }}" GEM_PATH: "{{ forum_gem_root }}" @@ -100,7 +102,7 @@ devstack_forum_env: MONGOID_AUTH_MECH: "{{ FORUM_MONGO_AUTH_MECH }}" forum_user: "forum" -FORUM_RUBY_VERSION: "2.5.7" +FORUM_RUBY_VERSION: "3.0.4" forum_source_repo: "https://github.com/openedx/cs_comments_service.git" FORUM_VERSION: "master" diff --git a/playbooks/roles/forum/tasks/deploy.yml b/playbooks/roles/forum/tasks/deploy.yml index c3cab2c68db..48ce1f407cd 100644 --- a/playbooks/roles/forum/tasks/deploy.yml +++ b/playbooks/roles/forum/tasks/deploy.yml @@ -51,8 +51,17 @@ - install - install:code +- name: configure comments service bundle + shell: "bundle config set --local deployment 'true' chdir={{ forum_code_dir }}" + become_user: "{{ forum_user }}" + environment: "{{ forum_base_env }}" + notify: restart the forum service + tags: + - install + - install:app-requirements + - name: install comments service bundle - shell: "bundle install --deployment --path {{ forum_gem_root }} chdir={{ forum_code_dir }}" + shell: "bundle install chdir={{ forum_code_dir }}" become_user: "{{ forum_user }}" environment: "{{ forum_base_env }}" notify: restart the forum service diff --git a/playbooks/roles/rbenv/defaults/main.yml b/playbooks/roles/rbenv/defaults/main.yml index 68d402f6ab9..5350faac24b 100644 --- a/playbooks/roles/rbenv/defaults/main.yml +++ b/playbooks/roles/rbenv/defaults/main.yml @@ -1,12 +1,13 @@ --- RBENV_VERSION: 'v1.0.0' -RBENV_BUNDLER_VERSION: '1.11.2' -RBENV_RAKE_VERSION: '10.4.2' +RBENV_BUNDLER_VERSION: '2.3.4' +RBENV_RAKE_VERSION: '13.0.6' rbenv_root: "{{ rbenv_dir }}/.rbenv" rbenv_gem_root: "{{ rbenv_dir }}/.gem" rbenv_gem_bin: "{{ rbenv_gem_root }}/bin" -RBENV_RUBYGEMS_VERSION: '2.7.8' +rbenv_gemfile: 'Gemfile3' +RBENV_RUBYGEMS_VERSION: '3.2.33' rbenv_bin: "{{ rbenv_dir }}/.rbenv/bin" rbenv_shims: "{{ rbenv_root }}/shims" rbenv_path: "{{ rbenv_bin }}:{{ rbenv_shims }}:{{ rbenv_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -20,6 +21,7 @@ rbenv_debian_pkgs: - libxslt1-dev - zlib1g-dev rbenv_environment: + BUNDLE_GEMFILE: "{{ rbenv_gemfile }}" RBENV_ROOT: "{{ rbenv_root }}" GEM_ROOT: "{{ rbenv_gem_root }}" GEM_HOME: "{{ rbenv_gem_root }}" From 70c147f6dcecfe56d370b8928b857a8cf82882b5 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Wed, 26 Jul 2023 08:08:20 -0400 Subject: [PATCH 460/664] feat: retry some flaky tasks (#6965) --- playbooks/roles/insightvm_agent/tasks/main.yml | 1 + playbooks/roles/security/tasks/security-amazon.yml | 1 + playbooks/roles/security/tasks/security-ubuntu.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index 8179bf48801..3d798ac2327 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -10,6 +10,7 @@ stat: path: /etc/systemd/system/ir_agent.service register: r7_service + retries: 3 tags: - manage_rapid7_check_agent diff --git a/playbooks/roles/security/tasks/security-amazon.yml b/playbooks/roles/security/tasks/security-amazon.yml index 6085d38061e..1ef8d15e2db 100644 --- a/playbooks/roles/security/tasks/security-amazon.yml +++ b/playbooks/roles/security/tasks/security-amazon.yml @@ -30,6 +30,7 @@ - name: "Take security updates during ansible runs" command: "{{ item }}" when: SECURITY_UPGRADE_ON_ANSIBLE + retries: 3 with_items: - yum check-update --security - yum update --security -y diff --git a/playbooks/roles/security/tasks/security-ubuntu.yml b/playbooks/roles/security/tasks/security-ubuntu.yml index 8dc273dc902..ffaad59b3ce 100644 --- a/playbooks/roles/security/tasks/security-ubuntu.yml +++ b/playbooks/roles/security/tasks/security-ubuntu.yml @@ -24,6 +24,7 @@ - name: Disable unattended-upgrades if Xenial (16.04) command: "{{ item }}" when: ansible_distribution_release == 'xenial' and not SECURITY_UNATTENDED_UPGRADES + retries: 3 with_items: - "systemctl disable apt-daily.service" - "systemctl disable apt-daily.timer" From 539d116a195e72b7f604862038f48d7cb0f57eb1 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 26 Jul 2023 11:23:18 -0600 Subject: [PATCH 461/664] chore: add tasks yaml for ami builder --- playbooks/roles/aws_devstack/tasks/main.yml | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 playbooks/roles/aws_devstack/tasks/main.yml diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml new file mode 100644 index 00000000000..648fba67350 --- /dev/null +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -0,0 +1,38 @@ +- name: Remove dependencies that are no longer required + ansible.builtin.apt: + autoremove: yes + +- name: Run script to sync all repos + ansible.builtin.script: + cmd: /home/ubuntu/pull-all-repos-in-directory.sh + +- name: Mark all directories as safe for git on devstack + shell: "git config --global --add safe.directory '*'" + +- name: Run make on dev.pull.large-and-slow + make: + chdir: /home/ubuntu/edx-repos-devstack + target: dev.pull.large-and-slow + +- name: Run make on lms-shell + make: + chdir: /home/ubuntu/edx-repos-devstack + target: lms-shell + +- name: Make requirements + make: + chdir: /home/ubuntu/edx-repos/devstack + target: requirements + +- name: Run npm ci + community.general.npm: + ci: true + path: /home/ubuntu/edx-repos/devstack + +- name: Update paver assets + ansible.builtin.shell: + cmd: paver update_assets + chdir: /home/ubuntu/edx-repos/devstack + +-name : LMS migrate + command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate From 6c3d0014568478f1206775fcfe448d2ed71bd8bc Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 26 Jul 2023 12:22:45 -0600 Subject: [PATCH 462/664] chore: fix space in step name --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 648fba67350..0d200eeb242 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -34,5 +34,5 @@ cmd: paver update_assets chdir: /home/ubuntu/edx-repos/devstack --name : LMS migrate +- name : LMS migrate command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate From 4fa49ca67d15072d4a22e9e26d946a54dd711faa Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 26 Jul 2023 12:44:03 -0600 Subject: [PATCH 463/664] chore: syntax for ansible version and troubleshooting --- playbooks/roles/aws_devstack/tasks/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 0d200eeb242..e3bef4e28fd 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -1,26 +1,26 @@ -- name: Remove dependencies that are no longer required - ansible.builtin.apt: - autoremove: yes - +#- name: Remove dependencies that are no longer required +# apt: +# autoremove: yes +# - name: Run script to sync all repos ansible.builtin.script: cmd: /home/ubuntu/pull-all-repos-in-directory.sh - name: Mark all directories as safe for git on devstack - shell: "git config --global --add safe.directory '*'" + ansible.builtin.shell: "git config --global --add safe.directory '*'" - name: Run make on dev.pull.large-and-slow - make: + community.general.make: chdir: /home/ubuntu/edx-repos-devstack target: dev.pull.large-and-slow - name: Run make on lms-shell - make: + community.general.make: chdir: /home/ubuntu/edx-repos-devstack target: lms-shell - name: Make requirements - make: + community.general.make: chdir: /home/ubuntu/edx-repos/devstack target: requirements From 1f25ecb59a84ab0ee62f3efb3e9ac8599f480be6 Mon Sep 17 00:00:00 2001 From: Salman Nawaz Date: Mon, 31 Jul 2023 12:47:10 +0500 Subject: [PATCH 464/664] fix: update git hash for all PRs which has django42 support --- playbooks/roles/edxapp/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 1ffcf07ffc9..61159f0b863 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -573,13 +573,13 @@ EDXAPP_PRIVATE_REQUIREMENTS: extra_args: -e - name: git+https://github.com/openedx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock extra_args: -e - - name: git+https://github.com/openedx/xblock-submit-and-compare@8f0b3279f36e25aff8cce3f0b1793b9f19dc8729#egg=xblock-submit-and-compare + - name: git+https://github.com/openedx/xblock-submit-and-compare@39dd7f96b35affab5244880672da665a8adec332#egg=xblock-submit-and-compare extra_args: -e - name: git+https://github.com/openedx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response extra_args: -e - - name: git+https://github.com/openedx/xblock-sql-grader@5257b4a1aa07aa572fd3865647d91d0628f0cbd3#egg=xblock-sql-grader + - name: git+https://github.com/openedx/xblock-sql-grader@5ae84bb9389bccacfb19a2444ef405a8eae93b13#egg=xblock-sql-grader extra_args: -e - - name: git+https://github.com/openedx/xblock-image-modal@bef91413447e15570863ab08316a04c0b546b268#egg=xblock-image-modal + - name: git+https://github.com/openedx/xblock-image-modal@f280a623ae58886fb3d6b577802e9d59b04de585#egg=xblock-image-modal extra_args: -e # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks From 898d89b2b092a35187e0b266f4005b50f7a9c2bf Mon Sep 17 00:00:00 2001 From: stv Date: Mon, 31 Jul 2023 15:38:27 -0600 Subject: [PATCH 465/664] docs: swap example file storage _away_ from deprecated lib (#6953) The S3BotoStorage backend was deprecated in favor of the S3Boto3Storage backend. This change updates the example backend to reference the latter. Fixes: FAL-3431 Fixes: public-engineering/128 --- playbooks/roles/analytics_api/defaults/main.yml | 2 +- playbooks/roles/edxapp/defaults/main.yml | 2 +- playbooks/sample_vars/server_vars.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index 42a200e7d53..c2f05cc51f9 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -121,7 +121,7 @@ ANALYTICS_API_AGGREGATE_PAGE_SIZE: 10 # allowing temporary report downloads from secured S3 file locations. # # ANALYTICS_API_REPORT_DOWNLOAD_BACKEND: -# DEFAULT_FILE_STORAGE: 'storages.backends.s3boto.S3BotoStorage' +# DEFAULT_FILE_STORAGE: 'storages.backends.s3boto3.S3Boto3Storage' # AWS_ACCESS_KEY_ID: 'put-your-access-key-id-here' # AWS_SECRET_ACCESS_KEY: 'put-your-secret-access-key-here' # AWS_STORAGE_BUCKET_NAME: 'report-download-bucket' diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 1ffcf07ffc9..331a9d8acbf 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -789,7 +789,7 @@ EDXAPP_SOCIAL_SHARING_SETTINGS: #To use AWS S3 as your backend, you need different kwargs: # EDXAPP_PROFILE_IMAGE_BACKEND_CONFIG: -# class: storages.backends.s3boto.S3BotoStorage +# class: storages.backends.s3boto3.S3Boto3Storage # options: # location: path/to/images # Note: The location should not begin with a leading slash. # bucket: mybucket diff --git a/playbooks/sample_vars/server_vars.yml b/playbooks/sample_vars/server_vars.yml index 99d9e125aab..7e99adf960b 100644 --- a/playbooks/sample_vars/server_vars.yml +++ b/playbooks/sample_vars/server_vars.yml @@ -123,7 +123,7 @@ # ROOT_PATH: "edx-video-upload-pipeline/unprocessed" # #EDXAPP_PROFILE_IMAGE_BACKEND: -# class: storages.backends.s3boto.S3BotoStorage +# class: storages.backends.s3boto3.S3Boto3Storage # options: # location: /{{ ansible_ec2_public_ipv4 }} # bucket: your-profile-image-bucket From 8bae3a02ab71f7a68f2e56ac95e2372c586f8c77 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 1 Aug 2023 08:12:38 -0600 Subject: [PATCH 466/664] chore: add log file to pull repos for testing --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index e3bef4e28fd..ddd5c23ab25 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -4,7 +4,7 @@ # - name: Run script to sync all repos ansible.builtin.script: - cmd: /home/ubuntu/pull-all-repos-in-directory.sh + cmd: /home/ubuntu/pull-all-repos-in-directory.sh > pullrepos.log - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 168af76d8bd4d49bde3c45d7147a401ec8b61eca Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 1 Aug 2023 09:23:41 -0600 Subject: [PATCH 467/664] add full path for log file --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index ddd5c23ab25..cf147575392 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -4,7 +4,7 @@ # - name: Run script to sync all repos ansible.builtin.script: - cmd: /home/ubuntu/pull-all-repos-in-directory.sh > pullrepos.log + cmd: /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 55e1d7705f6c3c85e7749c9c3e1733701eab1cb2 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 3 Aug 2023 13:53:49 -0400 Subject: [PATCH 468/664] fix: Add retries for a few python requirements installations (#6978) We've intermittently seen some connection failures to PyPI. Ideally all such tasks could be retried automatically, but Ansible doesn't provide for that. Making each task retry requires 3-4 lines, so just change a few of the more likely-to-fail ones. --- playbooks/roles/edxapp/tasks/deploy.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index c322f9fa38c..5e148e6b0ae 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -160,6 +160,10 @@ environment: "{{ edxapp_environment }}" when: item.stat.exists with_items: "{{ python_requirement_files.results }}" + register: edxapp_install_python_reqs + until: edxapp_install_python_reqs is succeeded + retries: 5 + delay: 15 tags: - install - install:app-requirements @@ -186,6 +190,10 @@ environment: GIT_SSH: "{{ edxapp_git_ssh }}" when: EDXAPP_INSTALL_PRIVATE_REQUIREMENTS + register: edxapp_install_private_python_reqs + until: edxapp_install_private_python_reqs is succeeded + retries: 5 + delay: 15 tags: - install - install:app-requirements @@ -200,6 +208,10 @@ state: present with_items: "{{ EDXAPP_EXTRA_REQUIREMENTS }}" become_user: "{{ edxapp_user }}" + register: edxapp_install_extra_python_reqs + until: edxapp_install_extra_python_reqs is succeeded + retries: 5 + delay: 15 tags: - install - install:app-requirements From 098818cebc97a618c8091a696d9a73241ea68086 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 3 Aug 2023 13:57:59 -0400 Subject: [PATCH 469/664] feat: Remove now-unneeded call to seed permissions for demo course forums (#6974) This is now redundant with the earlier `import` call, which at this point goes ahead and does the seeding. One of these is probably used by sandboxes, and the other is probably not, but both are referenced by `ansible-provision.sh`. Might as well remove from both so that at least they're consistent with devstack, and so that this is not propagated forward to whatever eventually replaces the sandbox setup scripts. See https://github.com/openedx/devstack/issues/1129 for details. --- playbooks/roles/demo/tasks/deploy.yml | 6 ------ util/jenkins/demo-course-provisioner.sh | 3 --- 2 files changed, 9 deletions(-) diff --git a/playbooks/roles/demo/tasks/deploy.yml b/playbooks/roles/demo/tasks/deploy.yml index 0c427c3d8d8..15f751481ef 100644 --- a/playbooks/roles/demo/tasks/deploy.yml +++ b/playbooks/roles/demo/tasks/deploy.yml @@ -46,9 +46,3 @@ with_items: - "{{ demo_test_and_staff_users }}" when: demo_checkout.changed - -- name: seed the forums for the demo course - shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} seed_permissions_roles {{ demo_course_id }}" - args: - chdir: "{{ demo_edxapp_code_dir }}" - when: demo_checkout.changed diff --git a/util/jenkins/demo-course-provisioner.sh b/util/jenkins/demo-course-provisioner.sh index 0715f1f30d1..651444c6850 100644 --- a/util/jenkins/demo-course-provisioner.sh +++ b/util/jenkins/demo-course-provisioner.sh @@ -32,8 +32,5 @@ for user in honor audit verified staff ; do # Enroll users in the demo course docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms enroll_user_in_course -e \$email -c course-v1:edX+DemoX+Demo_Course done - -# Seed forums for the demo course -docker run --network=host --rm -u='www-data' -e NO_PREREQ_INSTALL="1" -e SKIP_WS_MIGRATIONS="1" -e LMS_CFG=/edx/etc/lms.yml -e DJANGO_SETTINGS_MODULE=lms.envs.docker-production -e SERVICE_VARIANT=lms -e EDX_PLATFORM_SETTINGS=docker-production -v /edx/etc/lms.yml:/edx/etc/lms.yml -v /edx/var/edx-themes:/edx/var/edx-themes -v /edx/var/edxapp:/edx/var/edxapp -v /var/run/mysqld/mysqld.sock:/var/run/mysqld/mysqld.sock ${app_repo}:latest python3 manage.py lms seed_permissions_roles course-v1:edX+DemoX+Demo_Course EOF } From bee7eaa9b8753c173ba3adcd5807171ef56f4325 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Thu, 3 Aug 2023 11:49:57 -0400 Subject: [PATCH 470/664] fix: Update retry logic for certain ansible tasks. When we added retry logic, we missed that we also needed to register a variable and add an until condition to ensure that retries happen. This should ensure that this matches. https://github.com/edx/edx-arch-experiments/issues/359 --- playbooks/roles/insightvm_agent/tasks/main.yml | 1 + playbooks/roles/security/tasks/security-amazon.yml | 2 ++ playbooks/roles/security/tasks/security-ubuntu.yml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index 3d798ac2327..d4703676360 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -11,6 +11,7 @@ path: /etc/systemd/system/ir_agent.service register: r7_service retries: 3 + until: r7_service is succeeded tags: - manage_rapid7_check_agent diff --git a/playbooks/roles/security/tasks/security-amazon.yml b/playbooks/roles/security/tasks/security-amazon.yml index 1ef8d15e2db..513c9b4f343 100644 --- a/playbooks/roles/security/tasks/security-amazon.yml +++ b/playbooks/roles/security/tasks/security-amazon.yml @@ -30,7 +30,9 @@ - name: "Take security updates during ansible runs" command: "{{ item }}" when: SECURITY_UPGRADE_ON_ANSIBLE + register: result_amazon retries: 3 + until: result_amazon is succeeded with_items: - yum check-update --security - yum update --security -y diff --git a/playbooks/roles/security/tasks/security-ubuntu.yml b/playbooks/roles/security/tasks/security-ubuntu.yml index ffaad59b3ce..3352129bb8b 100644 --- a/playbooks/roles/security/tasks/security-ubuntu.yml +++ b/playbooks/roles/security/tasks/security-ubuntu.yml @@ -24,7 +24,9 @@ - name: Disable unattended-upgrades if Xenial (16.04) command: "{{ item }}" when: ansible_distribution_release == 'xenial' and not SECURITY_UNATTENDED_UPGRADES + register: result_ubuntu retries: 3 + until: result_ubuntu is succeeded with_items: - "systemctl disable apt-daily.service" - "systemctl disable apt-daily.timer" From b231f245da6fdf70af6c9f759a5273d85eec9ee4 Mon Sep 17 00:00:00 2001 From: Syed Awais Ali Date: Wed, 9 Aug 2023 12:22:37 +0500 Subject: [PATCH 471/664] chore: upgrade mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index bb416d8579c..f79d2ac56e0 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "11.15.1.7500-1" + version: "13.3.0.8373-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From 26e29a0fadcc4764047d84e71b1847248ea44b76 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 14 Aug 2023 15:52:39 -0400 Subject: [PATCH 472/664] fix: Stop insecure redirect when S3 hosting off This code was meant to be a no-op with PROSPECTUS_S3_HOSTING_PROXY_ENABLED off, but instead it adds location / {} to nginx, which causes nginx to produce redirects to http://hostname/url/ if a trailing slash is omitted. --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 7b340c68def..3d9967f90f2 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -186,15 +186,15 @@ server { add_header 'Cache-Control' 'public, max-age=1800'; try_files $uri $uri/index.html =404; - location / { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + location / { proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive proxy_redirect "/{{ PROSPECTUS_S3_HOSTING_PREFIX }}" " "; - {% endif %} } + {% endif %} # PROSPECTUS_STATIC_SITES will be a list of dictionaries which have a: # - router_path: The path you will go to on the router to access the content From 0f429dcf00648df1ef286f08be24f8f5542278e9 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 15 Aug 2023 16:27:55 +0500 Subject: [PATCH 473/664] chore: Update main.yml replacing hash with pypi versions. --- playbooks/roles/edxapp/defaults/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 61159f0b863..90206890cad 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -573,14 +573,11 @@ EDXAPP_PRIVATE_REQUIREMENTS: extra_args: -e - name: git+https://github.com/openedx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock extra_args: -e - - name: git+https://github.com/openedx/xblock-submit-and-compare@39dd7f96b35affab5244880672da665a8adec332#egg=xblock-submit-and-compare - extra_args: -e - name: git+https://github.com/openedx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response extra_args: -e - - name: git+https://github.com/openedx/xblock-sql-grader@5ae84bb9389bccacfb19a2444ef405a8eae93b13#egg=xblock-sql-grader - extra_args: -e - - name: git+https://github.com/openedx/xblock-image-modal@f280a623ae58886fb3d6b577802e9d59b04de585#egg=xblock-image-modal - extra_args: -e + - name: xblock-submit-and-compare==1.2.0 + - name: xblock-sql-grader==0.4.0 + - name: openedx-xblock-image-modal==3.1.0 # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e From fa4a57baa66e3338a90b95b6e56063e400075375 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 16 Aug 2023 00:48:53 +0500 Subject: [PATCH 474/664] chore: upgrading `edx-event-routing-backends` to latest version. --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 46f46545535..e1c63ad063a 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -585,7 +585,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/open-craft/lx-pathway-plugin.git@ba1d470217cd5908cbd8b56075628bd4eacf7b39#egg=lx-pathway-plugin extra_args: -e # Caliper and xAPI event routing plugin - - name: edx-event-routing-backends==4.1.1 + - name: edx-event-routing-backends==5.5.6 # List of custom middlewares that should be used in edxapp to process # incoming HTTP resquests. Should be a list of plain strings that fully From 17d8d9715f99a4bda6710da0879913fcc1420fb4 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Wed, 16 Aug 2023 13:38:28 +0500 Subject: [PATCH 475/664] chore: add pymemcache backend --- playbooks/roles/edxapp/defaults/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 46f46545535..21c618a9fc9 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -150,7 +150,7 @@ EDXAPP_LOG_LEVEL: 'INFO' EDXAPP_MEMCACHE: [ 'localhost:11211' ] EDXAPP_CACHE_COURSE_STRUCTURE_MEMCACHE: "{{ EDXAPP_MEMCACHE }}" -EDXAPP_CACHE_BACKEND: 'django.core.cache.backends.memcached.MemcachedCache' +EDXAPP_CACHE_BACKEND: 'django.core.cache.backends.memcached.PyMemcacheCache' EDXAPP_COMMENTS_SERVICE_URL: 'http://localhost:18080' # EDXAPP_COMMENTS_SERVICE_KEY must match FORUM_API_KEY EDXAPP_COMMENTS_SERVICE_KEY: 'password' @@ -1328,6 +1328,10 @@ edxapp_generic_auth_config: &edxapp_generic_auth generic_cache_config: &default_generic_cache BACKEND: "{{ EDXAPP_CACHE_BACKEND }}" KEY_FUNCTION: 'common.djangoapps.util.memcache.safe_key' + OPTIONS: + no_delay: true + ignore_exc: true + use_pooling: true edxapp_revisions_config: EDX_PLATFORM_REVISION: "{{ EDX_PLATFORM_VERSION }}" From 96a390dbf84686efd21e82a5e1f48f1d4f2e5fa3 Mon Sep 17 00:00:00 2001 From: "Glenn R. Martin" Date: Wed, 16 Aug 2023 13:59:10 -0400 Subject: [PATCH 476/664] feat: Payment MFE New Configuration Variables for Commerce Coordinator This is part of THES-261 --- playbooks/roles/payment/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/payment/defaults/main.yml b/playbooks/roles/payment/defaults/main.yml index 70512b3827f..466d339fa0d 100644 --- a/playbooks/roles/payment/defaults/main.yml +++ b/playbooks/roles/payment/defaults/main.yml @@ -2,3 +2,5 @@ payment_env_extra: APPLE_PAY_COUNTRY_CODE: "{{ PAYMENT_APPLE_PAY_COUNTRY_CODE }}" STRIPE_PUBLISHABLE_KEY: "{{ PAYMENT_STRIPE_PUBLISHABLE_KEY }}" STRIPE_RESPONSE_URL: "{{ PAYMENT_STRIPE_RESPONSE_URL }}" + WAFFLE_FLAGS: "{{ PAYMENT_WAFFLE_FLAGS|default(omit) }}" + COMMERCE_COORDINATOR_BASE_URL: "{{ PAYMENT_COMMERCE_COORDINATOR_BASE_URL }}" From 521bf04e32251a5ab0760f97cd111c1c9047e941 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Thu, 17 Aug 2023 17:13:46 +0500 Subject: [PATCH 477/664] chore: upgrading `xblock-qualtrics-survey` to latest version. --- playbooks/roles/edxapp/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index aa4462a44fb..d37724d75da 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -569,8 +569,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/open-craft/xblock-activetable.git@d3fb772435c382b59293e4e688a6a3096c4f6fd7#egg=activetable-xblock extra_args: -e # Stanford-developed XBlocks (technically unsupported, but here to ease migration of courses from Lagunita) - - name: git+https://github.com/openedx/xblock-qualtrics-survey.git@02d87f567ec2af4579642cf795bb851507d6edf9#egg=xblock_qualtrics_survey - extra_args: -e + - name: xblock-qualtrics-survey==1.3.0 - name: git+https://github.com/openedx/xblock-in-video-quiz.git@a703acd9ef82434fc7ca2bc230496f45a584bb9a#egg=invideoquiz-xblock extra_args: -e - name: git+https://github.com/openedx/xblock-free-text-response@83a389e0a4b0a464e5d1e4a4a201678aed5eee9a#egg=xblock-free-text-response From 1603653d0df52ea82ec920cfbeb500060904a99d Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 18 Aug 2023 11:10:44 +0500 Subject: [PATCH 478/664] fix: refactor condition for browsers role task using multiverse_installed.changed attribute --- playbooks/roles/browsers/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/browsers/tasks/main.yml b/playbooks/roles/browsers/tasks/main.yml index f011ebce8a1..124dc6d8419 100644 --- a/playbooks/roles/browsers/tasks/main.yml +++ b/playbooks/roles/browsers/tasks/main.yml @@ -26,7 +26,7 @@ - name: Update cache when adding multiverse repos apt: update_cache: true - when: multiverse_installed | changed + when: multiverse_installed.changed tags: - install - install:system-requirements From 21421ee0347af6b8f431eaafb342805348710309 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Fri, 18 Aug 2023 09:30:26 -0400 Subject: [PATCH 479/664] fix: add retries for ubuntu security updates (#6988) --- playbooks/roles/security/tasks/security-ubuntu.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/security/tasks/security-ubuntu.yml b/playbooks/roles/security/tasks/security-ubuntu.yml index 3352129bb8b..1929c59688f 100644 --- a/playbooks/roles/security/tasks/security-ubuntu.yml +++ b/playbooks/roles/security/tasks/security-ubuntu.yml @@ -69,3 +69,6 @@ with_items: - unattended-upgrade --dry-run - unattended-upgrade + register: ubuntu_security + retries: 3 + until: ubuntu_security is succeeded From c3ea23e6c45ddf74622c861b7cfc61b59b9c5bb6 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Mon, 21 Aug 2023 16:29:06 -0400 Subject: [PATCH 480/664] docs: comment on EDXAPP_PRIVATE_REQUIREMENTS (#6991) Add a note to clarify that EDXAPP_PRIVATE_REQUIREMENTS contains edx.org specific details, which is not ideal. The plan is to ultimately retire this repo according to the DEPR: https://github.com/openedx/public-engineering/issues/51 --- playbooks/roles/edxapp/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index d37724d75da..5fc3802f1b0 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -540,6 +540,9 @@ EDXAPP_EXTRA_REQUIREMENTS: [] # Example: # EDXAPP_PRIVATE_REQUIREMENTS: # - name: git+https://git.myproject.org/MyProject#egg=MyProject +# Note: This list contains edx.org specific dependencies, even though this is +# a public repo. The plan is to phase this out along with the rest of this +# repo as part of the DEPR https://github.com/openedx/public-engineering/issues/51. EDXAPP_PRIVATE_REQUIREMENTS: # For Harvard courses: - name: xblock-problem-builder==5.1.3 From 5e79d5d38f5bc9e3fef14c27f9fb9bfa91af6eaa Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 17 Aug 2023 15:32:52 -0400 Subject: [PATCH 481/664] feat: Allow running prospectus playbook in parts By default the playbook should run in order as normal. This change provides 3 new variables which allows a build pipeline to run the prospectus build in 3 parts. 1. PROSPECTUS_ENABLE_PRE_BUILD: Runs everything before `npm run build` 2. PROSPECTUS_ENABLE_NPM_BUILD: Runs `npm run build` 3. PROSPECTUS_ENABLE_PRE_BUILD: Runs everything after `npm run build` This will allow us to run `npm run build` directly via ssh so we can see error output without waiting for it to finish and return to ansible. This will also allow time for the nginx build to run in parallel as soon as the redirects file has been generated. --- playbooks/prospectus.yml | 9 +- playbooks/roles/prospectus/defaults/main.yml | 5 + playbooks/roles/prospectus/tasks/main.yml | 504 +++++++++---------- 3 files changed, 256 insertions(+), 262 deletions(-) diff --git a/playbooks/prospectus.yml b/playbooks/prospectus.yml index 61e5054cf10..eb9ebfbf386 100644 --- a/playbooks/prospectus.yml +++ b/playbooks/prospectus.yml @@ -14,9 +14,10 @@ PROSPECTUS_SANDBOX_BUILD: FALSE roles: - role: aws - when: COMMON_ENABLE_AWS_ROLE + when: COMMON_ENABLE_AWS_ROLE|bool and PROSPECTUS_ENABLE_PRE_BUILD|bool - role: automated AUTOMATED_USERS: "{{ PROSPECTUS_AUTOMATED_USERS | default({}) }}" + when: PROSPECTUS_ENABLE_PRE_BUILD|bool - role: prospectus when: PROSPECTUS_ENABLE_BUILD|bool - role: nginx @@ -28,8 +29,8 @@ - prospectus PROSPECTUS_NGINX_PORT: 8000 - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER + when: COMMON_ENABLE_SPLUNKFORWARDER|bool and PROSPECTUS_ENABLE_POST_BUILD|bool - role: newrelic_infrastructure - when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE|bool and PROSPECTUS_ENABLE_POST_BUILD|bool - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT + when: COMMON_ENABLE_INSIGHTVM_AGENT|bool and PROSPECTUS_ENABLE_POST_BUILD|bool diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 286ac46992a..493a8da90ea 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -63,6 +63,11 @@ prospectus_use_python3: true PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS: False PROSPECTUS_ENABLE_NGINX: True PROSPECTUS_ENABLE_BUILD: True + +PROSPECTUS_ENABLE_PRE_BUILD: True +PROSPECTUS_ENABLE_NPM_BUILD: True +PROSPECTUS_ENABLE_POST_BUILD: True + PROSPECTUS_S3_HOSTING_PROXY_ENABLED: False PROSPECTUS_S3_UPLOAD_ENABLED: False PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index 98c6200f490..f0f57a006ca 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -1,192 +1,207 @@ -# Remove all of the old code -- name: Remove read-only ssh key for the prospectus repo - file: - path: "{{ prospectus_git_identity }}" - state: absent - when: PROSPECTUS_GIT_IDENTITY != "none" - -- name: check if cache dir exists - stat: - path: "{{ prospectus_code_dir }}/.cache" - register: register_cache_dir - -- name: check if data dir exists - stat: - path: "{{ PROSPECTUS_DATA_DIR }}" - register: register_data_dir - -- name: create prospectus temp cache folder - file: - path: "/tmp/cache-data" - state: directory - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - -- name: move cache dir to temp - command: mv {{ prospectus_code_dir }}/.cache /tmp/cache-data/ - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - -- name: Remove old git repo - file: - state: absent - path: "{{ prospectus_code_dir }}/" - -- name: Remove old app repo - file: - state: absent - path: "{{ prospectus_app_dir }}" - -- name: Remove data directory - file: - state: absent - path: "{{ PROSPECTUS_DATA_DIR }}" - when: not PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool - -- name: Create prospectus app folder - file: - path: "{{ prospectus_app_dir }}" - state: directory - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - -# This key is only needed if you are pulling down a private -# prospectus repo -- name: Install read-only ssh key for the prospectus repo - copy: - content: "{{ PROSPECTUS_GIT_IDENTITY }}" - dest: "{{ prospectus_git_identity }}" - force: yes - owner: "{{ prospectus_user }}" - mode: "0600" - when: PROSPECTUS_GIT_IDENTITY != "none" - -- name: "Checkout prospectus repo into {{ prospectus_code_dir }} with key" - git: - dest: "{{ prospectus_code_dir }}" - repo: "{{ prospectus_repo }}" - version: "{{ PROSPECTUS_VERSION }}" - accept_hostkey: yes - key_file: "{{ prospectus_git_identity }}" - become_user: "{{ prospectus_user }}" - register: prospectus_checkout_with_key - when: PROSPECTUS_GIT_IDENTITY != "none" - -- name: Checkout prospectus repo into {{ prospectus_code_dir }} without key - git: - dest: "{{ prospectus_code_dir }}" - repo: "{{ prospectus_repo }}" - version: "{{ PROSPECTUS_VERSION }}" - accept_hostkey: yes - become_user: "{{ prospectus_user }}" - register: prospectus_checkout_without_key - when: PROSPECTUS_GIT_IDENTITY == "none" - -- name: move cache dir to {{ prospectus_code_dir }} - command: mv /tmp/cache-data/.cache "{{ prospectus_code_dir }}/" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists - -- name: create prospectus public folder - file: - path: "{{ prospectus_code_dir }}/public" - state: directory - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists - -- name: move data dir to {{ prospectus_code_dir }}/public - shell: "mv {{ PROSPECTUS_DATA_DIR }}/* {{ prospectus_code_dir }}/public/" - when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists - -- name: install python3.8 - apt: - pkg: - - python3.8-dev - - python3.8-distutils - update_cache: yes - register: install_pkgs - until: install_pkgs is success - retries: 10 - delay: 5 - when: prospectus_use_python3 - tags: - - install - - install:system-requirements - -- name: Add prospectus configuration file - template: - src: ".env.environment.j2" - dest: "{{ prospectus_code_dir }}/config/.env.keys" - mode: "0644" - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - -- name: Install nodeenv - apt: - name: nodeenv - update_cache: yes - become_user: "{{ prospectus_user }}" - environment: "{{ prospectus_env_vars }}" - register: install_pkgs - until: install_pkgs is success - retries: 10 - delay: 5 - tags: - - install - - install:system-requirements - -# Install node -- name: Create nodeenv - shell: "nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" - become_user: "{{ prospectus_user }}" - environment: "{{ prospectus_env_vars }}" - tags: - - install - - install:system-requirements - -- name: copy the template to the desired location - template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: "{{ item.owner }}" - group: "{{ item.group }}" - mode: "{{ item.mode }}" - with_items: - - { src: 'prospectus_env.j2', dest: '{{ prospectus_app_dir }}/prospectus_env', owner: '{{ prospectus_user }}', group: '{{ prospectus_user }}', mode: '0644' } - -# Set the npm registry -# This needs to be done as root since npm is weird about -# chown - https://github.com/npm/npm/issues/3565 -- name: Set the npm registry - shell: "npm config set registry '{{ COMMON_NPM_MIRROR_URL }}'" - args: - creates: "{{ prospectus_code_dir }}/.npmrc" - environment: "{{ prospectus_env_vars }}" - tags: - - install - - install:app-requirements - -# Set the npm registry permissions -- name: Set the npm registry permissions - file: - path: "{{ prospectus_code_dir }}/.npmrc" - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - tags: - - install - - install:app-requirements - -# Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json -- name: install node dependencies - shell: "{{ prospectus_nodeenv_bin }}/npm install --unsafe-perm=true --allow-root" - args: - chdir: "{{ prospectus_code_dir }}" - environment: "{{ prospectus_env_vars }}" - become_user: "{{ prospectus_user }}" - tags: - - install - - install:app-requirements +- name: Setup Prospectus Build Environment + when: PROSPECTUS_ENABLE_PRE_BUILD|bool + block: + # Remove all of the old code + - name: Remove read-only ssh key for the prospectus repo + file: + path: "{{ prospectus_git_identity }}" + state: absent + when: PROSPECTUS_GIT_IDENTITY != "none" + + - name: check if cache dir exists + stat: + path: "{{ prospectus_code_dir }}/.cache" + register: register_cache_dir + + - name: check if data dir exists + stat: + path: "{{ PROSPECTUS_DATA_DIR }}" + register: register_data_dir + + - name: create prospectus temp cache folder + file: + path: "/tmp/cache-data" + state: directory + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists + + - name: move cache dir to temp + command: mv {{ prospectus_code_dir }}/.cache /tmp/cache-data/ + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists + + - name: Remove old git repo + file: + state: absent + path: "{{ prospectus_code_dir }}/" + + - name: Remove old app repo + file: + state: absent + path: "{{ prospectus_app_dir }}" + + - name: Remove data directory + file: + state: absent + path: "{{ PROSPECTUS_DATA_DIR }}" + when: not PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool + + - name: Create prospectus app folder + file: + path: "{{ prospectus_app_dir }}" + state: directory + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + + # This key is only needed if you are pulling down a private + # prospectus repo + - name: Install read-only ssh key for the prospectus repo + copy: + content: "{{ PROSPECTUS_GIT_IDENTITY }}" + dest: "{{ prospectus_git_identity }}" + force: yes + owner: "{{ prospectus_user }}" + mode: "0600" + when: PROSPECTUS_GIT_IDENTITY != "none" + + - name: "Checkout prospectus repo into {{ prospectus_code_dir }} with key" + git: + dest: "{{ prospectus_code_dir }}" + repo: "{{ prospectus_repo }}" + version: "{{ PROSPECTUS_VERSION }}" + accept_hostkey: yes + key_file: "{{ prospectus_git_identity }}" + become_user: "{{ prospectus_user }}" + register: prospectus_checkout_with_key + when: PROSPECTUS_GIT_IDENTITY != "none" + + - name: Checkout prospectus repo into {{ prospectus_code_dir }} without key + git: + dest: "{{ prospectus_code_dir }}" + repo: "{{ prospectus_repo }}" + version: "{{ PROSPECTUS_VERSION }}" + accept_hostkey: yes + become_user: "{{ prospectus_user }}" + register: prospectus_checkout_without_key + when: PROSPECTUS_GIT_IDENTITY == "none" + + - name: move cache dir to {{ prospectus_code_dir }} + command: mv /tmp/cache-data/.cache "{{ prospectus_code_dir }}/" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_cache_dir.stat.exists + + - name: create prospectus public folder + file: + path: "{{ prospectus_code_dir }}/public" + state: directory + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists + + - name: move data dir to {{ prospectus_code_dir }}/public + shell: "mv {{ PROSPECTUS_DATA_DIR }}/* {{ prospectus_code_dir }}/public/" + when: PROSPECTUS_RETAIN_CACHE_PUBLIC_DIRS|bool and register_data_dir.stat.exists + + - name: install python3.8 + apt: + pkg: + - python3.8-dev + - python3.8-distutils + update_cache: yes + register: install_pkgs + until: install_pkgs is success + retries: 10 + delay: 5 + when: prospectus_use_python3 + tags: + - install + - install:system-requirements + + - name: Add prospectus configuration file + template: + src: ".env.environment.j2" + dest: "{{ prospectus_code_dir }}/config/.env.keys" + mode: "0644" + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + + - name: Install nodeenv + apt: + name: nodeenv + update_cache: yes + become_user: "{{ prospectus_user }}" + environment: "{{ prospectus_env_vars }}" + register: install_pkgs + until: install_pkgs is success + retries: 10 + delay: 5 + tags: + - install + - install:system-requirements + + # Install node + - name: Create nodeenv + shell: "nodeenv {{ prospectus_nodeenv_dir }} --node={{ PROSPECTUS_NODE_VERSION }} --prebuilt --force" + become_user: "{{ prospectus_user }}" + environment: "{{ prospectus_env_vars }}" + tags: + - install + - install:system-requirements + + - name: copy the template to the desired location + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ item.owner }}" + group: "{{ item.group }}" + mode: "{{ item.mode }}" + with_items: + - { src: 'prospectus_env.j2', dest: '{{ prospectus_app_dir }}/prospectus_env', owner: '{{ prospectus_user }}', group: '{{ prospectus_user }}', mode: '0644' } + + # Set the npm registry + # This needs to be done as root since npm is weird about + # chown - https://github.com/npm/npm/issues/3565 + - name: Set the npm registry + shell: "npm config set registry '{{ COMMON_NPM_MIRROR_URL }}'" + args: + creates: "{{ prospectus_code_dir }}/.npmrc" + environment: "{{ prospectus_env_vars }}" + tags: + - install + - install:app-requirements + + # Set the npm registry permissions + - name: Set the npm registry permissions + file: + path: "{{ prospectus_code_dir }}/.npmrc" + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + tags: + - install + - install:app-requirements + + # Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json + - name: install node dependencies + shell: "{{ prospectus_nodeenv_bin }}/npm install --unsafe-perm=true --allow-root" + args: + chdir: "{{ prospectus_code_dir }}" + environment: "{{ prospectus_env_vars }}" + become_user: "{{ prospectus_user }}" + tags: + - install + - install:app-requirements + - name: Install pngquant + apt: + name: "pngquant" + update_cache: yes + register: install_pkgs + until: install_pkgs is success + retries: 10 + delay: 5 + tags: + - install + - install:system-requirements # Install with the shell command instead of the ansible npm module so we don't accidentally re-write package.json - name: run prospectus build + when: PROSPECTUS_ENABLE_NPM_BUILD|bool shell: "{{ prospectus_nodeenv_bin }}/npm run build" args: chdir: "{{ prospectus_code_dir }}" @@ -196,75 +211,48 @@ - install - install:app-requirements -- name: Install pngquant - apt: - name: "pngquant" - update_cache: yes - register: install_pkgs - until: install_pkgs is success - retries: 10 - delay: 5 - tags: - - install - - install:system-requirements - -- name: Compress images - shell: "find public -name '*.png' -exec pngquant --skip-if-larger --quality 50-75 --ext .png --force -- {} \\;" - args: - chdir: "{{ prospectus_code_dir }}" - become_user: "{{ prospectus_user }}" - -# Copy over the target from the previous build to where it needs to be -- name: Create data folder - file: - path: "{{ PROSPECTUS_DATA_DIR }}" - state: directory - owner: "{{ prospectus_user }}" - group: "{{ prospectus_user }}" - -- name: Move prospectus public folder to var folder - shell: "mv {{ prospectus_code_dir }}/public/* {{ PROSPECTUS_DATA_DIR }}" - -- name: Set prospectus_s3_prefix to prospectus_checkout_without_key.after - ignore_errors: true - set_fact: - prospectus_s3_prefix: "{{ prospectus_checkout_without_key.after }}" - when: PROSPECTUS_GIT_IDENTITY == "none" - -- name: Set prospectus_s3_prefix to prospectus_checkout_with_key.after - ignore_errors: true - set_fact: - prospectus_s3_prefix: "{{ prospectus_checkout_with_key.after }}-{{ ansible_date_time.epoch }}" - when: PROSPECTUS_GIT_IDENTITY != "none" - -- name: Set prospectus_s3_prefix to PROSPECTUS_S3_HOSTING_PREFIX - ignore_errors: true - set_fact: - prospectus_s3_prefix: "{{ PROSPECTUS_S3_HOSTING_PREFIX }}" - when: PROSPECTUS_S3_HOSTING_PREFIX != None - -- name: Set Prospectus S3 max concurrency - ignore_errors: true - become_user: "{{ prospectus_user }}" - shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS }}' - when: PROSPECTUS_S3_UPLOAD_ENABLED|bool - tags: - - install - - install:system-requirements - -- name: Upload prospectus to S3 - ignore_errors: true - become_user: "{{ prospectus_user }}" - shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ prospectus_s3_prefix }}' - when: PROSPECTUS_S3_UPLOAD_ENABLED|bool - tags: - - install - - install:system-requirements - -- name: Download prospectus redirects to GoCD - ignore_errors: true - when: PROSPECTUS_S3_UPLOAD_ENABLED|bool - fetch: - src: "{{ prospectus_redirect_file }}" - dest: "{{ artifact_path }}/prospectus-redirects.conf" - flat: yes +- name: Post process prospectus gatsby build + when: PROSPECTUS_ENABLE_POST_BUILD|bool + block: + - name: Compress images + shell: "find public -name '*.png' -exec pngquant --skip-if-larger --quality 50-75 --ext .png --force -- {} \\;" + args: + chdir: "{{ prospectus_code_dir }}" + become_user: "{{ prospectus_user }}" + + # Copy over the target from the previous build to where it needs to be + - name: Create data folder + file: + path: "{{ PROSPECTUS_DATA_DIR }}" + state: directory + owner: "{{ prospectus_user }}" + group: "{{ prospectus_user }}" + + - name: Move prospectus public folder to var folder + shell: "mv {{ prospectus_code_dir }}/public/* {{ PROSPECTUS_DATA_DIR }}" + + - name: Set Prospectus S3 max concurrency + ignore_errors: true + become_user: "{{ prospectus_user }}" + shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS }}' + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool + tags: + - install + - install:system-requirements + + - name: Upload prospectus to S3 + ignore_errors: true + become_user: "{{ prospectus_user }}" + shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ PROSPECTUS_S3_HOSTING_PREFIX | default(PROSPECTUS_VERSION, true) }}' + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool + tags: + - install + - install:system-requirements + + - name: Download prospectus redirects to GoCD + ignore_errors: true + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool + fetch: + src: "{{ prospectus_redirect_file }}" + dest: "{{ artifact_path }}/prospectus-redirects.conf" + flat: yes From 64e3229dcde34373e496f48bae364fe302813c85 Mon Sep 17 00:00:00 2001 From: "Glenn R. Martin" Date: Thu, 24 Aug 2023 17:51:23 +0000 Subject: [PATCH 482/664] fix: Payment MFE GIT Version --- playbooks/roles/payment/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/payment/tasks/main.yml b/playbooks/roles/payment/tasks/main.yml index 85526cd3392..bab877de787 100644 --- a/playbooks/roles/payment/tasks/main.yml +++ b/playbooks/roles/payment/tasks/main.yml @@ -3,3 +3,4 @@ name: mfe vars: MFE_ENVIRONMENT_EXTRA: '{{ payment_env_extra | default(MFE_DEPLOY_ENVIRONMENT_EXTRA) }}' + MFE_VERSION: "{{ PAYMENT_MFE_VERSION | default('master') }}" From 4038060fa452336c9214b55d2938aa1c54e9194a Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 24 Aug 2023 14:39:13 -0600 Subject: [PATCH 483/664] fix: modify logic for ppa version exclusion --- playbooks/roles/common/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index b406c55a021..655dfd5ff44 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -74,13 +74,13 @@ state: "present" when: > ansible_distribution in common_debian_variants and - (ansible_distribution_release != 'bionic' and ansible_distribution_release != 'focal') + (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty' or ansible_distribution_release == 'xenial') - name: Update expired apt keys shell: apt-key adv --recv-keys --keyserver {{ COMMON_EDX_PPA_KEY_SERVER }} {{ COMMON_EDX_PPA_KEY_ID }} when: > ansible_distribution in common_debian_variants and - (ansible_distribution_release != 'bionic' and ansible_distribution_release != 'focal') + (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty' or ansible_distribution_release == 'xenial') - name: Add custom edX PPA # Ensure that we get the latest version of python 2.7 @@ -89,7 +89,7 @@ repo: "{{ COMMON_EDX_PPA }}" when: > ansible_distribution in common_debian_variants and - (ansible_distribution_release != 'bionic' and ansible_distribution_release != 'focal') + (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty' or ansible_distribution_release == 'xenial') # The deadsnakes PPA is required to install python3.5 on Bionic and Focal # Xenial comes with python3.5 installed. From bad00e46531a675e302441bb662b987a42810bc4 Mon Sep 17 00:00:00 2001 From: Zachary Hancock Date: Thu, 24 Aug 2023 16:45:14 -0400 Subject: [PATCH 484/664] feat: update masters sandbox to node 18 (#6995) --- playbooks/roles/learner_portal/defaults/main.yml | 2 +- playbooks/roles/program_console/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/learner_portal/defaults/main.yml b/playbooks/roles/learner_portal/defaults/main.yml index 3b077848fa1..751544d2833 100644 --- a/playbooks/roles/learner_portal/defaults/main.yml +++ b/playbooks/roles/learner_portal/defaults/main.yml @@ -17,7 +17,7 @@ edx_django_service_use_python3: false learner_portal_repo: 'https://github.com/openedx/frontend-app-learner-portal-programs.git' LEARNER_PORTAL_VERSION: 'master' learner_portal_service_name: 'learner_portal' -LEARNER_PORTAL_NODE_VERSION: '16.14.2' +LEARNER_PORTAL_NODE_VERSION: '18.17.0' learner_portal_nodeenv_dir: '{{ learner_portal_home }}/nodeenvs/{{ learner_portal_service_name }}' learner_portal_nodeenv_bin: '{{learner_portal_nodeenv_dir}}/bin' diff --git a/playbooks/roles/program_console/defaults/main.yml b/playbooks/roles/program_console/defaults/main.yml index 4e84536aaa3..0e88b32de50 100644 --- a/playbooks/roles/program_console/defaults/main.yml +++ b/playbooks/roles/program_console/defaults/main.yml @@ -24,7 +24,7 @@ program_console_git_identity: 'none' program_console_repo: 'https://github.com/openedx/frontend-app-program-console.git' PROGRAM_CONSOLE_VERSION: 'master' program_console_service_name: 'program-console' -PROGRAM_CONSOLE_NODE_VERSION: '12.11.1' +PROGRAM_CONSOLE_NODE_VERSION: '18.17.0' program_console_nodeenv_dir: '{{ program_console_home }}/nodeenvs/{{ program_console_service_name }}' program_console_nodeenv_bin: '{{program_console_nodeenv_dir}}/bin' program_console_app_dir: "{{ COMMON_APP_DIR }}/program-console" From 16ac207e728d2a74299922393349e38bb15172c8 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 24 Aug 2023 14:54:04 -0600 Subject: [PATCH 485/664] fix: add common release packages logic for Jammy --- playbooks/roles/common_vars/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 7ed87040a50..4801d27bd75 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -141,6 +141,8 @@ common_release_specific_debian_pkgs: - python3.5-dev focal: - python3.5-dev + jammy: + - python3.8 common_debian_pkgs: "{{ common_debian_pkgs_default + common_release_specific_debian_pkgs[ansible_distribution_release] }}" From 3ed0114aa1df9a616e9889a42b0f4d8dd6bc3654 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 24 Aug 2023 15:02:58 -0600 Subject: [PATCH 486/664] fix: add jammy to deadsnakes repo install --- playbooks/roles/common/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index 655dfd5ff44..793b960b8e8 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -101,7 +101,7 @@ until: add_repo is success retries: 10 delay: 5 - when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' or ansible_distribution_release == 'jammy' tags: - install - install:system-requirements From 2b9ddc55b1e979b71eca5309a22ac2dd71939a6b Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 24 Aug 2023 18:22:21 -0400 Subject: [PATCH 487/664] fix: Prevent S3 errors from large cookies --- .../app/nginx/sites-available/prospectus.j2 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 3d9967f90f2..72391c7403b 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -56,6 +56,8 @@ server { add_header Cache-Control "no-store, max-age=0" always; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie {% endif %} } @@ -64,6 +66,8 @@ server { add_header X-Frame-Options ''; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie {% endif %} } @@ -72,6 +76,8 @@ server { add_header 'Cache-Control' 'public, max-age=31536000, immutable'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie {% endif %} } @@ -81,6 +87,8 @@ server { add_header 'Cache-Control' 'public, max-age=3600'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie {% endif %} } @@ -90,6 +98,8 @@ server { add_header 'Cache-Control' 'public, max-age=86400'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie {% endif %} } @@ -98,6 +108,8 @@ server { location /HealthCheck { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/HealthCheck/index.html; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -110,6 +122,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -123,6 +137,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -189,6 +205,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} location / { proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # Prevent S3 errors from cookies being too large + proxy_hide_header Cookie # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive From 49a3ab977dc10bc618f79606342626a55f7f707e Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 28 Aug 2023 09:45:36 -0400 Subject: [PATCH 488/664] fix: Add missing semicolons in prospectus nginx --- .../app/nginx/sites-available/prospectus.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 72391c7403b..38e4ba1253e 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -57,7 +57,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; {% endif %} } @@ -67,7 +67,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; {% endif %} } @@ -77,7 +77,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; {% endif %} } @@ -88,7 +88,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; {% endif %} } @@ -99,7 +99,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; {% endif %} } @@ -109,7 +109,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/HealthCheck/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -123,7 +123,7 @@ server { rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -138,7 +138,7 @@ server { rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -206,7 +206,7 @@ server { location / { proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie + proxy_hide_header Cookie; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive From 7e71726fa88aa381a9cc98b2551c7139525e043d Mon Sep 17 00:00:00 2001 From: Bianca Severino Date: Tue, 29 Aug 2023 13:59:19 -0400 Subject: [PATCH 489/664] feat: add prospectus sidebar filter flag --- CHANGELOG.md | 4 ++++ playbooks/roles/prospectus/templates/.env.environment.j2 | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cbb42c8c0a..abdfea8d0c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2023-08-29 + - Role: prospectus + - Added `GATSBY_TURN_ON_SIDEBAR_FILTERS` flag to enable new sidebar changes + - 2023-06-20 - Role: prospectus - Added `GATSBY_XPERT_STG_API_URL`, `GATSBY_XPERT_UAT_API_URL` and `GATSBY_XPERT_PROD_API_URL` diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 337cec0f01f..9d637957d5b 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -14,4 +14,5 @@ PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} -GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} \ No newline at end of file +GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} +GATSBY_TURN_ON_SIDEBAR_FILTERS={{ PROSPECTUS_GATSBY_TURN_ON_SIDEBAR_FILTERS }} From dac23261f3fb90aa1e45a641d0e33a43b4498acf Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 29 Aug 2023 13:57:02 -0400 Subject: [PATCH 490/664] fix: Actually hide cookies from prospectus S3 It turns out that the proxy_hide_headers directive is for hiding headers sent by the upstream server (S3) from the client. Not for hiding client headers from the upstead server (S3), which is what we want to do. --- .../app/nginx/sites-available/prospectus.j2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 38e4ba1253e..44daa9c1f6e 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -57,7 +57,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; {% endif %} } @@ -67,7 +67,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; {% endif %} } @@ -77,7 +77,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; {% endif %} } @@ -88,7 +88,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; {% endif %} } @@ -99,7 +99,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; {% endif %} } @@ -109,7 +109,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/HealthCheck/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -123,7 +123,7 @@ server { rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -138,7 +138,7 @@ server { rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -206,7 +206,7 @@ server { location / { proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; # Prevent S3 errors from cookies being too large - proxy_hide_header Cookie; + proxy_set_header Cookie ""; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive From 12f6ac413e42511fff4d800e2aee07fa7e920112 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 29 Aug 2023 16:59:18 -0400 Subject: [PATCH 491/664] feat: remove runbooks related to deprecated video service, VEDA --- playbooks/oauth_client_setup.yml | 1 - .../veda_delivery_worker/defaults/main.yml | 44 ---- .../roles/veda_delivery_worker/meta/main.yml | 23 --- .../roles/veda_delivery_worker/tasks/main.yml | 111 ---------- .../veda_delivery_worker.conf.j2 | 12 -- .../veda_delivery_worker.sh.j2 | 9 - .../veda_delivery_worker_env.j2 | 7 - .../veda_encode_worker/defaults/main.yml | 47 ----- .../roles/veda_encode_worker/meta/main.yml | 25 --- .../roles/veda_encode_worker/tasks/main.yml | 109 ---------- .../veda_encode_worker.conf.j2 | 12 -- .../veda_encode_worker.sh.j2 | 6 - .../veda_encode_worker_env.j2 | 7 - playbooks/roles/veda_ffmpeg/defaults/main.yml | 17 -- playbooks/roles/veda_ffmpeg/meta/main.yml | 14 -- playbooks/roles/veda_ffmpeg/tasks/main.yml | 36 ---- .../veda_intake_worker/defaults/main.yml | 1 - .../roles/veda_intake_worker/meta/main.yml | 4 - .../roles/veda_intake_worker/tasks/main.yml | 1 - .../veda_pipeline_worker/defaults/main.yml | 57 ----- .../roles/veda_pipeline_worker/meta/main.yml | 25 --- .../roles/veda_pipeline_worker/tasks/main.yml | 113 ---------- .../veda_pipeline_worker.conf.j2 | 12 -- .../veda_pipeline_worker.sh.j2 | 6 - .../veda_pipeline_worker_env.j2 | 7 - .../roles/veda_web_frontend/defaults/main.yml | 83 -------- .../roles/veda_web_frontend/meta/main.yml | 40 ---- .../roles/veda_web_frontend/tasks/main.yml | 36 ---- .../video_pipeline_base/defaults/main.yml | 194 ------------------ .../roles/video_pipeline_base/meta/main.yml | 4 - .../roles/video_pipeline_base/tasks/main.yml | 7 - .../templates/config.yml.j2 | 2 - .../defaults/main.yml | 21 -- .../video_pipeline_integration/tasks/main.yml | 32 --- playbooks/veda_delivery_worker.yml | 9 - playbooks/veda_encode_worker.yml | 10 - playbooks/veda_intake_worker.yml | 5 - playbooks/veda_pipeline_worker.yml | 10 - playbooks/veda_web_frontend.yml | 14 -- playbooks/video_pipeline_integration.yml | 9 - util/bake_config.sh | 2 +- util/jenkins/ansible-provision.sh | 18 +- 42 files changed, 2 insertions(+), 1200 deletions(-) delete mode 100644 playbooks/roles/veda_delivery_worker/defaults/main.yml delete mode 100644 playbooks/roles/veda_delivery_worker/meta/main.yml delete mode 100644 playbooks/roles/veda_delivery_worker/tasks/main.yml delete mode 100644 playbooks/roles/veda_delivery_worker/templates/edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2 delete mode 100644 playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2 delete mode 100644 playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker_env.j2 delete mode 100644 playbooks/roles/veda_encode_worker/defaults/main.yml delete mode 100644 playbooks/roles/veda_encode_worker/meta/main.yml delete mode 100644 playbooks/roles/veda_encode_worker/tasks/main.yml delete mode 100644 playbooks/roles/veda_encode_worker/templates/edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2 delete mode 100644 playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker.sh.j2 delete mode 100644 playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker_env.j2 delete mode 100644 playbooks/roles/veda_ffmpeg/defaults/main.yml delete mode 100644 playbooks/roles/veda_ffmpeg/meta/main.yml delete mode 100644 playbooks/roles/veda_ffmpeg/tasks/main.yml delete mode 100644 playbooks/roles/veda_intake_worker/defaults/main.yml delete mode 100644 playbooks/roles/veda_intake_worker/meta/main.yml delete mode 100644 playbooks/roles/veda_intake_worker/tasks/main.yml delete mode 100644 playbooks/roles/veda_pipeline_worker/defaults/main.yml delete mode 100644 playbooks/roles/veda_pipeline_worker/meta/main.yml delete mode 100644 playbooks/roles/veda_pipeline_worker/tasks/main.yml delete mode 100644 playbooks/roles/veda_pipeline_worker/templates/edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2 delete mode 100644 playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2 delete mode 100644 playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2 delete mode 100644 playbooks/roles/veda_web_frontend/defaults/main.yml delete mode 100644 playbooks/roles/veda_web_frontend/meta/main.yml delete mode 100644 playbooks/roles/veda_web_frontend/tasks/main.yml delete mode 100644 playbooks/roles/video_pipeline_base/defaults/main.yml delete mode 100644 playbooks/roles/video_pipeline_base/meta/main.yml delete mode 100644 playbooks/roles/video_pipeline_base/tasks/main.yml delete mode 100644 playbooks/roles/video_pipeline_base/templates/config.yml.j2 delete mode 100644 playbooks/roles/video_pipeline_integration/defaults/main.yml delete mode 100644 playbooks/roles/video_pipeline_integration/tasks/main.yml delete mode 100644 playbooks/veda_delivery_worker.yml delete mode 100644 playbooks/veda_encode_worker.yml delete mode 100644 playbooks/veda_intake_worker.yml delete mode 100644 playbooks/veda_pipeline_worker.yml delete mode 100644 playbooks/veda_web_frontend.yml delete mode 100644 playbooks/video_pipeline_integration.yml diff --git a/playbooks/oauth_client_setup.yml b/playbooks/oauth_client_setup.yml index 335199f794f..c2bb54ff1cf 100644 --- a/playbooks/oauth_client_setup.yml +++ b/playbooks/oauth_client_setup.yml @@ -9,7 +9,6 @@ - "roles/ecommerce/defaults/main.yml" - "roles/credentials/defaults/main.yml" - "roles/discovery/defaults/main.yml" - - "roles/veda_web_frontend/defaults/main.yml" - "roles/registrar/defaults/main.yml" - "roles/designer/defaults/main.yml" - "roles/enterprise_catalog/defaults/main.yml" diff --git a/playbooks/roles/veda_delivery_worker/defaults/main.yml b/playbooks/roles/veda_delivery_worker/defaults/main.yml deleted file mode 100644 index 263b7e68544..00000000000 --- a/playbooks/roles/veda_delivery_worker/defaults/main.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for veda_delivery_worker role -# - -veda_delivery_worker_service_name: 'veda_delivery_worker' -veda_delivery_worker_user: '{{ veda_delivery_worker_service_name }}' -veda_delivery_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_delivery_worker_service_name }}' -veda_delivery_worker_code_dir: '{{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}' -veda_delivery_worker_venv_bin: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}/bin' -veda_delivery_worker_service_config: !!null - -veda_delivery_worker_requirements_base: '{{ veda_delivery_worker_code_dir }}' -veda_delivery_worker_requirements: - - requirements.txt - -veda_delivery_worker_environment: - PATH: '{{ veda_delivery_worker_venv_bin }}:{{ ansible_env.PATH }}' - PYTHONPATH: "{{ veda_delivery_worker_code_dir }}" - VEDA_DELIVERY_WORKER_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_delivery_worker_service_name }}.yml' - -# Requires that New Relic be enabled via COMMON_ENABLE_NEWRELIC, and that -# a key be provided via NEWRELIC_LICENSE_KEY. -VEDA_DELIVERY_WORKER_NEWRELIC_APPNAME: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ veda_delivery_worker_service_name }}' - -VEDA_DELIVERY_WORKER_GIT_IDENTITY: !!null -VEDA_DELIVERY_WORKER_VERSION: 'master' - -VEDA_DELIVERY_WORKER_REPOS: - - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' - DOMAIN: '{{ COMMON_GIT_MIRROR }}' - PATH: '{{ COMMON_GIT_PATH }}' - REPO: edx-video-pipeline.git - VERSION: '{{ VEDA_DELIVERY_WORKER_VERSION }}' - DESTINATION: '{{ veda_delivery_worker_code_dir }}' - SSH_KEY: '{{ VEDA_DELIVERY_WORKER_GIT_IDENTITY }}' diff --git a/playbooks/roles/veda_delivery_worker/meta/main.yml b/playbooks/roles/veda_delivery_worker/meta/main.yml deleted file mode 100644 index bceab068736..00000000000 --- a/playbooks/roles/veda_delivery_worker/meta/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for veda_delivery_worker role -# -dependencies: - - common - - supervisor - - role: video_pipeline_base - video_pipeline_base_service_name: '{{ veda_delivery_worker_service_name }}' - - role: edx_service_with_rendered_config - edx_service_with_rendered_config_service_name: "{{ veda_delivery_worker_service_name }}" - edx_service_with_rendered_config_service_config: "{{ veda_delivery_worker_service_config }}" - edx_service_with_rendered_config_repos: "{{ VEDA_DELIVERY_WORKER_REPOS }}" - edx_service_with_rendered_config_user: "{{ veda_delivery_worker_user }}" - edx_service_with_rendered_config_home: "{{ veda_delivery_worker_home }}" diff --git a/playbooks/roles/veda_delivery_worker/tasks/main.yml b/playbooks/roles/veda_delivery_worker/tasks/main.yml deleted file mode 100644 index 5914c6cb8c5..00000000000 --- a/playbooks/roles/veda_delivery_worker/tasks/main.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# Tasks for veda_delivery_worker role -# - -- name: install application requirements - pip: - requirements: '{{ veda_delivery_worker_requirements_base }}/{{ item }}' - virtualenv: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}' - state: present - become_user: '{{ veda_delivery_worker_user }}' - with_items: "{{ veda_delivery_worker_requirements }}" - tags: - - install - - install:app-requirements - -# TODO! Update VEDA_WORKING directory path with a better path -# https://openedx.atlassian.net/browse/EDUCATOR-1851 -- name: create worker VEDA_WORKING directory - file: - path: "{{ veda_delivery_worker_home }}/{{ item }}" - state: directory - owner: "{{ veda_delivery_worker_service_name }}" - group: "{{ common_web_group }}" - with_items: - - "VEDA_WORKING" - tags: - - install - - install:base - -- name: write out the supervisor wrapper - template: - src: edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2 - dest: '{{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}.sh' - mode: 0650 - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - tags: - - install - - install:configuration - -- name: write supervisord config - template: - src: edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2 - dest: '{{ supervisor_available_dir }}/{{ veda_delivery_worker_service_name }}.conf' - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - mode: 0644 - tags: - - install - - install:configuration - -- name: setup the veda_delivery_worker env file - template: - src: edx/app/veda_delivery_worker/veda_delivery_worker_env.j2 - dest: '{{ veda_delivery_worker_home }}/veda_delivery_worker_env' - owner: '{{ veda_delivery_worker_user }}' - group: '{{ veda_delivery_worker_user }}' - mode: 0644 - tags: - - install - - install:configuration - -- name: enable supervisor script - file: - src: '{{ supervisor_available_dir }}/{{ veda_delivery_worker_service_name }}.conf' - dest: '{{ supervisor_cfg_dir }}/{{ veda_delivery_worker_service_name }}.conf' - state: link - force: yes - when: not disable_edx_services - tags: - - install - - install:configuration - -- name: update supervisor configuration - shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update' - when: not disable_edx_services - tags: - - manage - - manage:start - -- name: create symlinks from the venv bin dir - file: - src: '{{ veda_delivery_worker_home }}/venvs/{{ veda_delivery_worker_service_name }}/bin/{{ item }}' - dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_delivery_worker' - state: link - with_items: - - python - - pip - tags: - - install - - install:configuration - -- name: restart the applicaton - supervisorctl: - state: restarted - supervisorctl_path: '{{ supervisor_ctl }}' - config: '{{ supervisor_cfg }}' - name: '{{ veda_delivery_worker_service_name }}' - when: not disable_edx_services - become_user: '{{ supervisor_service_user }}' - tags: - - manage - - manage:start diff --git a/playbooks/roles/veda_delivery_worker/templates/edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2 b/playbooks/roles/veda_delivery_worker/templates/edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2 deleted file mode 100644 index 6140d5faf4e..00000000000 --- a/playbooks/roles/veda_delivery_worker/templates/edx/app/supervisor/conf.d.available/veda_delivery_worker.conf.j2 +++ /dev/null @@ -1,12 +0,0 @@ -# -# {{ ansible_managed }} -# -[program:{{ veda_delivery_worker_service_name }}] - -command={{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}.sh -user={{ common_web_user }} -directory={{ veda_delivery_worker_code_dir }} -stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log -stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -killasgroup=true -stopasgroup=true diff --git a/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2 b/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2 deleted file mode 100644 index 4f086b0c626..00000000000 --- a/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker.sh.j2 +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -# {{ ansible_managed }} - -{% set veda_delivery_worker_venv_bin = veda_delivery_worker_home + '/venvs/' + veda_delivery_worker_service_name + '/bin' %} -{% set executable = veda_delivery_worker_venv_bin + '/python' %} - -source {{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}_env -{{ executable }} {{ veda_delivery_worker_home }}/{{ veda_delivery_worker_service_name }}/bin/deliver diff --git a/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker_env.j2 b/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker_env.j2 deleted file mode 100644 index e5d96d38da8..00000000000 --- a/playbooks/roles/veda_delivery_worker/templates/edx/app/veda_delivery_worker/veda_delivery_worker_env.j2 +++ /dev/null @@ -1,7 +0,0 @@ -# {{ ansible_managed }} - -{% for name,value in veda_delivery_worker_environment.items() -%} -{%- if value -%} -export {{ name }}="{{ value }}" -{% endif %} -{%- endfor %} diff --git a/playbooks/roles/veda_encode_worker/defaults/main.yml b/playbooks/roles/veda_encode_worker/defaults/main.yml deleted file mode 100644 index a932a510909..00000000000 --- a/playbooks/roles/veda_encode_worker/defaults/main.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for veda_encode_worker role -# - -veda_encode_worker_service_name: 'veda_encode_worker' -veda_encode_worker_user: '{{ veda_encode_worker_service_name }}' -veda_encode_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_encode_worker_service_name }}' -veda_encode_worker_code_dir: '{{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}' -veda_encode_worker_venv_bin: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}/bin' -veda_encode_worker_data_dir: '{{ COMMON_DATA_DIR }}/{{ veda_encode_worker_service_name }}/ENCODE_WORKDIR' -veda_encode_worker_service_config: !!null - -veda_encode_worker_directories: - ENCODE_WORK_DIR: '{{ veda_encode_worker_data_dir }}' - -veda_encode_worker_config_other: {} -veda_encode_worker_config_extra: '{{ veda_encode_worker_config_other | combine(veda_encode_worker_directories) }}' - -veda_encode_worker_requirements_base: '{{ veda_encode_worker_code_dir }}' -veda_encode_worker_requirements: - - requirements.txt - -veda_encode_worker_environment: - PATH: '{{ veda_encode_worker_venv_bin }}:{{ ansible_env.PATH }}' - PYTHONPATH: "{{ veda_encode_worker_code_dir }}" - VEDA_ENCODE_WORKER_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_encode_worker_service_name }}.yml' - -VEDA_ENCODE_WORKER_GIT_IDENTITY: !!null -VEDA_ENCODE_WORKER_VERSION: 'master' - -VEDA_ENCODE_WORKER_REPOS: - - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' - DOMAIN: '{{ COMMON_GIT_MIRROR }}' - PATH: '{{ COMMON_GIT_PATH }}' - REPO: edx-video-worker.git - VERSION: '{{ VEDA_ENCODE_WORKER_VERSION }}' - DESTINATION: '{{ veda_encode_worker_code_dir }}' - SSH_KEY: '{{ VEDA_ENCODE_WORKER_GIT_IDENTITY }}' diff --git a/playbooks/roles/veda_encode_worker/meta/main.yml b/playbooks/roles/veda_encode_worker/meta/main.yml deleted file mode 100644 index 48d582d5750..00000000000 --- a/playbooks/roles/veda_encode_worker/meta/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for veda_encode_worker role -# -dependencies: - - common - - supervisor - - veda_ffmpeg - - role: video_pipeline_base - video_pipeline_base_service_name: '{{ veda_encode_worker_service_name }}' - video_pipeline_base_config_extra: '{{ veda_encode_worker_config_extra }}' - - role: edx_service_with_rendered_config - edx_service_with_rendered_config_service_name: "{{ veda_encode_worker_service_name }}" - edx_service_with_rendered_config_service_config: "{{ veda_encode_worker_service_config }}" - edx_service_with_rendered_config_repos: "{{ VEDA_ENCODE_WORKER_REPOS }}" - edx_service_with_rendered_config_user: "{{ veda_encode_worker_user }}" - edx_service_with_rendered_config_home: "{{ veda_encode_worker_home }}" diff --git a/playbooks/roles/veda_encode_worker/tasks/main.yml b/playbooks/roles/veda_encode_worker/tasks/main.yml deleted file mode 100644 index 6dc6bb9b27d..00000000000 --- a/playbooks/roles/veda_encode_worker/tasks/main.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# Tasks for veda_encode_worker role -# - -- name: install application requirements - pip: - requirements: '{{ veda_encode_worker_requirements_base }}/{{ item }}' - virtualenv: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}' - state: present - become_user: '{{ veda_encode_worker_user }}' - with_items: "{{ veda_encode_worker_requirements }}" - tags: - - install - - install:app-requirements - -- name: create worker work directories - file: - path: "{{ item.value }}" - state: directory - owner: "{{ veda_encode_worker_service_name }}" - group: "{{ common_web_group }}" - mode: 0775 - with_dict: "{{ veda_encode_worker_directories }}" - tags: - - install - - install:base - -- name: write out the supervisor wrapper - template: - src: edx/app/veda_encode_worker/veda_encode_worker.sh.j2 - dest: '{{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}.sh' - mode: 0650 - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - tags: - - install - - install:configuration - -- name: write supervisord config - template: - src: edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2 - dest: '{{ supervisor_available_dir }}/{{ veda_encode_worker_service_name }}.conf' - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - mode: 0644 - tags: - - install - - install:configuration - -- name: setup the veda_encode_worker env file - template: - src: edx/app/veda_encode_worker/veda_encode_worker_env.j2 - dest: '{{ veda_encode_worker_home }}/veda_encode_worker_env' - owner: '{{ veda_encode_worker_user }}' - group: '{{ veda_encode_worker_user }}' - mode: 0644 - tags: - - install - - install:configuration - -- name: enable supervisor script - file: - src: '{{ supervisor_available_dir }}/{{ veda_encode_worker_service_name }}.conf' - dest: '{{ supervisor_cfg_dir }}/{{ veda_encode_worker_service_name }}.conf' - state: link - force: yes - when: not disable_edx_services - tags: - - install - - install:configuration - -- name: update supervisor configuration - shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update' - when: not disable_edx_services - tags: - - manage - - manage:start - -- name: create symlinks from the venv bin dir - file: - src: '{{ veda_encode_worker_home }}/venvs/{{ veda_encode_worker_service_name }}/bin/{{ item }}' - dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_encode_worker' - state: link - with_items: - - python - - pip - tags: - - install - - install:configuration - -- name: restart the applicaton - supervisorctl: - state: restarted - supervisorctl_path: '{{ supervisor_ctl }}' - config: '{{ supervisor_cfg }}' - name: '{{ veda_encode_worker_service_name }}' - when: not disable_edx_services - become_user: '{{ supervisor_service_user }}' - tags: - - manage - - manage:start diff --git a/playbooks/roles/veda_encode_worker/templates/edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2 b/playbooks/roles/veda_encode_worker/templates/edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2 deleted file mode 100644 index 0324b86c41a..00000000000 --- a/playbooks/roles/veda_encode_worker/templates/edx/app/supervisor/conf.d.available/veda_encode_worker.conf.j2 +++ /dev/null @@ -1,12 +0,0 @@ -# -# {{ ansible_managed }} -# -[program:{{ veda_encode_worker_service_name }}] - -command={{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}.sh -user={{ common_web_user }} -directory={{ veda_encode_worker_code_dir }} -stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log -stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -killasgroup=true -stopasgroup=true diff --git a/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker.sh.j2 b/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker.sh.j2 deleted file mode 100644 index bccf9b83261..00000000000 --- a/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# {{ ansible_managed }} - -source {{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}_env -source {{ veda_encode_worker_home }}/{{ veda_encode_worker_service_name }}/worker.sh diff --git a/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker_env.j2 b/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker_env.j2 deleted file mode 100644 index b4c9245d525..00000000000 --- a/playbooks/roles/veda_encode_worker/templates/edx/app/veda_encode_worker/veda_encode_worker_env.j2 +++ /dev/null @@ -1,7 +0,0 @@ -# {{ ansible_managed }} - -{% for name,value in veda_encode_worker_environment.items() -%} -{%- if value -%} -export {{ name }}="{{ value }}" -{% endif %} -{%- endfor %} diff --git a/playbooks/roles/veda_ffmpeg/defaults/main.yml b/playbooks/roles/veda_ffmpeg/defaults/main.yml deleted file mode 100644 index d730fc02bd0..00000000000 --- a/playbooks/roles/veda_ffmpeg/defaults/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for veda_ffmpeg role -# - -VEDA_FFMPEG_GIT_URL: 'https://github.com/edx/v_videocompile' -# used /opt path to clone the repo and then install ffmpeg using the cloned repo -veda_ffmpeg_root: '/var/tmp/veda_ffmpeg' -VEDA_FFMPEG_VERSION: 'master' diff --git a/playbooks/roles/veda_ffmpeg/meta/main.yml b/playbooks/roles/veda_ffmpeg/meta/main.yml deleted file mode 100644 index 6236747e3d2..00000000000 --- a/playbooks/roles/veda_ffmpeg/meta/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# Dependencies for veda_ffmpeg role -# - -# Allow this role to be duplicated in dependencies. -allow_duplicates: yes diff --git a/playbooks/roles/veda_ffmpeg/tasks/main.yml b/playbooks/roles/veda_ffmpeg/tasks/main.yml deleted file mode 100644 index d5f2b9833eb..00000000000 --- a/playbooks/roles/veda_ffmpeg/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# Tasks for veda_ffmpeg role -# - -- name: checkout v_videocompile - git: - repo: "{{ VEDA_FFMPEG_GIT_URL }}" - dest: "{{ veda_ffmpeg_root }}" - version: "{{ VEDA_FFMPEG_VERSION }}" - become: False - tags: - - install - - install:code - -- name: install v_videocompile - command: "python setup.py install" - args: - chdir: "{{ veda_ffmpeg_root }}" - tags: - - install - - install:code - -- name: compile v_videocompile - command: "v_videocompile" - become: False - tags: - - install - - install:system-requirements diff --git a/playbooks/roles/veda_intake_worker/defaults/main.yml b/playbooks/roles/veda_intake_worker/defaults/main.yml deleted file mode 100644 index ed97d539c09..00000000000 --- a/playbooks/roles/veda_intake_worker/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/playbooks/roles/veda_intake_worker/meta/main.yml b/playbooks/roles/veda_intake_worker/meta/main.yml deleted file mode 100644 index 9a3880c7935..00000000000 --- a/playbooks/roles/veda_intake_worker/meta/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -dependencies: - - video_pipeline_base diff --git a/playbooks/roles/veda_intake_worker/tasks/main.yml b/playbooks/roles/veda_intake_worker/tasks/main.yml deleted file mode 100644 index ed97d539c09..00000000000 --- a/playbooks/roles/veda_intake_worker/tasks/main.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/playbooks/roles/veda_pipeline_worker/defaults/main.yml b/playbooks/roles/veda_pipeline_worker/defaults/main.yml deleted file mode 100644 index c8ad796fd42..00000000000 --- a/playbooks/roles/veda_pipeline_worker/defaults/main.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for veda_pipeline_worker role -# - -veda_pipeline_worker_service_name: 'veda_pipeline_worker' -veda_pipeline_worker_user: '{{ veda_pipeline_worker_service_name }}' -veda_pipeline_worker_home: '{{ COMMON_APP_DIR }}/{{ veda_pipeline_worker_service_name }}' -veda_pipeline_worker_code_dir: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}' -veda_pipeline_worker_venv_bin: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin' -veda_pipeline_worker_service_config: !!null -veda_pipeline_worker_variants: - - deliver - - ingest - - youtubecallback - -veda_pipeline_worker_requirements_base: '{{ veda_pipeline_worker_code_dir }}' -veda_pipeline_worker_requirements: - - requirements.txt - -veda_pipeline_worker_deliver_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/DELIVER_WORK_DIR' -veda_pipeline_worker_ingest_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/INGEST_WORK_DIR' -veda_pipeline_worker_work_dir: '{{ COMMON_DATA_DIR }}/{{ veda_pipeline_worker_service_name }}/VEDA_WORKING' - -veda_pipeline_worker_directories: - DELIVER_WORK_DIR: '{{ veda_pipeline_worker_deliver_work_dir }}' - INGEST_WORK_DIR: '{{ veda_pipeline_worker_ingest_work_dir }}' - VEDA_WORKING: '{{ veda_pipeline_worker_work_dir }}' - -veda_pipeline_worker_config_other: {} -veda_pipeline_worker_config_extra: '{{ veda_pipeline_worker_config_other | combine(veda_pipeline_worker_directories) }}' - -veda_pipeline_worker_environment: - PATH: '{{ veda_pipeline_worker_venv_bin }}:{{ ansible_env.PATH }}' - PYTHONPATH: "{{ veda_pipeline_worker_code_dir }}" - VIDEO_PIPELINE_CFG: '{{ COMMON_CFG_DIR }}/{{ veda_pipeline_worker_service_name }}.yml' - DJANGO_SETTINGS_MODULE: "VEDA.settings.production" - -VEDA_PIPELINE_WORKER_GIT_IDENTITY: !!null -VEDA_PIPELINE_WORKER_VERSION: 'master' - -VEDA_PIPELINE_WORKER_REPOS: - - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' - DOMAIN: '{{ COMMON_GIT_MIRROR }}' - PATH: '{{ COMMON_GIT_PATH }}' - REPO: edx-video-pipeline.git - VERSION: '{{ VEDA_PIPELINE_WORKER_VERSION }}' - DESTINATION: '{{ veda_pipeline_worker_code_dir }}' - SSH_KEY: '{{ VEDA_PIPELINE_WORKER_GIT_IDENTITY }}' diff --git a/playbooks/roles/veda_pipeline_worker/meta/main.yml b/playbooks/roles/veda_pipeline_worker/meta/main.yml deleted file mode 100644 index 13d2fd431b0..00000000000 --- a/playbooks/roles/veda_pipeline_worker/meta/main.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for veda_pipeline_worker role -# -dependencies: - - common - - supervisor - - veda_ffmpeg - - role: video_pipeline_base - video_pipeline_base_service_name: '{{ veda_pipeline_worker_service_name }}' - video_pipeline_base_config_extra: '{{ veda_pipeline_worker_config_extra }}' - - role: edx_service_with_rendered_config - edx_service_with_rendered_config_service_name: "{{ veda_pipeline_worker_service_name }}" - edx_service_with_rendered_config_service_config: "{{ veda_pipeline_worker_service_config }}" - edx_service_with_rendered_config_repos: "{{ VEDA_PIPELINE_WORKER_REPOS }}" - edx_service_with_rendered_config_user: "{{ veda_pipeline_worker_user }}" - edx_service_with_rendered_config_home: "{{ veda_pipeline_worker_home }}" diff --git a/playbooks/roles/veda_pipeline_worker/tasks/main.yml b/playbooks/roles/veda_pipeline_worker/tasks/main.yml deleted file mode 100644 index ae5de3a6b36..00000000000 --- a/playbooks/roles/veda_pipeline_worker/tasks/main.yml +++ /dev/null @@ -1,113 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# Tasks for veda_pipeline_worker role -# - -- name: install application requirements - pip: - requirements: '{{ veda_pipeline_worker_requirements_base }}/{{ item }}' - virtualenv: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}' - state: present - become_user: '{{ veda_pipeline_worker_user }}' - with_items: "{{ veda_pipeline_worker_requirements }}" - tags: - - install - - install:app-requirements - -- name: create worker work directories - file: - path: "{{ item.value }}" - state: directory - owner: "{{ veda_pipeline_worker_service_name }}" - group: "{{ common_web_group }}" - mode: 0775 - with_dict: "{{ veda_pipeline_worker_directories }}" - tags: - - install - - install:base - -- name: create symlinks from the venv bin dir - file: - src: '{{ veda_pipeline_worker_home }}/venvs/{{ veda_pipeline_worker_service_name }}/bin/{{ item }}' - dest: '{{ COMMON_BIN_DIR }}/{{ item }}.veda_pipeline_worker' - state: link - with_items: - - python - - pip - tags: - - install - - install:configuration - -- name: write out the supervisor wrapper - template: - src: edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2 - dest: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.sh' - mode: 0650 - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - with_items: "{{ veda_pipeline_worker_variants }}" - tags: - - install - - install:configuration - -- name: write supervisord config - template: - src: edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2 - dest: '{{ supervisor_available_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf' - owner: '{{ supervisor_user }}' - group: '{{ common_web_user }}' - mode: 0644 - with_items: "{{ veda_pipeline_worker_variants }}" - tags: - - install - - install:configuration - -- name: setup the veda_pipeline_worker env file - template: - src: edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2 - dest: '{{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_env' - owner: '{{ veda_pipeline_worker_user }}' - group: '{{ veda_pipeline_worker_user }}' - mode: 0644 - tags: - - install - - install:configuration - -- name: enable supervisor script - file: - src: '{{ supervisor_available_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf' - dest: '{{ supervisor_cfg_dir }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.conf' - state: link - force: yes - with_items: "{{ veda_pipeline_worker_variants }}" - when: not disable_edx_services - tags: - - install - - install:configuration - -- name: update supervisor configuration - shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update' - when: not disable_edx_services - tags: - - manage - - manage:start - -- name: restart the applicaton - supervisorctl: - state: restarted - supervisorctl_path: '{{ supervisor_ctl }}' - config: '{{ supervisor_cfg }}' - name: '{{ veda_pipeline_worker_service_name }}_{{ item }}' - with_items: "{{ veda_pipeline_worker_variants }}" - when: not disable_edx_services - become_user: '{{ supervisor_service_user }}' - tags: - - manage - - manage:start diff --git a/playbooks/roles/veda_pipeline_worker/templates/edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2 b/playbooks/roles/veda_pipeline_worker/templates/edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2 deleted file mode 100644 index 4bbf4344f72..00000000000 --- a/playbooks/roles/veda_pipeline_worker/templates/edx/app/supervisor/conf.d.available/veda_pipeline_worker.conf.j2 +++ /dev/null @@ -1,12 +0,0 @@ -# -# {{ ansible_managed }} -# -[program:{{ veda_pipeline_worker_service_name }}_{{ item }}] - -command={{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_{{ item }}.sh -user={{ common_web_user }} -directory={{ veda_pipeline_worker_code_dir }} -stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log -stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -killasgroup=true -stopasgroup=true diff --git a/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2 b/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2 deleted file mode 100644 index 703fb609b9d..00000000000 --- a/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# {{ ansible_managed }} - -source {{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}_env -{{ veda_pipeline_worker_venv_bin }}/python {{ veda_pipeline_worker_home }}/{{ veda_pipeline_worker_service_name }}/bin/{{ item }} diff --git a/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2 b/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2 deleted file mode 100644 index 0f9922d2c92..00000000000 --- a/playbooks/roles/veda_pipeline_worker/templates/edx/app/veda_pipeline_worker/veda_pipeline_worker_env.j2 +++ /dev/null @@ -1,7 +0,0 @@ -# {{ ansible_managed }} - -{% for name,value in veda_pipeline_worker_environment.items() -%} -{%- if value -%} -export {{ name }}="{{ value }}" -{% endif %} -{%- endfor %} diff --git a/playbooks/roles/veda_web_frontend/defaults/main.yml b/playbooks/roles/veda_web_frontend/defaults/main.yml deleted file mode 100644 index 87337dd18de..00000000000 --- a/playbooks/roles/veda_web_frontend/defaults/main.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for role veda_web_frontend -# - -# TODO! https://openedx.atlassian.net/browse/EDUCATOR-1852 -veda_web_frontend_service_name: "veda" - -veda_web_frontend_user: "{{ veda_web_frontend_service_name }}" -veda_web_frontend_home: "{{ COMMON_APP_DIR }}/{{ veda_web_frontend_service_name }}" - -veda_web_frontend_service_home: "{{ COMMON_APP_DIR }}/{{ veda_web_frontend_service_name }}" -veda_web_frontend_code_dir: "{{ veda_web_frontend_service_home }}/{{ veda_web_frontend_service_name }}" -veda_web_frontend_venv_dir: "{{ veda_web_frontend_service_home }}/venvs/{{ veda_web_frontend_service_name }}" -veda_web_frontend_work_dir: "{{ COMMON_DATA_DIR }}/{{ veda_web_frontend_service_name }}/VEDA_WORKING" - -veda_web_frontend_directories: - VEDA_WORKING: '{{ veda_web_frontend_work_dir }}' - -veda_web_frontend_config_other: {} -veda_web_frontend_config_extra: "{{ veda_web_frontend_config_other | combine(veda_web_frontend_directories) }}" - -VEDA_WEB_FRONTEND_LOAD_DATA: false -veda_web_frontend_post_migrate_commands: - - command: 'python manage.py loaddata encodes' - when: '{{ VEDA_WEB_FRONTEND_LOAD_DATA }}' - -# -# OS packages -# - -veda_web_frontend_debian_pkgs: - - libffi6 - - libffi-dev - -VEDA_WEB_FRONTEND_REPO: "edx-video-pipeline" - -VEDA_WEB_FRONTEND_VERSION: "master" - -VEDA_WEB_FRONTEND_MEMCACHE: [] - -VEDA_WEB_FRONTEND_DJANGO_SETTINGS_MODULE: "VEDA.settings.production" - -VEDA_WEB_FRONTEND_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true - -VEDA_WEB_FRONTEND_DATA_DIR: "{{ COMMON_DATA_DIR }}/{{ veda_web_frontend_service_name }}" -VEDA_WEB_FRONTEND_MEDIA_ROOT: "{{ VEDA_WEB_FRONTEND_DATA_DIR }}/media" -VEDA_WEB_FRONTEND_MEDIA_URL: "/media/" - -VEDA_WEB_FRONTEND_MEDIA_STORAGE_BACKEND: - DEFAULT_FILE_STORAGE: "django.core.files.storage.FileSystemStorage" - MEDIA_ROOT: "{{ VEDA_WEB_FRONTEND_MEDIA_ROOT }}" - MEDIA_URL: "{{ VEDA_WEB_FRONTEND_MEDIA_URL }}" - -VEDA_WEB_FRONTEND_ENVIRONMENT: - VIDEO_PIPELINE_CFG: "{{ COMMON_CFG_DIR }}/{{ veda_web_frontend_service_name }}.yml" - PYTHONPATH: "{{ veda_web_frontend_code_dir }}" - -VEDA_WEB_FRONTEND_STATICFILES_STORAGE: "django.contrib.staticfiles.storage.StaticFilesStorage" - -VEDA_WEB_FRONTEND_SERVICE_CONFIG: !!null -VEDA_WEB_FRONTEND_SECRET_KEY: '{{ VIDEO_PIPELINE_BASE_SECRET_KEY }}' - -VEDA_WEB_FRONTEND_GUNICORN_PORT: '{{ VIDEO_PIPELINE_BASE_GUNICORN_PORT }}' -VEDA_WEB_FRONTEND_NGINX_PORT: '{{ VIDEO_PIPELINE_BASE_NGINX_PORT }}' -VEDA_WEB_FRONTEND_SSL_NGINX_PORT: '{{ VIDEO_PIPELINE_BASE_SSL_NGINX_PORT }}' - -VEDA_WEB_FRONTEND_DEFAULT_DB_NAME: '{{ VIDEO_PIPELINE_BASE_DEFAULT_DB_NAME }}' -VEDA_WEB_FRONTEND_MYSQL_HOST: '{{ VIDEO_PIPELINE_BASE_MYSQL_HOST }}' -VEDA_WEB_FRONTEND_MYSQL_USER: '{{ VIDEO_PIPELINE_BASE_MYSQL_USER }}' -VEDA_WEB_FRONTEND_MYSQL_PASSWORD: '{{ VIDEO_PIPELINE_BASE_MYSQL_PASSWORD }}' -VEDA_WEB_FRONTEND_MYSQLE_CONN_MAX_AGE: 60 - -VEDA_WEB_FRONTEND_OAUTH2_URL: '{{ VIDEO_PIPELINE_BASE_URL_ROOT }}/api/val/v0' -VEDA_WEB_FRONTEND_LOGOUT_URL: '{{ VIDEO_PIPELINE_BASE_URL_ROOT }}/logout/' diff --git a/playbooks/roles/veda_web_frontend/meta/main.yml b/playbooks/roles/veda_web_frontend/meta/main.yml deleted file mode 100644 index 0cb8a8ae558..00000000000 --- a/playbooks/roles/veda_web_frontend/meta/main.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Role includes for role veda_web_frontend -# -dependencies: - - role: video_pipeline_base - video_pipeline_base_service_name: '{{ veda_web_frontend_service_name }}' - video_pipeline_base_config_extra: '{{ veda_web_frontend_config_extra }}' - - role: edx_django_service_with_rendered_config - edx_django_service_with_rendered_config_repo: '{{ VEDA_WEB_FRONTEND_REPO }}' - edx_django_service_with_rendered_config_version: '{{ VEDA_WEB_FRONTEND_VERSION }}' - edx_django_service_with_rendered_config_service_name: '{{ veda_web_frontend_service_name }}' - edx_django_service_with_rendered_config_user: '{{ veda_web_frontend_user }}' - edx_django_service_with_rendered_config_debian_pkgs_extra: '{{ veda_web_frontend_debian_pkgs }}' - edx_django_service_with_rendered_config_gunicorn_port: '{{ VEDA_WEB_FRONTEND_GUNICORN_PORT }}' - edx_django_service_with_rendered_config_django_settings_module: '{{ VEDA_WEB_FRONTEND_DJANGO_SETTINGS_MODULE }}' - edx_django_service_with_rendered_config_environment_extra: '{{ VEDA_WEB_FRONTEND_ENVIRONMENT }}' - edx_django_service_with_rendered_config_wsgi_name: 'VEDA' - edx_django_service_with_rendered_config_nginx_port: '{{ VEDA_WEB_FRONTEND_NGINX_PORT }}' - edx_django_service_with_rendered_config_ssl_nginx_port: '{{ VEDA_WEB_FRONTEND_SSL_NGINX_PORT }}' - edx_django_service_with_rendered_config_secret_key: '{{ VEDA_WEB_FRONTEND_SECRET_KEY }}' - edx_django_service_with_rendered_config_staticfiles_storage: '{{ VEDA_WEB_FRONTEND_STATICFILES_STORAGE }}' - edx_django_service_with_rendered_config_media_storage_backend: '{{ VEDA_WEB_FRONTEND_MEDIA_STORAGE_BACKEND }}' - edx_django_service_with_rendered_config_memcache: '{{ VEDA_WEB_FRONTEND_MEMCACHE }}' - edx_django_service_with_rendered_config_default_db_host: '{{ VEDA_WEB_FRONTEND_MYSQL_HOST }}' - edx_django_service_with_rendered_config_default_db_name: '{{ VEDA_WEB_FRONTEND_DEFAULT_DB_NAME }}' - edx_django_service_with_rendered_config_db_user: '{{ VEDA_WEB_FRONTEND_MYSQL_USER }}' - edx_django_service_with_rendered_config_db_password: '{{ VEDA_WEB_FRONTEND_MYSQL_PASSWORD }}' - edx_django_service_default_db_conn_max_age: '{{ VEDA_WEB_FRONTEND_MYSQL_CONN_MAX_AGE }}' - edx_django_service_with_rendered_config_use_python3: false - edx_django_service_with_rendered_config_service_config: '{{ VEDA_WEB_FRONTEND_SERVICE_CONFIG }}' - edx_django_service_with_rendered_config_post_migrate_commands: '{{ veda_web_frontend_post_migrate_commands }}' diff --git a/playbooks/roles/veda_web_frontend/tasks/main.yml b/playbooks/roles/veda_web_frontend/tasks/main.yml deleted file mode 100644 index 0d67dbd7676..00000000000 --- a/playbooks/roles/veda_web_frontend/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Tasks for role veda_web_frontend -# -- name: create work directory - file: - path: "{{ veda_web_frontend_work_dir }}" - state: directory - owner: "{{ veda_web_frontend_service_name }}" - group: "{{ common_web_group }}" - mode: 0775 - tags: - - install - - install:base - -# This is creating a client in VEDA application not LMS. -- name: create OAuth application clients - shell: > - {{ veda_web_frontend_venv_dir }}/bin/python {{ COMMON_BIN_DIR }}/manage.{{ veda_web_frontend_service_name }} create_oauth_client --settings={{ VEDA_WEB_FRONTEND_DJANGO_SETTINGS_MODULE }} - {{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_ID }} - confidential - client-credentials - --client_name {{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_NAME }} - --client_secret {{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_SECRET }} - --redirect_uris {{ VIDEO_PIPELINE_BASE_OAUTH_TOKEN_URL }} - --skip_authorization - become_user: "{{ veda_web_frontend_user }}" - environment: "{{ VEDA_WEB_FRONTEND_ENVIRONMENT }}" diff --git a/playbooks/roles/video_pipeline_base/defaults/main.yml b/playbooks/roles/video_pipeline_base/defaults/main.yml deleted file mode 100644 index 6ded12f6692..00000000000 --- a/playbooks/roles/video_pipeline_base/defaults/main.yml +++ /dev/null @@ -1,194 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for common variables for video_pipeline. This is the base role for all video pipeline roles. -# - -video_pipeline_base_service_name: "video_pipeline_base" - -video_pipeline_base_val_base_url: "{{ VIDEO_PIPELINE_BASE_LMS_BASE_URL | default('http://127.0.0.1:8000') }}" - -VIDEO_PIPELINE_BASE_GUNICORN_PORT: 8555 -VIDEO_PIPELINE_BASE_NGINX_PORT: "1{{ VIDEO_PIPELINE_BASE_GUNICORN_PORT }}" -VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: "4{{ VIDEO_PIPELINE_BASE_GUNICORN_PORT }}" - -VIDEO_PIPELINE_BASE_URL_ROOT: "http://127.0.0.1:{{ VIDEO_PIPELINE_BASE_NGINX_PORT }}" - -VIDEO_PIPELINE_BASE_API_URL: "{{ VIDEO_PIPELINE_BASE_URL_ROOT }}/api" -VIDEO_PIPELINE_BASE_OAUTH_URL: "{{ VIDEO_PIPELINE_BASE_URL_ROOT }}/veda_auth" -VIDEO_PIPELINE_BASE_OAUTH_TOKEN_URL: "{{ VIDEO_PIPELINE_BASE_URL_ROOT }}/o/token" -VIDEO_PIPELINE_BASE_OAUTH_CLIENT_ID: "video-pipeline-client-id" -VIDEO_PIPELINE_BASE_OAUTH_CLIENT_NAME: "video-pipeline" -VIDEO_PIPELINE_BASE_OAUTH_CLIENT_SECRET: "video-pipeline-client-secret" - -VIDEO_PIPELINE_BASE_SECRET_KEY: "SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING" - -VIDEO_PIPELINE_BASE_DEFAULT_DB_NAME: "veda" -VIDEO_PIPELINE_BASE_MYSQL_HOST: "localhost" -VIDEO_PIPELINE_BASE_MYSQL_USER: "veda001" -VIDEO_PIPELINE_BASE_MYSQL_PASSWORD: "password" - -VIDEO_PIPELINE_BASE_RABBITMQ_BROKER: "localhost" -VIDEO_PIPELINE_BASE_RABBITMQ_USER: "celery" -VIDEO_PIPELINE_BASE_RABBITMQ_PASS: "celery" - -# video pipeline config overrides - -VIDEO_PIPELINE_BASE_EDX_S3_INGEST: - BUCKET: "SET-ME-PLEASE" - ROOT_PATH: "ingest/" - -VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES: - BUCKET: "SET-ME-PLEASE" - ROOT_PATH: "video-images/" - -VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS: - BUCKET: "SET-ME-PLEASE" - ROOT_PATH: "video-transcripts/" - -VIDEO_PIPELINE_BASE_EDX_CLOUDFRONT_PREFIX: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_EDX_S3_ENDPOINT_BUCKET: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_VEDA_S3_HOTSTORE_BUCKET: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_VEDA_DELIVERABLE_BUCKET: "SET-ME-PLEASE" - -VIDEO_PIPELINE_BASE_VEDA_BASE_URL: "{{ VIDEO_PIPELINE_BASE_URL_ROOT }}" -VIDEO_PIPELINE_BASE_VEDA_ACCESS_KEY_ID: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_VEDA_SECRET_ACCESS_KEY: "SET-ME-PLEASE" - -VIDEO_PIPELINE_BASE_ADMIN_EMAIL: "veda@example.com" -VIDEO_PIPELINE_BASE_VEDA_NOREPLY_EMAIL: "veda-noreply@example.com" - -VIDEO_PIPELINE_BASE_CIELO24_API_ENVIRONMENT: "sandbox" -VIDEO_PIPELINE_BASE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN: "SET-ME-TO-A-UNIQUE-LONG-RANDOM-STRING" - -VIDEO_PIPELINE_BASE_VAL_API_URL: "{{ video_pipeline_base_val_base_url }}/api/val/v0/videos" -VIDEO_PIPELINE_BASE_VAL_TOKEN_URL: "{{ video_pipeline_base_val_base_url }}/oauth2/access_token" -VIDEO_PIPELINE_BASE_VAL_VIDEO_IMAGES_URL: "{{ video_pipeline_base_val_base_url }}/api/val/v0/videos/video-images/update/" -VIDEO_PIPELINE_BASE_VAL_CLIENT_ID: "{{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_ID }}" -VIDEO_PIPELINE_BASE_VAL_SECRET_KEY: "{{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_SECRET }}" -VIDEO_PIPELINE_BASE_VAL_USERNAME: "staff" -VIDEO_PIPELINE_BASE_VAL_PASSWORD: "edx" -VIDEO_PIPELINE_BASE_VAL_TRANSCRIPT_CREATE_URL: "{{ video_pipeline_base_val_base_url }}/api/val/v0/videos/video-transcripts/create/" -VIDEO_PIPELINE_BASE_VAL_VIDEO_TRANSCRIPT_STATUS_URL: "{{ video_pipeline_base_val_base_url }}/api/val/v0/videos/status/" - -VIDEO_PIPELINE_BASE_SG_SERVER_PATH: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_SG_SCRIPT_NAME: "SET-ME-PLEASE" -VIDEO_PIPELINE_BASE_SG_SCRIPT_KEY: "SET-ME-PLEASE" - -VIDEO_PIPELINE_BASE_HOST_ENVIRONMENT: "sandbox" - -# video pipeline config - -video_pipeline_base_config_default: - DATABASES: - default: - ENGINE: 'django.db.backends.mysql' - NAME: "{{ VIDEO_PIPELINE_BASE_DEFAULT_DB_NAME }}" - USER: "{{ VIDEO_PIPELINE_BASE_MYSQL_USER }}" - PASSWORD: "{{ VIDEO_PIPELINE_BASE_MYSQL_PASSWORD }}" - HOST: "{{ VIDEO_PIPELINE_BASE_MYSQL_HOST }}" - PORT: 3306 - - SECRET_KEY: "{{ VIDEO_PIPELINE_BASE_SECRET_KEY }}" - - # django staticfile settings - STATIC_ROOT: "{{ COMMON_DATA_DIR }}/{{ video_pipeline_base_service_name }}/staticfiles" - - SERVICE_VARIANT_NAME: "{{ video_pipeline_base_service_name }}" - - # --- - # AWS Buckets, Prefixes - # --- - # Studio/Platform - edx_s3_ingest_prefix: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST.ROOT_PATH }}" - edx_s3_ingest_bucket: "{{ VIDEO_PIPELINE_BASE_EDX_S3_INGEST.BUCKET }}" - edx_s3_endpoint_bucket: "{{ VIDEO_PIPELINE_BASE_EDX_S3_ENDPOINT_BUCKET }}" - - # CF - edx_cloudfront_prefix: "{{ VIDEO_PIPELINE_BASE_EDX_CLOUDFRONT_PREFIX }}" - - # Images - aws_video_images_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES.BUCKET }}" - aws_video_images_prefix: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_IMAGES.ROOT_PATH }}" - - # Transcripts - aws_video_transcripts_bucket: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS.BUCKET }}" - aws_video_transcripts_prefix: "{{ VIDEO_PIPELINE_BASE_AWS_VIDEO_TRANSCRIPTS.ROOT_PATH }}" - - # VEDA Internal - veda_s3_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET }}" - veda_upload_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_UPLOAD_BUCKET }}" - veda_s3_hotstore_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_S3_HOTSTORE_BUCKET }}" - veda_deliverable_bucket: "{{ VIDEO_PIPELINE_BASE_VEDA_DELIVERABLE_BUCKET }}" - veda_base_url: "{{ VIDEO_PIPELINE_BASE_VEDA_BASE_URL }}" - s3_base_url: https://s3.amazonaws.com - - # cielo24 api urls - cielo24_api_base_url: 'https://{{ VIDEO_PIPELINE_BASE_CIELO24_API_ENVIRONMENT }}.cielo24.com/api' - - # 3playmedia api urls - three_play_api_base_url: https://api.3playmedia.com/ - three_play_api_transcript_url: https://static.3playmedia.com/ - - # a token identifying a valid request from transcript provider - transcript_provider_request_token: "{{ VIDEO_PIPELINE_BASE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN }}" - - # Ingest Secret - veda_secret_access_key: "{{ VIDEO_PIPELINE_BASE_VEDA_SECRET_ACCESS_KEY }}" - veda_access_key_id: "{{ VIDEO_PIPELINE_BASE_VEDA_ACCESS_KEY_ID }}" - - # --- - # email vars - # --- - veda_noreply_email: "{{ VIDEO_PIPELINE_BASE_VEDA_NOREPLY_EMAIL }}" - admin_email: "{{ VIDEO_PIPELINE_BASE_ADMIN_EMAIL }}" - - # --- - # VEDA API - # --- - ## VEDA API Auth - veda_api_url: "{{ VIDEO_PIPELINE_BASE_API_URL }}" - veda_auth_url: "{{ VIDEO_PIPELINE_BASE_OAUTH_URL }}" - veda_client_id: "{{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_ID }}" - veda_secret_key: "{{ VIDEO_PIPELINE_BASE_OAUTH_CLIENT_SECRET }}" - veda_token_url: "{{ VIDEO_PIPELINE_BASE_OAUTH_TOKEN_URL }}" - - # --- - # VAL - # --- - val_api_url: "{{ VIDEO_PIPELINE_BASE_VAL_API_URL }}" - val_token_url: "{{ VIDEO_PIPELINE_BASE_VAL_TOKEN_URL }}" - val_video_images_url: "{{ VIDEO_PIPELINE_BASE_VAL_VIDEO_IMAGES_URL }}" - # Credentials - val_client_id: "{{ VIDEO_PIPELINE_BASE_VAL_CLIENT_ID }}" - val_secret_key: "{{ VIDEO_PIPELINE_BASE_VAL_SECRET_KEY }}" - val_password: "{{ VIDEO_PIPELINE_BASE_VAL_PASSWORD }}" - val_username: "{{ VIDEO_PIPELINE_BASE_VAL_USERNAME }}" - val_transcript_create_url: "{{ VIDEO_PIPELINE_BASE_VAL_TRANSCRIPT_CREATE_URL }}" - val_video_transcript_status_url: "{{ VIDEO_PIPELINE_BASE_VAL_VIDEO_TRANSCRIPT_STATUS_URL }}" - - rabbitmq_broker: "{{ VIDEO_PIPELINE_BASE_RABBITMQ_BROKER }}" - rabbitmq_user: "{{ VIDEO_PIPELINE_BASE_RABBITMQ_USER }}" - rabbitmq_pass: "{{ VIDEO_PIPELINE_BASE_RABBITMQ_PASS }}" - onsite_worker: False - - # --- - # Shotgun Variables (internal mediateam) - # --- - sg_server_path: "{{ VIDEO_PIPELINE_BASE_SG_SERVER_PATH }}" - sg_script_name: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_NAME }}" - sg_script_key: "{{ VIDEO_PIPELINE_BASE_SG_SCRIPT_KEY }}" - - lms_base_url: "{{ video_pipeline_base_val_base_url }}" - instance_prefix: "{{ ansible_ec2_public_ipv4 }}" - -video_pipeline_base_config_extra: {} -VIDEO_PIPELINE_BASE_CONFIG: '{{ video_pipeline_base_config_default | combine(video_pipeline_base_config_extra) }}' diff --git a/playbooks/roles/video_pipeline_base/meta/main.yml b/playbooks/roles/video_pipeline_base/meta/main.yml deleted file mode 100644 index 39847fea450..00000000000 --- a/playbooks/roles/video_pipeline_base/meta/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -# Allow this role to be duplicated in dependencies. -allow_duplicates: yes diff --git a/playbooks/roles/video_pipeline_base/tasks/main.yml b/playbooks/roles/video_pipeline_base/tasks/main.yml deleted file mode 100644 index ad0fa941886..00000000000 --- a/playbooks/roles/video_pipeline_base/tasks/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -- name: Write out app config file - template: - src: "config.yml.j2" - dest: "{{ COMMON_CFG_DIR }}/{{ video_pipeline_base_service_name }}.yml" - mode: "0644" diff --git a/playbooks/roles/video_pipeline_base/templates/config.yml.j2 b/playbooks/roles/video_pipeline_base/templates/config.yml.j2 deleted file mode 100644 index 39f67872024..00000000000 --- a/playbooks/roles/video_pipeline_base/templates/config.yml.j2 +++ /dev/null @@ -1,2 +0,0 @@ ---- -{{ VIDEO_PIPELINE_BASE_CONFIG | to_nice_yaml }} diff --git a/playbooks/roles/video_pipeline_integration/defaults/main.yml b/playbooks/roles/video_pipeline_integration/defaults/main.yml deleted file mode 100644 index 96b46051ce5..00000000000 --- a/playbooks/roles/video_pipeline_integration/defaults/main.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for role video_pipeline_integration -# - -# -# vars are namespaced with the module name. -# - -EDXAPP_VIDEO_PIPELINE_INTEGRATION_CONFIG: - - client_name: "{{ EDXAPP_VEDA_SERVICE_CLIENT_NAME }}" - api_url: "{{ EDXAPP_VEDA_SERVICE_API_URL }}" - service_username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME }}" diff --git a/playbooks/roles/video_pipeline_integration/tasks/main.yml b/playbooks/roles/video_pipeline_integration/tasks/main.yml deleted file mode 100644 index 29ab9ea0cad..00000000000 --- a/playbooks/roles/video_pipeline_integration/tasks/main.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# -# -# Tasks for role video_pipeline_integration -# -# Overview: -# -# -# Dependencies: -# -# -# Example play: -# -# - -- name: create edxapp video pipeline integration - shell: > - {{ COMMON_APP_DIR }}/edxapp/venvs/edxapp/bin/python {{ COMMON_BIN_DIR }}/manage.edxapp lms --settings={{ COMMON_EDXAPP_SETTINGS }} - create_video_pipeline_integration {{ item.client_name}} {{ item.api_url }} {{ item.service_username }} --enabled - args: - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - with_items: "{{ EDXAPP_VIDEO_PIPELINE_INTEGRATION_CONFIG }}" - when: CREATE_SERVICE_WORKER_USERS and CREATE_EDXAPP_VIDEO_PIPELINE_INTEGRATION is defined diff --git a/playbooks/veda_delivery_worker.yml b/playbooks/veda_delivery_worker.yml deleted file mode 100644 index e6b2bb9523b..00000000000 --- a/playbooks/veda_delivery_worker.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: Deploy edX VEDA delivery Worker - hosts: all - gather_facts: True - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - veda_delivery_worker - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER diff --git a/playbooks/veda_encode_worker.yml b/playbooks/veda_encode_worker.yml deleted file mode 100644 index c9e5aea08e9..00000000000 --- a/playbooks/veda_encode_worker.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deploy edX VEDA Encode Worker - hosts: all - become: True - gather_facts: True - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - veda_encode_worker - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER diff --git a/playbooks/veda_intake_worker.yml b/playbooks/veda_intake_worker.yml deleted file mode 100644 index 5c01c2eccf1..00000000000 --- a/playbooks/veda_intake_worker.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: Deploy edX VEDA Intake Worker - hosts: all - gather_facts: True - roles: - - veda_intake_worker diff --git a/playbooks/veda_pipeline_worker.yml b/playbooks/veda_pipeline_worker.yml deleted file mode 100644 index d370d6b5dd0..00000000000 --- a/playbooks/veda_pipeline_worker.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Deploy edX VEDA pipeline Worker - hosts: all - become: True - gather_facts: True - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - veda_pipeline_worker - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER diff --git a/playbooks/veda_web_frontend.yml b/playbooks/veda_web_frontend.yml deleted file mode 100644 index 0b2be285611..00000000000 --- a/playbooks/veda_web_frontend.yml +++ /dev/null @@ -1,14 +0,0 @@ -- name: Deploy edX Video Pipeline Web Frontend - hosts: all - become: True - gather_facts: True - roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE - - role: nginx - nginx_default_sites: - - veda_web_frontend - - role: veda_web_frontend - - role: splunkforwarder - when: COMMON_ENABLE_SPLUNKFORWARDER - # TODO! Add new relic instrumentation once all the other pieces of video pipeline are in place. diff --git a/playbooks/video_pipeline_integration.yml b/playbooks/video_pipeline_integration.yml deleted file mode 100644 index 887fa3ea189..00000000000 --- a/playbooks/video_pipeline_integration.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: Create edxapp video pipeline integration - hosts: all - become: True - gather_facts: True - vars_files: - - "roles/common_vars/defaults/main.yml" - - "roles/edxapp/defaults/main.yml" - roles: - - video_pipeline_integration diff --git a/util/bake_config.sh b/util/bake_config.sh index 4c72e50b9d0..545a9bc456c 100755 --- a/util/bake_config.sh +++ b/util/bake_config.sh @@ -12,7 +12,7 @@ set -e # Optional envs you can override if you wish to render config for different EDPs # these are expected to be comma separated with no spaces, see defaults. ENVIRONMENT_DEPLOYMENTS=${ENVIRONMENT_DEPLOYMENTS:=stage-edx,prod-edx,prod-edge,developer-sandbox} -PLAYS=${PLAYS:=edxapp,veda_web_frontend,analyticsapi,credentials,ecommerce,discovery,ecomworker,insights,registrar,notes} +PLAYS=${PLAYS:=edxapp,analyticsapi,credentials,ecommerce,discovery,ecomworker,insights,registrar,notes} rm -rf $CONFIG_RENDERING_TARGET cd $WORKSPACE/configuration/playbooks diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index a528eff95e6..19e11b4334d 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -371,9 +371,6 @@ PAYMENT_MFE_VERSION: $payment_version PAYMENT_MFE_ENABLED: $payment PAYMENT_SANDBOX_BUILD: True -VIDEO_PIPELINE_BASE_NGINX_PORT: 80 -VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: 443 - LICENSE_MANAGER_NGINX_PORT: 80 LICENSE_MANAGER_SSL_NGINX_PORT: 443 LICENSE_MANAGER_VERSION: $license_manager_version @@ -568,14 +565,6 @@ CREDENTIALS_URL_ROOT: "https://{{ CREDENTIALS_DOMAIN }}" CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true CREDENTIALS_DISCOVERY_API_URL: "{{ DISCOVERY_URL_ROOT }}/api/v1/" -VIDEO_PIPELINE_DOMAIN: "veda-${deploy_host}" -VIDEO_PIPELINE_BASE_URL_ROOT: "https://{{ VIDEO_PIPELINE_DOMAIN }}" -VIDEO_PIPELINE_BASE_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}" - -VEDA_WEB_FRONTEND_VERSION: ${video_pipeline_version:-master} -VEDA_PIPELINE_WORKER_VERSION: ${video_pipeline_version:-master} -VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master} - LICENSE_MANAGER_URL_ROOT: "https://license-manager-${deploy_host}" COMMERCE_COORDINATOR_URL_ROOT: "https://commerce-coordinator-${deploy_host}" @@ -638,11 +627,6 @@ EOF fi fi -veda_web_frontend=${video_pipeline:-false} -veda_pipeline_worker=${video_pipeline:-false} -veda_encode_worker=${video_encode_worker:-false} -video_pipeline_integration=${video_pipeline:-false} - # ansible overrides for master's integration environment setup if [[ $masters_integration_environment == "true" ]]; then cat << EOF >> $extra_vars_file @@ -668,7 +652,7 @@ EOF fi declare -A deploy -plays="prospectus edxapp forum ecommerce credentials discovery enterprise_catalog analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration xqueue certs demo testcourses registrar program_console learner_portal" +plays="prospectus edxapp forum ecommerce credentials discovery enterprise_catalog analyticsapi xqueue certs demo testcourses registrar program_console learner_portal" for play in $plays; do deploy[$play]=${!play} From d3e9bac32c6eaa4b3e840b12eae1049668e49bb3 Mon Sep 17 00:00:00 2001 From: Syed Awais Ali Date: Wed, 30 Aug 2023 13:35:55 +0500 Subject: [PATCH 492/664] chore: remove ignore_error from prospectus --- playbooks/roles/prospectus/tasks/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index f0f57a006ca..f178f759a82 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -232,7 +232,6 @@ shell: "mv {{ prospectus_code_dir }}/public/* {{ PROSPECTUS_DATA_DIR }}" - name: Set Prospectus S3 max concurrency - ignore_errors: true become_user: "{{ prospectus_user }}" shell: 'aws configure set s3.max_concurrent_requests {{ PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS }}' when: PROSPECTUS_S3_UPLOAD_ENABLED|bool @@ -241,7 +240,6 @@ - install:system-requirements - name: Upload prospectus to S3 - ignore_errors: true become_user: "{{ prospectus_user }}" shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ PROSPECTUS_S3_HOSTING_PREFIX | default(PROSPECTUS_VERSION, true) }}' when: PROSPECTUS_S3_UPLOAD_ENABLED|bool @@ -250,7 +248,6 @@ - install:system-requirements - name: Download prospectus redirects to GoCD - ignore_errors: true when: PROSPECTUS_S3_UPLOAD_ENABLED|bool fetch: src: "{{ prospectus_redirect_file }}" From 84f6bafb98302701a9f034096281dc40640a8f27 Mon Sep 17 00:00:00 2001 From: k8 Date: Wed, 30 Aug 2023 08:55:38 -0600 Subject: [PATCH 493/664] chore: fix something with nodesource --- playbooks/roles/edxapp/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index eb62c343f58..ed478c1efbc 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -130,7 +130,7 @@ - name: Install the nodejs LTS repos apt_repository: - repo: "deb https://deb.nodesource.com/node_{{ EDXAPP_NODE_VERSION }}.x {{ ansible_distribution_release }} main" + repo: "deb https://deb.nodesource.com/node_{{ EDXAPP_NODE_VERSION }}.x nodistro main" state: present update_cache: yes register: add_repo From ab1486e755fdab535b9324a717e95da53df5b1d8 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 30 Aug 2023 11:46:17 -0400 Subject: [PATCH 494/664] feat: fix signature for nodesource installation --- playbooks/roles/edxapp/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index ed478c1efbc..7f0ca23b418 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -122,7 +122,7 @@ - name: Install the gpg key for nodejs LTS apt_key: - url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" + url: "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" state: present tags: - install From 0be4913415d2e90d2c16ddaa2bfca87915ea6c73 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 5 Sep 2023 17:19:46 -0400 Subject: [PATCH 495/664] fix: Custom prospectus S3 404 page ISRE-1998 --- .../nginx/templates/edx/app/nginx/sites-available/prospectus.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 44daa9c1f6e..ec31438ee4f 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -55,7 +55,7 @@ server { root {{ PROSPECTUS_DATA_DIR }}; add_header Cache-Control "no-store, max-age=0" always; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} - proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/404.html; # Prevent S3 errors from cookies being too large proxy_set_header Cookie ""; {% endif %} From a9daadf7ed857fc7a3309a0b120aec1255fb8d20 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 08:21:21 -0600 Subject: [PATCH 496/664] chore: remove docker-tools role --- playbooks/roles/aws_devstack/meta/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/meta/main.yml b/playbooks/roles/aws_devstack/meta/main.yml index f779906f1e9..fdda41bb3a1 100644 --- a/playbooks/roles/aws_devstack/meta/main.yml +++ b/playbooks/roles/aws_devstack/meta/main.yml @@ -1,4 +1,3 @@ --- dependencies: - role: common - - role: docker-tools From 5e9cd1b540abfd1ce18273d96f5afa353d513a78 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 08:42:13 -0600 Subject: [PATCH 497/664] chore: logic change for script execution --- playbooks/roles/aws_devstack/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index cf147575392..e5cc79e600b 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,8 +3,11 @@ # autoremove: yes # - name: Run script to sync all repos - ansible.builtin.script: - cmd: /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log + hosts: server + sudo: yes + tasks: + - name: Execute Script + - command: sh /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From fd3f6edac0c221d6daf2a9fb91df117db7626894 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 08:49:13 -0600 Subject: [PATCH 498/664] chore: update logic for sync script --- playbooks/roles/aws_devstack/tasks/main.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index e5cc79e600b..c42e1f71e89 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,11 +3,7 @@ # autoremove: yes # - name: Run script to sync all repos - hosts: server - sudo: yes - tasks: - - name: Execute Script - - command: sh /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log + command: sh /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 9c75d210cc898121692000f33a84254076ba9f8f Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 09:32:24 -0600 Subject: [PATCH 499/664] chore: new logic to repo sync --- playbooks/roles/aws_devstack/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index c42e1f71e89..2bdd6b071d3 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,7 +3,12 @@ # autoremove: yes # - name: Run script to sync all repos - command: sh /home/ubuntu/pull-all-repos-in-directory.sh > /home/ubuntu/pullrepos.log + shell: + cmd: | + find . -maxdepth 1 -type d -exec sh -c '(cd {} && git checkout master && git pull)' ';' + args: + chdir: /home/ubuntu/edx-repos + warn: false - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 299bf8f5e44e15fc3e5f236271ce9bbc766946fe Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 11:52:46 -0600 Subject: [PATCH 500/664] chore: add async logic to repo sync command --- playbooks/roles/aws_devstack/tasks/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 2bdd6b071d3..a4ea23e3d57 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -8,19 +8,21 @@ find . -maxdepth 1 -type d -exec sh -c '(cd {} && git checkout master && git pull)' ';' args: chdir: /home/ubuntu/edx-repos - warn: false + ignore_errors: True + async: 600 + poll: 60 - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" - name: Run make on dev.pull.large-and-slow community.general.make: - chdir: /home/ubuntu/edx-repos-devstack + chdir: /home/ubuntu/edx-repos/devstack target: dev.pull.large-and-slow - name: Run make on lms-shell community.general.make: - chdir: /home/ubuntu/edx-repos-devstack + chdir: /home/ubuntu/edx-repos/devstack target: lms-shell - name: Make requirements From d8e58f441970230a81bbf4087c8ebbfd7cc55a30 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 12:05:10 -0600 Subject: [PATCH 501/664] chore: update logic for sync command --- playbooks/roles/aws_devstack/tasks/main.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index a4ea23e3d57..4eaef8e19c9 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,14 +3,9 @@ # autoremove: yes # - name: Run script to sync all repos - shell: - cmd: | - find . -maxdepth 1 -type d -exec sh -c '(cd {} && git checkout master && git pull)' ';' + ansible.builtin.shell: /home/ubuntu/pull-all-repos-in-directory.sh args: - chdir: /home/ubuntu/edx-repos - ignore_errors: True - async: 600 - poll: 60 + chdir: /home/ubuntu - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 3805df400493dc197f9d156fe9c3668e4d3ff33b Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 13:14:07 -0600 Subject: [PATCH 502/664] chore: troubleshooting async with repo pull --- playbooks/roles/aws_devstack/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 4eaef8e19c9..59fa8c97bc1 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,9 +3,9 @@ # autoremove: yes # - name: Run script to sync all repos - ansible.builtin.shell: /home/ubuntu/pull-all-repos-in-directory.sh - args: - chdir: /home/ubuntu + command: pull-all-repos-in-directory.sh chdir=/home/ubuntu + async: 600 + poll: 10 - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From d3777adeb8d71f1c77652d46034d91b0c1f6927d Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 13:30:21 -0600 Subject: [PATCH 503/664] chore: adjust path logic to repo pull --- playbooks/roles/aws_devstack/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 59fa8c97bc1..c90b187d692 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,7 +3,9 @@ # autoremove: yes # - name: Run script to sync all repos - command: pull-all-repos-in-directory.sh chdir=/home/ubuntu + shell: + cmd: | + find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git checkout master && git pull)' ';' async: 600 poll: 10 From 806285affca3ffe044d08f404c4297815c885e5e Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 13:46:15 -0600 Subject: [PATCH 504/664] chore: remote checkout master from git logic --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index c90b187d692..40b3b75204a 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -5,7 +5,7 @@ - name: Run script to sync all repos shell: cmd: | - find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git checkout master && git pull)' ';' + find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' async: 600 poll: 10 From 4c36d53afac103e5c8d95a945e6debaee2a1c9f8 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 15:07:01 -0600 Subject: [PATCH 505/664] chore: Add github.com for known hosts --- playbooks/roles/aws_devstack/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 40b3b75204a..318e287bff6 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -2,6 +2,10 @@ # apt: # autoremove: yes # +- name: SSH Keyscan for github + ansible.builtin.shell: + cmd: ssh-keyscan -t rsa github.com >> /home/ubuntu/.ssh/known_hosts + - name: Run script to sync all repos shell: cmd: | From 26f4974c2921b55ef631dc38cb9c5f0c5be5a853 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Wed, 6 Sep 2023 15:28:00 -0600 Subject: [PATCH 506/664] chore: add become --- playbooks/roles/aws_devstack/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 318e287bff6..060d759ec2a 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -12,6 +12,7 @@ find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' async: 600 poll: 10 + become: true - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From 74958e7a98bc942300e8365431f7f7f147067eb8 Mon Sep 17 00:00:00 2001 From: Hamza Kazmi Date: Thu, 7 Sep 2023 15:42:36 +0500 Subject: [PATCH 507/664] chore: update jenkins new playbook --- playbooks/jenkins_data_engineering_new.yml | 2 +- playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/jenkins_data_engineering_new.yml b/playbooks/jenkins_data_engineering_new.yml index 5d7176f8198..08e2fd3a28c 100644 --- a/playbooks/jenkins_data_engineering_new.yml +++ b/playbooks/jenkins_data_engineering_new.yml @@ -23,7 +23,7 @@ COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE: True COMMON_SECURITY_UPDATES: yes SECURITY_UPGRADE_ON_ANSIBLE: true - ansible_distribution_release: bionic + ansible_distribution_release: focal roles: - role: aws diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index e4373020299..1d2aa6350a5 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -80,6 +80,9 @@ DE_JENKINS_VAULT_CLI_VERSION: 1.9.4 # plugins de_jenkins_plugins_list: + - name: 'audit-trail' + version: '3.4' + group: 'org.jenkins-ci.plugins' - name: 'ansicolor' version: '0.5.2' group: 'org.jenkins-ci.plugins' From 767c7cb373a27fc6287d851c22011fa25af4b732 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Fri, 8 Sep 2023 12:59:09 -0400 Subject: [PATCH 508/664] fix: Tag EC2 instance retries (#7012) --- playbooks/continuous_delivery/launch_instance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/continuous_delivery/launch_instance.yml b/playbooks/continuous_delivery/launch_instance.yml index 807b1b33f4d..d132c3fdc89 100644 --- a/playbooks/continuous_delivery/launch_instance.yml +++ b/playbooks/continuous_delivery/launch_instance.yml @@ -95,8 +95,10 @@ gocd_pipepline_label: "{{ lookup('ansible.builtin.env', 'GO_PIPELINE_LABEL')|default('', True) }}" gocd_pipepline_counter: "{{ lookup('ansible.builtin.env', 'GO_PIPELINE_COUNTER')|default('', True) }}" gocd_stage_counter: "{{ lookup('ansible.builtin.env', 'GO_STAGE_COUNTER')|default('', True) }}" + register: tag_ec2_instance retries: 6 delay: 15 + until: tag_ec2_instance is succeeded - name: Wait for SSH to come up wait_for: From 3537f254efbd5dbd0670e2093c4c7556d471790d Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 7 Sep 2023 16:43:28 -0400 Subject: [PATCH 509/664] fix: Hide all client headers from S3 This option hides cookies and all other client headers S3 doesn't need to see. --- .../app/nginx/sites-available/prospectus.j2 | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index ec31438ee4f..639160b9fde 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -56,8 +56,8 @@ server { add_header Cache-Control "no-store, max-age=0" always; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/404.html; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; {% endif %} } @@ -66,8 +66,8 @@ server { add_header X-Frame-Options ''; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; {% endif %} } @@ -76,8 +76,8 @@ server { add_header 'Cache-Control' 'public, max-age=31536000, immutable'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; {% endif %} } @@ -87,8 +87,8 @@ server { add_header 'Cache-Control' 'public, max-age=3600'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; {% endif %} } @@ -98,8 +98,8 @@ server { add_header 'Cache-Control' 'public, max-age=86400'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; {% endif %} } @@ -108,8 +108,8 @@ server { location /HealthCheck { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/HealthCheck/index.html; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -122,8 +122,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/es/bio/index.html; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -137,8 +137,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} rewrite ^ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html break; proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/bio/index.html; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive @@ -205,8 +205,8 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} location / { proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; - # Prevent S3 errors from cookies being too large - proxy_set_header Cookie ""; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive From 2a4652f149f38a005b2a182d2860c3a63b08b646 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 7 Sep 2023 19:25:26 -0400 Subject: [PATCH 510/664] fix: Redirect trailing slash to non-trailing slash --- .../app/nginx/sites-available/prospectus.j2 | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 639160b9fde..e4264ff6469 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -203,10 +203,28 @@ server { try_files $uri $uri/index.html =404; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + # Redirect URLs with trailing slashes to non-trailing slash equivalents + # Gatsby is setup to have the canonical URLs be the versions without trailing slashes + location ~ ^.+/$ { + # absolute_redirect off prevents adding http://localhost to the redirect + # We want a relative redirect, i.e. /learn/ -> /924c142-1/learn, not to http://localhost/924c142-1/learn + absolute_redirect off; + rewrite ^/(.*)/$ /$1 permanent; + } + location / { - proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + # absolute_redirect off prevents adding http://localhost to the redirect + # We want a relative redirect, i.e. /learn/ -> /924c142-1/learn, not to http://localhost/924c142-1/learn + absolute_redirect off; + # Adds trailing slash that S3 requires in order to serve index.html from a folder + # For instance in order to server edx.org/learn we need to ask s3 for {{ PROSPECTUS_S3_HOSTING_PREFIX }}/learn/ + rewrite ^/((.*/)*[^.]*[^/])$ /{{ PROSPECTUS_S3_HOSTING_PREFIX }}/$1/ break; + + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}/; + # Hide client headers from S3 to prevent request headers too big error proxy_pass_request_headers off; + # proxy_redirect ensures redirects from s3 are rewritten # For example it will fix a redirect from s3 to prevent /school/mitx from trying to redirect to /924c142-1/school/mitx/ # The second parameter being " " is to prevent nginx sticking http://hostname in front of the location directive From 95ed8c83ec0b46904b8420baacfaf6b9441de300 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Mon, 11 Sep 2023 21:43:11 -0600 Subject: [PATCH 511/664] chore: add creds fix for git pull, fix docker compose --- playbooks/roles/aws_devstack/tasks/main.yml | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 060d759ec2a..b6a4464f74f 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -2,6 +2,14 @@ # apt: # autoremove: yes # +- name: Install read-only ssh key for the devstack repos + copy: + content: "{{ DEVSTACK_GIT_IDENTITY }}" + dest: /home/ubuntu/.ssh/id_rsa.ppk + force: yes + owner: "ubuntu" + mode: "0600" + - name: SSH Keyscan for github ansible.builtin.shell: cmd: ssh-keyscan -t rsa github.com >> /home/ubuntu/.ssh/known_hosts @@ -17,6 +25,22 @@ - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" +- name: Uninstall any docker packages + shell: + cmd: | + pip uninstall docker docker-py docker-compose + async: 600 + poll: 10 + become: true + +- name: Reinstall docker compose + shell: + cmd: | + pip install docker-compose + async: 600 + poll: 10 + become: true + - name: Run make on dev.pull.large-and-slow community.general.make: chdir: /home/ubuntu/edx-repos/devstack @@ -44,3 +68,8 @@ - name : LMS migrate command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate + +- name: Delete read-only ssh key + ansible.builtin.file: + state: absent + path: /home/ubuntu/.ssh/id_rsa.ppk From db331d9f482d83dd67550bacd620900218004916 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 07:16:27 -0600 Subject: [PATCH 512/664] chore: yaml corrections --- playbooks/roles/aws_devstack/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index b6a4464f74f..1e3f7f75eb4 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -3,12 +3,12 @@ # autoremove: yes # - name: Install read-only ssh key for the devstack repos - copy: - content: "{{ DEVSTACK_GIT_IDENTITY }}" - dest: /home/ubuntu/.ssh/id_rsa.ppk - force: yes - owner: "ubuntu" - mode: "0600" + copy: + content: "{{ DEVSTACK_GIT_IDENTITY }}" + dest: /home/ubuntu/.ssh/id_rsa.ppk + force: yes + owner: "ubuntu" + mode: "0600" - name: SSH Keyscan for github ansible.builtin.shell: From 432d9426a055ee2b9be622c4f14bba3921863de9 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 07:47:36 -0600 Subject: [PATCH 513/664] chore: reference common git identity --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 1e3f7f75eb4..c17bd2b3b38 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -4,7 +4,7 @@ # - name: Install read-only ssh key for the devstack repos copy: - content: "{{ DEVSTACK_GIT_IDENTITY }}" + content: "{{ COMMON_GIT_IDENTITY }}" dest: /home/ubuntu/.ssh/id_rsa.ppk force: yes owner: "ubuntu" From b9119e48541a709d2b4b409bcdc77e8429fe028c Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 09:01:09 -0600 Subject: [PATCH 514/664] chore: remove ppk file extensions --- playbooks/roles/aws_devstack/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index c17bd2b3b38..511f44ca494 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -5,7 +5,7 @@ - name: Install read-only ssh key for the devstack repos copy: content: "{{ COMMON_GIT_IDENTITY }}" - dest: /home/ubuntu/.ssh/id_rsa.ppk + dest: /home/ubuntu/.ssh/id_rsa force: yes owner: "ubuntu" mode: "0600" @@ -72,4 +72,4 @@ - name: Delete read-only ssh key ansible.builtin.file: state: absent - path: /home/ubuntu/.ssh/id_rsa.ppk + path: /home/ubuntu/.ssh/id_rsa From 2b68c565dab74353f3036de9ac205c9adce82168 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 09:24:46 -0600 Subject: [PATCH 515/664] chore: remove become for git pull --- playbooks/roles/aws_devstack/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 511f44ca494..ce3f1eb3233 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -20,7 +20,6 @@ find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' async: 600 poll: 10 - become: true - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" From e9c5bd33b16f9bd7915a664b2da15377f2c5997b Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 10:02:43 -0600 Subject: [PATCH 516/664] chore: add all github host keys --- playbooks/roles/aws_devstack/tasks/main.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index ce3f1eb3233..57c5b64ff48 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -10,9 +10,23 @@ owner: "ubuntu" mode: "0600" -- name: SSH Keyscan for github +- name: Remove old github hostkey ansible.builtin.shell: - cmd: ssh-keyscan -t rsa github.com >> /home/ubuntu/.ssh/known_hosts + cmd: ssh-keygen -R github.com + +#- name: SSH Keyscan for github +# ansible.builtin.shell: +# cmd: ssh-keyscan -t rsa github.com >> /home/ubuntu/.ssh/known_hosts + +- name: add all github hostkeys + blockinfile: + state: present + insertafter: EOF + dest: /home/ubuntu/.ssh/known_hosts + content: | + github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl + github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= + github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= - name: Run script to sync all repos shell: From 0c102d34b10aff98046e08601538c0eef6f81c89 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 10:30:58 -0600 Subject: [PATCH 517/664] chore: add become_user ubuntu for steps --- playbooks/roles/aws_devstack/tasks/main.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 57c5b64ff48..c729f2e5990 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -13,6 +13,7 @@ - name: Remove old github hostkey ansible.builtin.shell: cmd: ssh-keygen -R github.com + become_user: ubuntu #- name: SSH Keyscan for github # ansible.builtin.shell: @@ -34,9 +35,11 @@ find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' async: 600 poll: 10 + become_user: ubuntu - name: Mark all directories as safe for git on devstack ansible.builtin.shell: "git config --global --add safe.directory '*'" + become_user: ubuntu - name: Uninstall any docker packages shell: @@ -44,7 +47,7 @@ pip uninstall docker docker-py docker-compose async: 600 poll: 10 - become: true + become_user: ubuntu - name: Reinstall docker compose shell: @@ -52,35 +55,41 @@ pip install docker-compose async: 600 poll: 10 - become: true + become_user: ubuntu - name: Run make on dev.pull.large-and-slow community.general.make: chdir: /home/ubuntu/edx-repos/devstack target: dev.pull.large-and-slow + become_user: ubuntu - name: Run make on lms-shell community.general.make: chdir: /home/ubuntu/edx-repos/devstack target: lms-shell + become_user: ubuntu - name: Make requirements community.general.make: chdir: /home/ubuntu/edx-repos/devstack target: requirements + become_user: ubuntu - name: Run npm ci community.general.npm: ci: true path: /home/ubuntu/edx-repos/devstack - + become_user: ubuntu + - name: Update paver assets ansible.builtin.shell: cmd: paver update_assets chdir: /home/ubuntu/edx-repos/devstack + become_user: ubuntu - name : LMS migrate command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate + become_user: ubuntu - name: Delete read-only ssh key ansible.builtin.file: From cc2d551844eb11981870c98d8f1168f0e3ad8103 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 10:57:15 -0600 Subject: [PATCH 518/664] chore: comment out docker-compose interactions --- playbooks/roles/aws_devstack/tasks/main.yml | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index c729f2e5990..f4cf1234fb7 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -41,21 +41,21 @@ ansible.builtin.shell: "git config --global --add safe.directory '*'" become_user: ubuntu -- name: Uninstall any docker packages - shell: - cmd: | - pip uninstall docker docker-py docker-compose - async: 600 - poll: 10 - become_user: ubuntu +#- name: Uninstall any docker packages +# shell: +# cmd: | +# pip uninstall -y docker docker-py docker-compose +# async: 600 +# poll: 10 +# become_user: ubuntu -- name: Reinstall docker compose - shell: - cmd: | - pip install docker-compose - async: 600 - poll: 10 - become_user: ubuntu +#- name: Reinstall docker compose +# shell: +# cmd: | +# pip install docker-compose +# async: 600 +# poll: 10 +# become_user: ubuntu - name: Run make on dev.pull.large-and-slow community.general.make: From b27396ddd5cbdee9273d49e4255fe7c47674e8af Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 11:46:55 -0600 Subject: [PATCH 519/664] chore: refactor docker-compose install --- playbooks/roles/aws_devstack/tasks/main.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index f4cf1234fb7..5648b5d044b 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -49,13 +49,12 @@ # poll: 10 # become_user: ubuntu -#- name: Reinstall docker compose -# shell: -# cmd: | -# pip install docker-compose -# async: 600 -# poll: 10 -# become_user: ubuntu +- name: Reinstall docker compose + shell: + cmd: | + apt-get install -y docker-compose + async: 600 + poll: 10 - name: Run make on dev.pull.large-and-slow community.general.make: From 813496ebbc758497c60498508f1ca6e02291bef7 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 12:52:47 -0600 Subject: [PATCH 520/664] chore: test logic for all update steps in one task --- playbooks/roles/aws_devstack/tasks/main.yml | 97 ++++++++++++--------- 1 file changed, 57 insertions(+), 40 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 5648b5d044b..d36c6ac1b4a 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -32,14 +32,31 @@ - name: Run script to sync all repos shell: cmd: | - find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' + cd ~ + ./pull-all-repos-in-directory.sh + cd edx-repos/devstack/ + make dev.pull + make dev.up.lms + make lms.shell + make requirements + npm ci + paver update_assets + python manage.py lms migrate async: 600 - poll: 10 - become_user: ubuntu + poll: 10 + become_user: ubuntu -- name: Mark all directories as safe for git on devstack - ansible.builtin.shell: "git config --global --add safe.directory '*'" - become_user: ubuntu +#- name: Run script to sync all repos +# shell: +# cmd: | +# find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' +# async: 600 +# poll: 10 +# become_user: ubuntu +# +#- name: Mark all directories as safe for git on devstack +# ansible.builtin.shell: "git config --global --add safe.directory '*'" +# become_user: ubuntu #- name: Uninstall any docker packages # shell: @@ -49,46 +66,46 @@ # poll: 10 # become_user: ubuntu -- name: Reinstall docker compose - shell: - cmd: | - apt-get install -y docker-compose - async: 600 - poll: 10 +#- name: Reinstall docker compose +# shell: +# cmd: | +# apt-get install -y docker-compose +# async: 600 +# poll: 10 -- name: Run make on dev.pull.large-and-slow - community.general.make: - chdir: /home/ubuntu/edx-repos/devstack - target: dev.pull.large-and-slow - become_user: ubuntu +#- name: Run make on dev.pull.large-and-slow +# community.general.make: +# chdir: /home/ubuntu/edx-repos/devstack +# target: dev.pull.large-and-slow +# become_user: ubuntu -- name: Run make on lms-shell - community.general.make: - chdir: /home/ubuntu/edx-repos/devstack - target: lms-shell - become_user: ubuntu +#- name: Run make on lms-shell +# community.general.make: +# chdir: /home/ubuntu/edx-repos/devstack +# target: lms-shell +# become_user: ubuntu -- name: Make requirements - community.general.make: - chdir: /home/ubuntu/edx-repos/devstack - target: requirements - become_user: ubuntu +#- name: Make requirements +# community.general.make: +# chdir: /home/ubuntu/edx-repos/devstack +# target: requirements +# become_user: ubuntu -- name: Run npm ci - community.general.npm: - ci: true - path: /home/ubuntu/edx-repos/devstack - become_user: ubuntu +#- name: Run npm ci +# community.general.npm: +# ci: true +# path: /home/ubuntu/edx-repos/devstack +# become_user: ubuntu -- name: Update paver assets - ansible.builtin.shell: - cmd: paver update_assets - chdir: /home/ubuntu/edx-repos/devstack - become_user: ubuntu +#- name: Update paver assets +# ansible.builtin.shell: +# cmd: paver update_assets +# chdir: /home/ubuntu/edx-repos/devstack +# become_user: ubuntu -- name : LMS migrate - command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate - become_user: ubuntu +#- name : LMS migrate +# command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate +# become_user: ubuntu - name: Delete read-only ssh key ansible.builtin.file: From 9612f2c17e0e490cb58276f855e9721bf747ee9e Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 12:58:01 -0600 Subject: [PATCH 521/664] chore: fix spacing --- playbooks/roles/aws_devstack/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index d36c6ac1b4a..f9038591508 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -43,8 +43,8 @@ paver update_assets python manage.py lms migrate async: 600 - poll: 10 - become_user: ubuntu + poll: 10 + become_user: ubuntu #- name: Run script to sync all repos # shell: From cff5a91ef2d2d4096ca9cbb5658c97d8c8951846 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 13:38:24 -0600 Subject: [PATCH 522/664] chore: change common to user role dependency --- playbooks/roles/aws_devstack/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/meta/main.yml b/playbooks/roles/aws_devstack/meta/main.yml index fdda41bb3a1..046f8425d4f 100644 --- a/playbooks/roles/aws_devstack/meta/main.yml +++ b/playbooks/roles/aws_devstack/meta/main.yml @@ -1,3 +1,3 @@ --- dependencies: - - role: common + - role: user From adf59adcd9f812e87cb37ba97a989736f964d4de Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 12 Sep 2023 14:06:48 -0600 Subject: [PATCH 523/664] chore: remove aws role --- playbooks/devstack_ami.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/devstack_ami.yml b/playbooks/devstack_ami.yml index 8f8631f8a67..7a5807f179a 100644 --- a/playbooks/devstack_ami.yml +++ b/playbooks/devstack_ami.yml @@ -6,6 +6,6 @@ ENABLE_DATADOG: False ENABLE_NEWRELIC: False roles: - - role: aws - when: COMMON_ENABLE_AWS_ROLE +# - role: aws +# when: COMMON_ENABLE_AWS_ROLE - role: aws_devstack From c6025c82209c95504068107a9481ef946debaebf Mon Sep 17 00:00:00 2001 From: Alie Langston Date: Wed, 13 Sep 2023 13:20:35 -0400 Subject: [PATCH 524/664] fix: add pkg-config to insights --- playbooks/roles/insights/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/insights/defaults/main.yml b/playbooks/roles/insights/defaults/main.yml index 01021c59fc2..c673e7fe4e9 100644 --- a/playbooks/roles/insights/defaults/main.yml +++ b/playbooks/roles/insights/defaults/main.yml @@ -240,6 +240,7 @@ insights_debian_pkgs: - libssl-dev # needed for mysqlclient python library - build-essential - gettext + - pkg-config - python3-pip - python3-dev From ec9a68af848ba8aa0e422acc890d8fdcc6862efb Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 09:10:49 -0600 Subject: [PATCH 525/664] Chore: Add source for bash profile --- playbooks/roles/aws_devstack/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index f9038591508..dd44472a491 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -33,9 +33,10 @@ shell: cmd: | cd ~ + source .profile ./pull-all-repos-in-directory.sh cd edx-repos/devstack/ - make dev.pull + make dev.pull.large-and-slow make dev.up.lms make lms.shell make requirements From ae012cd23686f55e2f0184e110bd2c599bb7cdf7 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 11:56:43 -0600 Subject: [PATCH 526/664] chore: modify syntax for shell --- playbooks/roles/aws_devstack/tasks/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index dd44472a491..bf83730ecfe 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -30,10 +30,9 @@ github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= - name: Run script to sync all repos - shell: - cmd: | - cd ~ - source .profile + shell: | + cd /home/ubuntu + source /home/ubuntu/.profile ./pull-all-repos-in-directory.sh cd edx-repos/devstack/ make dev.pull.large-and-slow From 230e74a4ffdd7da9cb8b69019738e91b5f85e00b Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 12:11:03 -0600 Subject: [PATCH 527/664] Chore: add time to async task --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index bf83730ecfe..f5973d1ce17 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -42,7 +42,7 @@ npm ci paver update_assets python manage.py lms migrate - async: 600 + async: 1200 poll: 10 become_user: ubuntu From 52e3b51ffdd85bac62a7e15104cd6c1cbd8e22b4 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 12:37:14 -0600 Subject: [PATCH 528/664] chore: typo fix, add time to async --- playbooks/roles/aws_devstack/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index f5973d1ce17..3fb47a381e4 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,12 +37,12 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms - make lms.shell + make lms-shell make requirements npm ci paver update_assets python manage.py lms migrate - async: 1200 + async: 1800 poll: 10 become_user: ubuntu From 9fee7d966831094d09bdc00a8f6ec19b67685b63 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 13:16:39 -0600 Subject: [PATCH 529/664] chore: wrangle lms shell commands --- playbooks/roles/aws_devstack/tasks/main.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 3fb47a381e4..382a15bcac3 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,11 +37,15 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms - make lms-shell - make requirements - npm ci - paver update_assets - python manage.py lms migrate +# make lms-shell + docker compose exec lms env TERM=xterm-256color bash -c 'make requirements' +# make requirements + docker compose exec lms env TERM=xterm-256color bash -c 'npm ci' +# npm ci + docker compose exec lms env TERM=xterm-256color bash -c 'paver update_assets' +# paver update_assets + docker compose exec lms env TERM=xterm-256color bash -c 'python manage.py lms migrate' +# python manage.py lms migrate async: 1800 poll: 10 become_user: ubuntu From 82c56abb0918698dccc8476055407c257a5263e6 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 14 Sep 2023 13:21:43 -0600 Subject: [PATCH 530/664] chore: combine lms shell commands --- playbooks/roles/aws_devstack/tasks/main.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 382a15bcac3..213b7718721 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,15 +37,7 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms -# make lms-shell - docker compose exec lms env TERM=xterm-256color bash -c 'make requirements' -# make requirements - docker compose exec lms env TERM=xterm-256color bash -c 'npm ci' -# npm ci - docker compose exec lms env TERM=xterm-256color bash -c 'paver update_assets' -# paver update_assets - docker compose exec lms env TERM=xterm-256color bash -c 'python manage.py lms migrate' -# python manage.py lms migrate + docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' async: 1800 poll: 10 become_user: ubuntu From 0f9671de59b331b35e33ac75ecc92953d43f2810 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Fri, 15 Sep 2023 11:13:35 -0600 Subject: [PATCH 531/664] chore: source profile prior to docker compose exec --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 213b7718721..aaa8fcbe3a7 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,7 +37,7 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms - docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' + source /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' async: 1800 poll: 10 become_user: ubuntu From 4079238db4c4807f2ed663c6d5e0418b71e4ac21 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Tue, 12 Sep 2023 15:57:04 -0400 Subject: [PATCH 532/664] chore: Remove support for Ubntu older than 20.04 --- playbooks/roles/docker-tools/defaults/main.yml | 5 ----- playbooks/roles/docker-tools/tasks/main.yml | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/playbooks/roles/docker-tools/defaults/main.yml b/playbooks/roles/docker-tools/defaults/main.yml index 8d46f0513d6..003e6dd7342 100644 --- a/playbooks/roles/docker-tools/defaults/main.yml +++ b/playbooks/roles/docker-tools/defaults/main.yml @@ -1,11 +1,6 @@ cache_valid_time: 3600 docker_tools_deps_deb_pkgs: - - apt-transport-https - - ca-certificates - - python-pip - -docker_tools_deps_deb_pkgs_focal: - apt-transport-https - ca-certificates - python3-pip diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml index 34e010eb769..4a8fd324c32 100644 --- a/playbooks/roles/docker-tools/tasks/main.yml +++ b/playbooks/roles/docker-tools/tasks/main.yml @@ -28,16 +28,6 @@ - install:system-requirements when: ansible_distribution_release != 'focal' -- name: install package dependencies for focal - apt: - name: "{{ docker_tools_deps_deb_pkgs_focal }}" - update_cache: yes - cache_valid_time: "{{ cache_valid_time }}" - tags: - - install - - install:system-requirements - when: ansible_distribution_release == 'focal' - - name: add docker apt key apt_key: url: "{{ docker_apt_key_url }}" From 29d183fa05435bf3fd5ee3571b670ec773869af7 Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Thu, 14 Sep 2023 15:16:37 -0400 Subject: [PATCH 533/664] chore: Fix docker-tools for post-Focal Ubuntu --- playbooks/roles/docker-tools/tasks/main.yml | 53 ++++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/playbooks/roles/docker-tools/tasks/main.yml b/playbooks/roles/docker-tools/tasks/main.yml index 4a8fd324c32..49f61c581de 100644 --- a/playbooks/roles/docker-tools/tasks/main.yml +++ b/playbooks/roles/docker-tools/tasks/main.yml @@ -28,20 +28,47 @@ - install:system-requirements when: ansible_distribution_release != 'focal' -- name: add docker apt key - apt_key: - url: "{{ docker_apt_key_url }}" - tags: - - install - - install:configuration +- name: Docker repo Ubuntu Focal + when: ansible_distribution_release == 'focal' + block: + - name: add docker apt key Ubuntu Focal + apt_key: + url: "{{ docker_apt_key_url }}" + tags: + - install + - install:configuration -- name: add docker repo - apt_repository: - repo: "{{ item }}" - with_items: "{{ docker_repos }}" - tags: - - install - - install:configuration + - name: add docker repo Ubuntu Focal + ansible.builtin.apt_repository: + repo: "{{ item }}" + filename: 'docker' + with_items: "{{ docker_repos }}" + tags: + - install + - install:configuration + +- name: Docker repo post Ubuntu Focal + when: ansible_distribution_release != 'focal' + block: + - name: download docker repo key post Ubuntu Focal + ansible.builtin.get_url: + url: "{{ docker_apt_key_url }}" + dest: /etc/apt/keyrings/docker.asc + +# Apt can handle armored .asc files, but the offical docker instructions tell people to dearmor the key. +# If we try to use the asc file on a box with docker already installed ansible will create a duplicate +# entry and apt will complain with the error "Conflicting values set for option Signed-By..." + - name: Dearmor docker repo gpg key post Ubuntu Focal + ansible.builtin.shell: + cmd: 'gpg --dearmor < /etc/apt/keyrings/docker.asc > /etc/apt/keyrings/docker.gpg' + + - name: add docker repo post Ubuntu Focal + apt_repository: + repo: "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + filename: 'docker' + tags: + - install + - install:configuration - name: install docker-engine apt: From 3136212944f094f55785c0f5133a4f8a396fed6c Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 15 Sep 2023 18:06:19 -0400 Subject: [PATCH 534/664] feat: Add cache to prospectus S3 proxy ISRE-2053 --- playbooks/roles/nginx/defaults/main.yml | 1 + playbooks/roles/nginx/tasks/main.yml | 1 + .../templates/edx/app/nginx/sites-available/prospectus.j2 | 8 ++++++-- playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 | 4 ++++ playbooks/roles/prospectus/defaults/main.yml | 1 + 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 4b5be5f65af..20aa8cc4ad7 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -132,6 +132,7 @@ NGINX_APT_REPO: deb http://nginx.org/packages/ubuntu/ {{ ansible_distribution_re nginx_app_dir: "{{ COMMON_APP_DIR }}/nginx" nginx_data_dir: "{{ COMMON_DATA_DIR }}/nginx" nginx_server_static_dir: "{{ nginx_data_dir }}/server-static" +nginx_server_cache_dir: "{{ nginx_data_dir }}/cache" nginx_conf_dir: "{{ nginx_app_dir }}/conf.d" nginx_log_dir: "{{ COMMON_LOG_DIR }}/nginx" nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available" diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 592c22b1d20..28d93ad2898 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -35,6 +35,7 @@ - { path: '{{ nginx_data_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_log_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_server_static_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } + - { path: '{{ nginx_server_cache_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } notify: restart nginx tags: - install diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index e4264ff6469..c37b98752a5 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -11,9 +11,13 @@ server { add_header X-Frame-Options DENY; - {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} +{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} resolver 127.0.0.53; - {% endif %} +{% if PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} + proxy_cache STATIC; + proxy_cache_valid any 1m; +{% endif %} +{% endif %} {% if NGINX_PROSPECTUS_DISABLE_INDEXING %} diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index 1c3494e5434..e5c231d8b3c 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -139,6 +139,10 @@ http { gzip_http_version 1.1; gzip_types text/plain text/css text/xml application/xml application/xml+rss image/svg+xml application/json application/javascript application/x-javascript text/javascript; +{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED and PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} + proxy_cache_path {{ nginx_server_cache_dir }} levels=1:2 keys_zone=STATIC:10m max_size=10g use_temp_path=off; +{% endif %} + ## # Virtual Host Configs ## diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 493a8da90ea..9577543f0e5 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -69,6 +69,7 @@ PROSPECTUS_ENABLE_NPM_BUILD: True PROSPECTUS_ENABLE_POST_BUILD: True PROSPECTUS_S3_HOSTING_PROXY_ENABLED: False +PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED: False PROSPECTUS_S3_UPLOAD_ENABLED: False PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 PROSPECTUS_S3_HOSTING_BUCKET: !!null From e5775fca6a10c81a1d18d224316ecda569fe597b Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Mon, 18 Sep 2023 09:21:54 -0400 Subject: [PATCH 535/664] feat: Add prospectus cache status header ISRE-2053 --- .../nginx/templates/edx/app/nginx/sites-available/prospectus.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index c37b98752a5..2a5bc40c55f 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -16,6 +16,7 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} proxy_cache STATIC; proxy_cache_valid any 1m; + add_header X-nginx-cache-status $upstream_cache_status; {% endif %} {% endif %} From 49ea3211ff5774c42459160d17311db97291e3ad Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Mon, 18 Sep 2023 11:15:02 -0400 Subject: [PATCH 536/664] Revert "feat: Add prospectus cache status header ISRE-2053" This reverts commit e5775fca6a10c81a1d18d224316ecda569fe597b. --- .../nginx/templates/edx/app/nginx/sites-available/prospectus.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 2a5bc40c55f..c37b98752a5 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -16,7 +16,6 @@ server { {% if PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} proxy_cache STATIC; proxy_cache_valid any 1m; - add_header X-nginx-cache-status $upstream_cache_status; {% endif %} {% endif %} From d4f516dc2aa0bb2f034996325752f3187379663f Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Mon, 18 Sep 2023 11:15:02 -0400 Subject: [PATCH 537/664] Revert "feat: Add cache to prospectus S3 proxy ISRE-2053" This reverts commit 3136212944f094f55785c0f5133a4f8a396fed6c. --- playbooks/roles/nginx/defaults/main.yml | 1 - playbooks/roles/nginx/tasks/main.yml | 1 - .../templates/edx/app/nginx/sites-available/prospectus.j2 | 8 ++------ playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 | 4 ---- playbooks/roles/prospectus/defaults/main.yml | 1 - 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 20aa8cc4ad7..4b5be5f65af 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -132,7 +132,6 @@ NGINX_APT_REPO: deb http://nginx.org/packages/ubuntu/ {{ ansible_distribution_re nginx_app_dir: "{{ COMMON_APP_DIR }}/nginx" nginx_data_dir: "{{ COMMON_DATA_DIR }}/nginx" nginx_server_static_dir: "{{ nginx_data_dir }}/server-static" -nginx_server_cache_dir: "{{ nginx_data_dir }}/cache" nginx_conf_dir: "{{ nginx_app_dir }}/conf.d" nginx_log_dir: "{{ COMMON_LOG_DIR }}/nginx" nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available" diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 28d93ad2898..592c22b1d20 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -35,7 +35,6 @@ - { path: '{{ nginx_data_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_log_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_server_static_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - - { path: '{{ nginx_server_cache_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } notify: restart nginx tags: - install diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index c37b98752a5..e4264ff6469 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -11,13 +11,9 @@ server { add_header X-Frame-Options DENY; -{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} resolver 127.0.0.53; -{% if PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} - proxy_cache STATIC; - proxy_cache_valid any 1m; -{% endif %} -{% endif %} + {% endif %} {% if NGINX_PROSPECTUS_DISABLE_INDEXING %} diff --git a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 index e5c231d8b3c..1c3494e5434 100644 --- a/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 +++ b/playbooks/roles/nginx/templates/etc/nginx/nginx.conf.j2 @@ -139,10 +139,6 @@ http { gzip_http_version 1.1; gzip_types text/plain text/css text/xml application/xml application/xml+rss image/svg+xml application/json application/javascript application/x-javascript text/javascript; -{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED and PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} - proxy_cache_path {{ nginx_server_cache_dir }} levels=1:2 keys_zone=STATIC:10m max_size=10g use_temp_path=off; -{% endif %} - ## # Virtual Host Configs ## diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 9577543f0e5..493a8da90ea 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -69,7 +69,6 @@ PROSPECTUS_ENABLE_NPM_BUILD: True PROSPECTUS_ENABLE_POST_BUILD: True PROSPECTUS_S3_HOSTING_PROXY_ENABLED: False -PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED: False PROSPECTUS_S3_UPLOAD_ENABLED: False PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 PROSPECTUS_S3_HOSTING_BUCKET: !!null From 0ede094714c0f35adf4a84cec71fc3831aa1f46d Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 15 Sep 2023 18:06:19 -0400 Subject: [PATCH 538/664] feat: Add cache to prospectus S3 proxy ISRE-2053 --- playbooks/roles/nginx/defaults/main.yml | 1 + playbooks/roles/nginx/tasks/main.yml | 1 + .../edx/app/nginx/sites-available/prospectus.j2 | 14 ++++++++++++-- playbooks/roles/prospectus/defaults/main.yml | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 4b5be5f65af..20aa8cc4ad7 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -132,6 +132,7 @@ NGINX_APT_REPO: deb http://nginx.org/packages/ubuntu/ {{ ansible_distribution_re nginx_app_dir: "{{ COMMON_APP_DIR }}/nginx" nginx_data_dir: "{{ COMMON_DATA_DIR }}/nginx" nginx_server_static_dir: "{{ nginx_data_dir }}/server-static" +nginx_server_cache_dir: "{{ nginx_data_dir }}/cache" nginx_conf_dir: "{{ nginx_app_dir }}/conf.d" nginx_log_dir: "{{ COMMON_LOG_DIR }}/nginx" nginx_sites_available_dir: "{{ nginx_app_dir }}/sites-available" diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 592c22b1d20..28d93ad2898 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -35,6 +35,7 @@ - { path: '{{ nginx_data_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_log_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } - { path: '{{ nginx_server_static_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } + - { path: '{{ nginx_server_cache_dir }}', owner: '{{ common_web_user }}', group: '{{ nginx_user }}' } notify: restart nginx tags: - install diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index e4264ff6469..be8ade41525 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -4,6 +4,11 @@ {%- set default_site = "" -%} {%- endif -%} + +{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED and PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} +proxy_cache_path {{ nginx_server_cache_dir }} levels=1:2 keys_zone=STATIC:10m max_size=10g use_temp_path=off; +{% endif %} + server { # Prospectus configuration file for nginx, templated by ansible @@ -11,9 +16,14 @@ server { add_header X-Frame-Options DENY; - {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} +{% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} resolver 127.0.0.53; - {% endif %} +{% if PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED %} + proxy_cache STATIC; + proxy_cache_valid any 1m; + add_header X-nginx-cache-status $upstream_cache_status; +{% endif %} +{% endif %} {% if NGINX_PROSPECTUS_DISABLE_INDEXING %} diff --git a/playbooks/roles/prospectus/defaults/main.yml b/playbooks/roles/prospectus/defaults/main.yml index 493a8da90ea..9577543f0e5 100644 --- a/playbooks/roles/prospectus/defaults/main.yml +++ b/playbooks/roles/prospectus/defaults/main.yml @@ -69,6 +69,7 @@ PROSPECTUS_ENABLE_NPM_BUILD: True PROSPECTUS_ENABLE_POST_BUILD: True PROSPECTUS_S3_HOSTING_PROXY_ENABLED: False +PROSPECTUS_S3_HOSTING_PROXY_CACHE_ENABLED: False PROSPECTUS_S3_UPLOAD_ENABLED: False PROSPECTUS_S3_UPLOAD_MAX_CONCURENCT_REQUESTS: 50 PROSPECTUS_S3_HOSTING_BUCKET: !!null From ea67f93efa40a5798788c6a328c6a089b90f87f9 Mon Sep 17 00:00:00 2001 From: connorhaugh <49422820+connorhaugh@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:08:43 -0400 Subject: [PATCH 539/664] feat: add authoring MFEs to sandboxes (#6777) * feat: add authoring MFEs to sandboxes In order to accomplish https://2u-internal.atlassian.net/browse/TNL-9975 I need to get the course authoring MFE on a sandbox. I anticipate we will also soon the the library authoring MFE for other BD-14 related work. --- playbooks/course_authoring.yml | 16 +++++++++++ playbooks/edx_continuous_integration.yml | 6 ++++ playbooks/library_authoring.yml | 16 +++++++++++ .../roles/course_authoring/defaults/main.yml | 3 ++ .../roles/course_authoring/meta/main.yml | 4 +++ .../roles/course_authoring/tasks/main.yml | 5 ++++ .../roles/edx_ansible/templates/update.j2 | 2 ++ playbooks/roles/launch_ec2/tasks/main.yml | 3 +- .../roles/library_authoring/defaults/main.yml | 2 ++ .../roles/library_authoring/meta/main.yml | 4 +++ .../roles/library_authoring/tasks/main.yml | 5 ++++ util/jenkins/ansible-provision.sh | 28 +++++++++++++++++++ 12 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 playbooks/course_authoring.yml create mode 100644 playbooks/library_authoring.yml create mode 100644 playbooks/roles/course_authoring/defaults/main.yml create mode 100644 playbooks/roles/course_authoring/meta/main.yml create mode 100644 playbooks/roles/course_authoring/tasks/main.yml create mode 100644 playbooks/roles/library_authoring/defaults/main.yml create mode 100644 playbooks/roles/library_authoring/meta/main.yml create mode 100644 playbooks/roles/library_authoring/tasks/main.yml diff --git a/playbooks/course_authoring.yml b/playbooks/course_authoring.yml new file mode 100644 index 00000000000..8e6fae9fb04 --- /dev/null +++ b/playbooks/course_authoring.yml @@ -0,0 +1,16 @@ +- name: Deploy the Course Authoring MFE + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'course-authoring' + COURSE_AUTHORING_ENABLED: True + COURSE_AUTHORING_SANDBOX_BUILD: False + roles: + - role: course_authoring + MFE_NAME: course-authoring + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELICE_INFRASTRUCTURE \ No newline at end of file diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 62d56e2feea..012f423e69d 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -65,6 +65,12 @@ - role: mfe MFE_NAME: learning when: LEARNING_MFE_ENABLED + - role: course_authoring + MFE_NAME: course-authoring + when: COURSE_AUTHORING_MFE_ENABLED + - role: mfe + MFE_NAME: library-authoring + when: LIBRARY_AUTHORING_MFE_ENABLED - role: mfe MFE_NAME: ora-grading when: ORA_GRADING_MFE_ENABLED diff --git a/playbooks/library_authoring.yml b/playbooks/library_authoring.yml new file mode 100644 index 00000000000..b5881bb25c0 --- /dev/null +++ b/playbooks/library_authoring.yml @@ -0,0 +1,16 @@ +- name: Deploy the Library Authoring MFE + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'library-authoring' + LIBRARY_AUTHORING_ENABLED: True + LIBRARY_AUTHORING_SANDBOX_BUILD: False + roles: + - role: library_authoring + MFE_NAME: library-authoring + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELICE_INFRASTRUCTURE \ No newline at end of file diff --git a/playbooks/roles/course_authoring/defaults/main.yml b/playbooks/roles/course_authoring/defaults/main.yml new file mode 100644 index 00000000000..c2b03296969 --- /dev/null +++ b/playbooks/roles/course_authoring/defaults/main.yml @@ -0,0 +1,3 @@ +course_authoring_env_extra: + STUDIO_BASE_URL: 'https://studio-{{COMMON_DEPLOY_HOSTNAME}}' + diff --git a/playbooks/roles/course_authoring/meta/main.yml b/playbooks/roles/course_authoring/meta/main.yml new file mode 100644 index 00000000000..60343ce5141 --- /dev/null +++ b/playbooks/roles/course_authoring/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - common + - nginx \ No newline at end of file diff --git a/playbooks/roles/course_authoring/tasks/main.yml b/playbooks/roles/course_authoring/tasks/main.yml new file mode 100644 index 00000000000..bcf0fb3ae72 --- /dev/null +++ b/playbooks/roles/course_authoring/tasks/main.yml @@ -0,0 +1,5 @@ +- name: Build Course Authoring MFE + include_role: + name: mfe + vars: + MFE_ENVIRONMENT_EXTRA: '{{ course_authoring_env_extra | default(MFE_DEPLOY_ENVIRONMENT_EXTRA) }}' \ No newline at end of file diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index 6c40c60d4b5..e838b537f05 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -81,6 +81,8 @@ repos_to_cmd["prospectus"]="$edx_ansible_cmd prospectus.yml -e 'PROSPECTUS_VERSI repos_to_cmd["authn"]="$edx_ansible_cmd authn_frontend.yml -e 'AUTHN_MFE_VERSION=$2'" repos_to_cmd["payment"]="$edx_ansible_cmd payment.yml -e 'PAYMENT_MFE_VERSION=$2'" repos_to_cmd["learning"]="$edx_ansible_cmd learning.yml -e 'LEARNING_MFE_VERSION=$2'" +repos_to_cmd["course_authoring"]="$edx_ansible_cmd course_authoring.yml -e 'COURSE_AUTHORING_MFE_VERSION=$2'" +repos_to_cmd["library_authoring"]="$edx_ansible_cmd library_authoring.yml -e 'COURSE_AUTHORING_MFE_VERSION=$2'" repos_to_cmd["ora_grading"]="$edx_ansible_cmd ora_grading.yml -e 'ORA_GRADING_MFE_VERSION=$2'" repos_to_cmd["enterprise_catalog"]="$edx_ansible_cmd enterprise_catalog.yml -e 'ENTERPRISE_CATALOG_MFE_VERSION=$2'" diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index 09bafc6e3ff..af43fdcc579 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -90,7 +90,8 @@ with_nested: - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', - 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', 'commerce-coordinator', 'edx-exams', 'subscriptions'] + 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', + 'course-authoring','library-authoring', 'commerce-coordinator', 'edx-exams', 'subscriptions'] - name: Add new instance to host group local_action: diff --git a/playbooks/roles/library_authoring/defaults/main.yml b/playbooks/roles/library_authoring/defaults/main.yml new file mode 100644 index 00000000000..e07157b6d70 --- /dev/null +++ b/playbooks/roles/library_authoring/defaults/main.yml @@ -0,0 +1,2 @@ +library_authoring_env_extra: + STUDIO_BASE_URL: 'https://studio-{{COMMON_DEPLOY_HOSTNAME}}' \ No newline at end of file diff --git a/playbooks/roles/library_authoring/meta/main.yml b/playbooks/roles/library_authoring/meta/main.yml new file mode 100644 index 00000000000..3b786d6eea7 --- /dev/null +++ b/playbooks/roles/library_authoring/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - common + - nginx \ No newline at end of file diff --git a/playbooks/roles/library_authoring/tasks/main.yml b/playbooks/roles/library_authoring/tasks/main.yml new file mode 100644 index 00000000000..45524e163cc --- /dev/null +++ b/playbooks/roles/library_authoring/tasks/main.yml @@ -0,0 +1,5 @@ +- name: Build Library Authoring MFE + include_role: + name: mfe + vars: + MFE_ENVIRONMENT_EXTRA: '{{ library_authoring_env_extra | default(MFE_DEPLOY_ENVIRONMENT_EXTRA) }}' \ No newline at end of file diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 19e11b4334d..0de9033166d 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -292,6 +292,22 @@ if [[ -z $ora_grading_version ]]; then ORA_GRADING_MFE_VERSION="master" fi +if [[ -z $course_authoring ]]; then + course_authoring="false" +fi + +if [[ -z $course_authoring_version ]]; then + COURSE_AUTHORING_MFE_VERSION="master" +fi + +if [[ -z $library_authoring ]]; then + library_authoring="false" +fi + +if [[ -z $library_authoring_version ]]; then + LIBRARY_AUTHORING_MFE_VERSION="master" +fi + # Lowercase the dns name to deal with an ansible bug dns_name="${dns_name,,}" @@ -440,6 +456,18 @@ ORA_GRADING_MFE_VERSION: $ora_grading_version ORA_GRADING_MFE_ENABLED: $ora_grading ORA_GRADING_SANDBOX_BUILD: True +COURSE_AUTHORING_NGINX_PORT: 80 +COURSE_AUTHORING_SSL_NGINX_PORT: 443 +COURSE_AUTHORING_MFE_VERSION: $course_authoring_version +COURSE_AUTHORING_MFE_ENABLED: $course_authoring +COURSE_AUTHORING_SANDBOX_BUILD: True + +LIBRARY_AUTHORING_NGINX_PORT: 80 +LIBRARY_AUTHORING_SSL_NGINX_PORT: 443 +LIBRARY_AUTHORING_MFE_VERSION: $library_authoring_version +LIBRARY_AUTHORING_MFE_ENABLED: $library_authoring_grading +LIBRARY_AUTHORING_SANDBOX_BUILD: True + mysql_server_version_5_7: True edxapp_container_enabled: $edxapp_container_enabled From 5390c0a1542ebcf4e6763620a4b8dbcfc18d4754 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Thu, 21 Sep 2023 19:21:47 +0500 Subject: [PATCH 540/664] feat: upgrade redis to 7.2 --- playbooks/roles/redis/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/redis/defaults/main.yml b/playbooks/roles/redis/defaults/main.yml index 337a31174f3..595d002f6c9 100644 --- a/playbooks/roles/redis/defaults/main.yml +++ b/playbooks/roles/redis/defaults/main.yml @@ -9,7 +9,7 @@ # ## # Defaults for role redis -# +# REDIS_PASSWORD: !!null REDIS_BIND_IP: 127.0.0.1 REDIS_PERSISTENCE_DIR: "/var/lib/redis" @@ -28,7 +28,7 @@ redis_group: redis # REDIS_REPO: "deb https://packages.redis.io/deb {{ ansible_distribution_release }} main" -REDIS_VERSION: "6:6.2.6-3rl1~focal1" +REDIS_VERSION: "6:7.2.0-1rl1~focal1" redis_debian_pkgs: - "redis-tools={{ REDIS_VERSION }}" From c62fcd177d2fb9fb5c672c8e8bf5e68959365e5c Mon Sep 17 00:00:00 2001 From: connorhaugh <49422820+connorhaugh@users.noreply.github.com> Date: Thu, 21 Sep 2023 16:01:38 -0400 Subject: [PATCH 541/664] fix: typo with library authoring config (#7021) --- util/jenkins/ansible-provision.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 0de9033166d..c5426e4c238 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -465,7 +465,7 @@ COURSE_AUTHORING_SANDBOX_BUILD: True LIBRARY_AUTHORING_NGINX_PORT: 80 LIBRARY_AUTHORING_SSL_NGINX_PORT: 443 LIBRARY_AUTHORING_MFE_VERSION: $library_authoring_version -LIBRARY_AUTHORING_MFE_ENABLED: $library_authoring_grading +LIBRARY_AUTHORING_MFE_ENABLED: $library_authoring LIBRARY_AUTHORING_SANDBOX_BUILD: True mysql_server_version_5_7: True From ac6e4778813c3550b1a4d4c544eae07ccb3a4c5e Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Fri, 22 Sep 2023 08:59:52 -0600 Subject: [PATCH 542/664] chore: update source for /bin/sh --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index aaa8fcbe3a7..28717433f58 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,7 +37,7 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms - source /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' + . /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' async: 1800 poll: 10 become_user: ubuntu From 82b6d4a66826ecfc7315a120b63fd5c22cf0e2a0 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Fri, 22 Sep 2023 11:24:23 -0600 Subject: [PATCH 543/664] chore: typo fix for npm ci command --- playbooks/roles/aws_devstack/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index 28717433f58..da162f7fe36 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -37,7 +37,7 @@ cd edx-repos/devstack/ make dev.pull.large-and-slow make dev.up.lms - . /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm-ci && paver update_assets && python manage.py lms migrate' + . /home/ubuntu/.profile && docker compose exec lms env TERM=xterm-256color bash -c 'make requirements && npm ci && paver update_assets && python manage.py lms migrate' async: 1800 poll: 10 become_user: ubuntu From af802c6edb68cffd8c7fb39f3ffbec0aed686abf Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Fri, 22 Sep 2023 12:10:46 -0600 Subject: [PATCH 544/664] chore: revert to include aws role --- playbooks/devstack_ami.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/devstack_ami.yml b/playbooks/devstack_ami.yml index 7a5807f179a..8f8631f8a67 100644 --- a/playbooks/devstack_ami.yml +++ b/playbooks/devstack_ami.yml @@ -6,6 +6,6 @@ ENABLE_DATADOG: False ENABLE_NEWRELIC: False roles: -# - role: aws -# when: COMMON_ENABLE_AWS_ROLE + - role: aws + when: COMMON_ENABLE_AWS_ROLE - role: aws_devstack From 9121b5ea4908da54903b72e6fa5f424cebeee132 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Fri, 22 Sep 2023 12:12:05 -0600 Subject: [PATCH 545/664] chore: revert aws devstack meta to common role --- playbooks/roles/aws_devstack/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/aws_devstack/meta/main.yml b/playbooks/roles/aws_devstack/meta/main.yml index 046f8425d4f..fdda41bb3a1 100644 --- a/playbooks/roles/aws_devstack/meta/main.yml +++ b/playbooks/roles/aws_devstack/meta/main.yml @@ -1,3 +1,3 @@ --- dependencies: - - role: user + - role: common From 99266dab309750667578afdaebbfe6f569a2cf03 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Mon, 25 Sep 2023 07:00:45 -0600 Subject: [PATCH 546/664] chore: cleanup individual command comments --- playbooks/roles/aws_devstack/tasks/main.yml | 61 --------------------- 1 file changed, 61 deletions(-) diff --git a/playbooks/roles/aws_devstack/tasks/main.yml b/playbooks/roles/aws_devstack/tasks/main.yml index da162f7fe36..91263ea4c45 100644 --- a/playbooks/roles/aws_devstack/tasks/main.yml +++ b/playbooks/roles/aws_devstack/tasks/main.yml @@ -42,67 +42,6 @@ poll: 10 become_user: ubuntu -#- name: Run script to sync all repos -# shell: -# cmd: | -# find /home/ubuntu/edx-repos -maxdepth 1 -type d -exec sh -c '(cd {} && git pull)' ';' -# async: 600 -# poll: 10 -# become_user: ubuntu -# -#- name: Mark all directories as safe for git on devstack -# ansible.builtin.shell: "git config --global --add safe.directory '*'" -# become_user: ubuntu - -#- name: Uninstall any docker packages -# shell: -# cmd: | -# pip uninstall -y docker docker-py docker-compose -# async: 600 -# poll: 10 -# become_user: ubuntu - -#- name: Reinstall docker compose -# shell: -# cmd: | -# apt-get install -y docker-compose -# async: 600 -# poll: 10 - -#- name: Run make on dev.pull.large-and-slow -# community.general.make: -# chdir: /home/ubuntu/edx-repos/devstack -# target: dev.pull.large-and-slow -# become_user: ubuntu - -#- name: Run make on lms-shell -# community.general.make: -# chdir: /home/ubuntu/edx-repos/devstack -# target: lms-shell -# become_user: ubuntu - -#- name: Make requirements -# community.general.make: -# chdir: /home/ubuntu/edx-repos/devstack -# target: requirements -# become_user: ubuntu - -#- name: Run npm ci -# community.general.npm: -# ci: true -# path: /home/ubuntu/edx-repos/devstack -# become_user: ubuntu - -#- name: Update paver assets -# ansible.builtin.shell: -# cmd: paver update_assets -# chdir: /home/ubuntu/edx-repos/devstack -# become_user: ubuntu - -#- name : LMS migrate -# command: /home/ubuntu/edx-repos/devstack/manage.py lms migrate -# become_user: ubuntu - - name: Delete read-only ssh key ansible.builtin.file: state: absent From 495bcb184551c40b5dedb6495293689e5108a5ca Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Mon, 25 Sep 2023 17:19:43 -0400 Subject: [PATCH 547/664] fix: apparent issue with ansible parsing --- playbooks/roles/common/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/common/tasks/main.yml b/playbooks/roles/common/tasks/main.yml index b406c55a021..cc80e4f841f 100644 --- a/playbooks/roles/common/tasks/main.yml +++ b/playbooks/roles/common/tasks/main.yml @@ -209,7 +209,7 @@ src: "{{ item.src }}" owner: root group: root - mode: "{{ item.mode | default(0644) }}" + mode: "{{ item.mode | default('0644') }}" register: config_templates with_items: - { src: 'edx_rsyslog.j2', dest: '/etc/rsyslog.d/99-edx.conf' } From 3172e4decfca28754f48c75f06b09f3296fafe8f Mon Sep 17 00:00:00 2001 From: Syed Imran Hassan <45480841+syedimranhassan@users.noreply.github.com> Date: Tue, 26 Sep 2023 16:33:38 +0500 Subject: [PATCH 548/664] fix: apparent issue with ansible parsing --- playbooks/roles/edxapp/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/tasks/main.yml b/playbooks/roles/edxapp/tasks/main.yml index 7f0ca23b418..219a0288a89 100644 --- a/playbooks/roles/edxapp/tasks/main.yml +++ b/playbooks/roles/edxapp/tasks/main.yml @@ -18,7 +18,7 @@ state: directory owner: "{{ edxapp_user }}" group: "{{ common_web_group }}" - mode: "{{ item.mode | default(0755) }}" + mode: "{{ item.mode | default('0755') }}" with_items: - { path: "{{ edxapp_app_dir }}" } # needed for the ansible 1.5 git module From 9fc8af2314ade8e2f886536bb45fcac6c2d3211b Mon Sep 17 00:00:00 2001 From: Bianca Severino Date: Thu, 28 Sep 2023 09:46:54 -0400 Subject: [PATCH 549/664] fix: remove GATSBY_TURN_ON_SIDEBAR_FILTERS --- CHANGELOG.md | 4 ++++ playbooks/roles/prospectus/templates/.env.environment.j2 | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abdfea8d0c9..3504014ac1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2023-09-28 + - Role: prospectus + - Remove `GATSBY_TURN_ON_SIDEBAR_FILTERS` flag + - 2023-08-29 - Role: prospectus - Added `GATSBY_TURN_ON_SIDEBAR_FILTERS` flag to enable new sidebar changes diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 9d637957d5b..72c005f9375 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -15,4 +15,3 @@ GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} -GATSBY_TURN_ON_SIDEBAR_FILTERS={{ PROSPECTUS_GATSBY_TURN_ON_SIDEBAR_FILTERS }} From a53e63bbef7c7e910e5cb19b93f4a0dca617ed13 Mon Sep 17 00:00:00 2001 From: UsamaSadiq Date: Thu, 28 Sep 2023 21:03:24 +0500 Subject: [PATCH 550/664] build: replace django-admin.py with django-admin --- playbooks/roles/xqueue/tasks/main.yml | 4 ++-- playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/xqueue/tasks/main.yml b/playbooks/roles/xqueue/tasks/main.yml index 57f2fbe2915..5804e01522d 100644 --- a/playbooks/roles/xqueue/tasks/main.yml +++ b/playbooks/roles/xqueue/tasks/main.yml @@ -178,7 +178,7 @@ # If there is a common user for migrations run migrations using his username # and credentials. If not we use the xqueue mysql user - name: Migrate - shell: "{{ xqueue_venv_bin }}/django-admin.py migrate --noinput --settings=xqueue.{{ XQUEUE_SETTINGS }} --pythonpath={{ xqueue_code_dir }}" + shell: "{{ xqueue_venv_bin }}/django-admin migrate --noinput --settings=xqueue.{{ XQUEUE_SETTINGS }} --pythonpath={{ xqueue_code_dir }}" become_user: "{{ xqueue_user }}" environment: DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}" @@ -191,7 +191,7 @@ - migrate:db - name: Create users - shell: "{{ xqueue_venv_bin }}/django-admin.py update_users --settings=xqueue.{{ XQUEUE_SETTINGS }} --pythonpath={{ xqueue_code_dir }}" + shell: "{{ xqueue_venv_bin }}/django-admin update_users --settings=xqueue.{{ XQUEUE_SETTINGS }} --pythonpath={{ xqueue_code_dir }}" become_user: "{{ xqueue_user }}" environment: XQUEUE_CFG: '{{ COMMON_CFG_DIR }}/xqueue.yml' diff --git a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 index 287502fa0de..fe6a0c58496 100644 --- a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 @@ -1,9 +1,9 @@ [program:xqueue_consumer] {% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = xqueue_venv_bin + '/newrelic-admin run-program ' + xqueue_venv_bin + '/django-admin.py run_consumer' %} +{% set executable = xqueue_venv_bin + '/newrelic-admin run-program ' + xqueue_venv_bin + '/django-admin run_consumer' %} {% else %} -{% set executable = xqueue_venv_bin + '/django-admin.py run_consumer' %} +{% set executable = xqueue_venv_bin + '/django-admin run_consumer' %} {% endif %} command={{ executable }} --pythonpath={{ xqueue_code_dir }} --settings=xqueue.{{ XQUEUE_SETTINGS }} From 29b964415cc72fb19f7fcd350c9ede5a04aa9f6c Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 28 Sep 2023 12:36:50 -0400 Subject: [PATCH 551/664] build: Remove the Auto-merge automation. This automation is not documented and has not been used in over seven months. Remove references to it before we remove it from the organization. See https://github.com/openedx/axim-engineering/issues/898 for more details. --- .../workflows/pr-automerge-open-release.yml | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 .github/workflows/pr-automerge-open-release.yml diff --git a/.github/workflows/pr-automerge-open-release.yml b/.github/workflows/pr-automerge-open-release.yml deleted file mode 100644 index 25af91e0527..00000000000 --- a/.github/workflows/pr-automerge-open-release.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Enable automerging for named release branches. -# See the reusable workflow for details: -# https://github.com/openedx/.github/.github/workflows/pr-automerge-open-release.yml - -name: Automerge BTR open-release PRs - -on: - issue_comment: - branches: - - open-release/* - types: - - created - - edited - pull_request_target: - branches: - - open-release/* - types: - - opened - - edited - - ready_for_review - -jobs: - automerge: - uses: openedx/.github/.github/workflows/pr-automerge-open-release.yml@master From 2fdde44dfcd807fee402a9fbbba7a72731012b4e Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 3 Oct 2023 15:44:09 +0500 Subject: [PATCH 552/664] chore: Added script for retrieving ACL of S3 objects --- util/s3_obj_acl.py | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 util/s3_obj_acl.py diff --git a/util/s3_obj_acl.py b/util/s3_obj_acl.py new file mode 100644 index 00000000000..9b08111b882 --- /dev/null +++ b/util/s3_obj_acl.py @@ -0,0 +1,101 @@ +import csv +import sys +import urllib.parse +import logging +from botocore.exceptions import ClientError +import backoff +import click +import boto3 +import concurrent.futures + + +MAX_TRIES = 5 +inconsistent_acl_objects = [] +consistent_acl_objects = [] + + +# logging config + +# Set logging configuration +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# create file handler that logs messages +filehandler = logging.FileHandler('result.txt') +filehandler.setLevel(logging.INFO) +formatter = logging.Formatter( + '%(message)s') +filehandler.setFormatter(formatter) +# add the handlers to logger +logger.addHandler(filehandler) + + +class S3BotoWrapper: + def __init__(self): + self.client = boto3.client('s3') + + @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES) + def get_object_acl(self, bucket_name, obj_key): + return self.client.get_object_acl(Bucket=bucket_name, Key=obj_key) + + +def get_object_acl(bucket_name, object_name): + client = S3BotoWrapper() + try: + # Try encoding + object_name_decoded = urllib.parse.unquote(object_name) + # Get the ACL for the object + response = client.get_object_acl(bucket_name, object_name_decoded) + acl = response['Grants'] + return acl + except Exception as e: + print(f"Error retrieving ACL for {object_name_decoded}: {str(e)}") + return None + + +def check_acl_uniformity(object_acls): + # Extract the first object's ACL as the baseline + baseline_acl = list(object_acls.values())[0] + # for acl in object_acls.values(): + for key, acl in object_acls.items(): + if acl != baseline_acl: + acl_dict = {key: acl} + inconsistent_acl_objects.append(acl_dict) + # return False + else: + acl_cons_dict = {key: acl} + consistent_acl_objects.append(acl_cons_dict) + + return True + + +def read_csv_file(file_path): + object_acls = {} + with open(file_path, 'r') as file: + reader = csv.reader(file) + objects = list(reader) + with concurrent.futures.ProcessPoolExecutor() as executor: + futures = {executor.submit(get_object_acl, obj[0], obj[1]): obj for obj in objects} + + for future in concurrent.futures.as_completed(futures): + obj = futures[future] + acl = future.result() + object_acls[(obj[0], obj[1])] = acl + + return object_acls + + +@click.command() +@click.option('--file_name', required=True, help='Use to identify the file name') +def controller(file_name): + obj_dict = read_csv_file(file_name) + is_acl_uniform = check_acl_uniformity(obj_dict) + logger.info("Objects with same acl") + for obj in consistent_acl_objects: + logger.info(obj) + logger.info("\n\nObjects with different acl") + for in_obj in inconsistent_acl_objects: + logger.info(in_obj) + + +if __name__ == '__main__': + controller() From 06a14203c75a5c7cfaf9543595b9cacb27ee0fd8 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 3 Oct 2023 15:06:32 -0400 Subject: [PATCH 553/664] feat: update mongo agent version JIRA:PSRE-1597 --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index f79d2ac56e0..a2fc6e3bc34 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "13.3.0.8373-1" + version: "13.5.0.8451-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" From 44ef003daedfcaf9ed01f9a1026948cd0f77ccb4 Mon Sep 17 00:00:00 2001 From: Chris Deery <3932645+cdeery@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:50:47 -0400 Subject: [PATCH 554/664] feat: add profile and learner-dashboard to sandbox Add profile and learner-dashboard to enable the FrontendPlugin POC --- playbooks/edx_continuous_integration.yml | 6 ++++ playbooks/learner_dashboard.yml | 16 +++++++++++ playbooks/profile.yml | 16 +++++++++++ .../roles/edx_ansible/templates/update.j2 | 2 ++ playbooks/roles/launch_ec2/tasks/main.yml | 2 +- util/jenkins/ansible-provision.sh | 28 +++++++++++++++++++ 6 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 playbooks/learner_dashboard.yml create mode 100644 playbooks/profile.yml diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 012f423e69d..8f7737dfa4e 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -74,6 +74,12 @@ - role: mfe MFE_NAME: ora-grading when: ORA_GRADING_MFE_ENABLED + - role: mfe + MFE_NAME: profile + when: PROFILE_MFE_ENABLED + - role: mfe + MFE_NAME: learner-dashboard + when: LEARNER_DASHBOARD_MFE_ENABLED - { role: oauth_client_setup, when: edxapp_containerized is defined and not edxapp_containerized } - role: datadog when: COMMON_ENABLE_DATADOG diff --git a/playbooks/learner_dashboard.yml b/playbooks/learner_dashboard.yml new file mode 100644 index 00000000000..231b6e7f54c --- /dev/null +++ b/playbooks/learner_dashboard.yml @@ -0,0 +1,16 @@ +- name: Deploy Learner Dashboard Frontend + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'learner-dashboard' + LEARNER_DASHBOARD_MFE_ENABLED: True + LEARNER_DASHBOARD_MFE_SANDBOX_BUILD: False + roles: + - role: mfe + MFE_NAME: learner-dashboard + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE diff --git a/playbooks/profile.yml b/playbooks/profile.yml new file mode 100644 index 00000000000..08dc8824b5b --- /dev/null +++ b/playbooks/profile.yml @@ -0,0 +1,16 @@ +- name: Deploy profile Frontend + hosts: all + become: True + gather_facts: True + vars: + ENABLE_NEWRELIC: False + CLUSTER_NAME: 'profile' + PROFILE_MFE_ENABLED: True + PROFILE_MFE_SANDBOX_BUILD: False + roles: + - role: mfe + MFE_NAME: profile + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE diff --git a/playbooks/roles/edx_ansible/templates/update.j2 b/playbooks/roles/edx_ansible/templates/update.j2 index e838b537f05..dbca95613bd 100644 --- a/playbooks/roles/edx_ansible/templates/update.j2 +++ b/playbooks/roles/edx_ansible/templates/update.j2 @@ -85,6 +85,8 @@ repos_to_cmd["course_authoring"]="$edx_ansible_cmd course_authoring.yml -e 'COUR repos_to_cmd["library_authoring"]="$edx_ansible_cmd library_authoring.yml -e 'COURSE_AUTHORING_MFE_VERSION=$2'" repos_to_cmd["ora_grading"]="$edx_ansible_cmd ora_grading.yml -e 'ORA_GRADING_MFE_VERSION=$2'" repos_to_cmd["enterprise_catalog"]="$edx_ansible_cmd enterprise_catalog.yml -e 'ENTERPRISE_CATALOG_MFE_VERSION=$2'" +repos_to_cmd["profile"]="$edx_ansible_cmd profile.yml -e 'PROFILE_MFE_VERSION=$2'" +repos_to_cmd["learner_dashboard"]="$edx_ansible_cmd learner_dashboard.yml -e 'LEARNER_DASHBOARD_MFE_VERSION=$2'" if [[ -z $1 || -z $2 ]]; then echo diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index af43fdcc579..df95a730ab6 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -91,7 +91,7 @@ - "{{ ec2.instances }}" - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api', 'registrar', 'program-console', 'learner-portal', 'prospectus', 'authn', 'payment', 'license-manager', 'learning', 'enterprise-catalog', 'ora-grading', - 'course-authoring','library-authoring', 'commerce-coordinator', 'edx-exams', 'subscriptions'] + 'course-authoring','library-authoring', 'commerce-coordinator', 'edx-exams', 'subscriptions', 'profile', 'learner-dashboard'] - name: Add new instance to host group local_action: diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index c5426e4c238..3cb05453d9d 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -308,6 +308,22 @@ if [[ -z $library_authoring_version ]]; then LIBRARY_AUTHORING_MFE_VERSION="master" fi +if [[ -z $profile ]]; then + profile="false" +fi + +if [[ -z $profile_version ]]; then + PROFILE_MFE_VERSION="master" +fi + +if [[ -z $learner_dashboard ]]; then + learner_dashboard="false" +fi + +if [[ -z $learner_dashboard_version ]]; then + LEARNER_DASHBOARD_MFE_VERSION="master" +fi + # Lowercase the dns name to deal with an ansible bug dns_name="${dns_name,,}" @@ -468,6 +484,18 @@ LIBRARY_AUTHORING_MFE_VERSION: $library_authoring_version LIBRARY_AUTHORING_MFE_ENABLED: $library_authoring LIBRARY_AUTHORING_SANDBOX_BUILD: True +PROFILE_NGINX_PORT: 80 +PROFILE_SSL_NGINX_PORT: 443 +PROFILE_MFE_VERSION: $profile_version +PROFILE_MFE_ENABLED: $profile +PROFILE_SANDBOX_BUILD: True + +LEARNER_DASHBOARD_NGINX_PORT: 80 +LEARNER_DASHBOARD_SSL_NGINX_PORT: 443 +LEARNER_DASHBOARD_MFE_VERSION: $learner_dashboard_version +LEARNER_DASHBOARD_MFE_ENABLED: $learner_dashboard +LEARNER_DASHBOARD_SANDBOX_BUILD: True + mysql_server_version_5_7: True edxapp_container_enabled: $edxapp_container_enabled From 7d52984ab5f8c2020968dc6fe3b68e4692364320 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 5 Oct 2023 11:41:39 +0500 Subject: [PATCH 555/664] chore: use variable to install decrypt config private key, and NGINX SSL cert and key --- playbooks/roles/edx_service/tasks/main.yml | 15 ++++++++++ .../edxapp/tasks/service_variant_config.yml | 16 ++++++++++ playbooks/roles/nginx/tasks/main.yml | 29 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/playbooks/roles/edx_service/tasks/main.yml b/playbooks/roles/edx_service/tasks/main.yml index 54a9d3ff7b4..ca733b30d79 100644 --- a/playbooks/roles/edx_service/tasks/main.yml +++ b/playbooks/roles/edx_service/tasks/main.yml @@ -105,6 +105,21 @@ - install:configuration - install:app-configuration +- name: Install decrypt config private key from variable + local_action: + module: copy + content: "{{ DECRYPT_CONFIG_PRIVATE_KEY_VAR }}" + dest: "{{ DECRYPT_CONFIG_PRIVATE_KEY_PATH | default('/var/tmp') }}/private.key" + force: yes + mode: "0644" + become: false + no_log: True + when: edx_service_decrypt_config_enabled and DECRYPT_CONFIG_PRIVATE_KEY_VAR is defined + tags: + - install + - install:configuration + - install:app-configuration + - name: Decrypt app config file local_action: command asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path {{ ENCRYPTED_CFG_DIR }}/{{ edx_service_name }}.yml --private_key_path {{ DECRYPT_CONFIG_PRIVATE_KEY }} --outfile_path {{ UNENCRYPTED_CFG_DIR }}/{{ edx_service_name }}.yml become: false diff --git a/playbooks/roles/edxapp/tasks/service_variant_config.yml b/playbooks/roles/edxapp/tasks/service_variant_config.yml index 5ed089a8647..c9b169d32b2 100644 --- a/playbooks/roles/edxapp/tasks/service_variant_config.yml +++ b/playbooks/roles/edxapp/tasks/service_variant_config.yml @@ -77,6 +77,22 @@ - edxapp_cfg # Old deprecated tag, will remove when possible - edxapp_cfg_yaml_only # Used to render the yaml without the json until we remove the json configs +- name: Install decrypt config private key from variable + local_action: + module: copy + content: "{{ DECRYPT_CONFIG_PRIVATE_KEY_VAR }}" + dest: "{{ DECRYPT_CONFIG_PRIVATE_KEY_PATH | default('/var/tmp') }}/private.key" + force: yes + mode: "0644" + become: false + no_log: True + when: EDXAPP_DECRYPT_CONFIG_ENABLED and DECRYPT_CONFIG_PRIVATE_KEY_VAR is defined + tags: + - install + - install:configuration + - install:app-configuration + - edxapp_cfg # Old deprecated tag, will remove when possible + - name: Decrypt lms config local_action: command asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path {{ ENCRYPTED_CFG_DIR }}/lms.yml --private_key_path {{ DECRYPT_CONFIG_PRIVATE_KEY }} --outfile_path {{ UNENCRYPTED_CFG_DIR }}/lms.yml become: false diff --git a/playbooks/roles/nginx/tasks/main.yml b/playbooks/roles/nginx/tasks/main.yml index 28d93ad2898..21cb9b4f292 100644 --- a/playbooks/roles/nginx/tasks/main.yml +++ b/playbooks/roles/nginx/tasks/main.yml @@ -361,6 +361,35 @@ - install - install:configuration +- name: copy ssl cert from variable + copy: + content: "{{ NGINX_SSL_CERT_VAR }}" + dest: "/etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }}" + owner: root + group: root + mode: 0644 + when: + - not ssl_cert.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_CERTIFICATE != 'ssl-cert-snakeoil.pem' + - NGINX_SSL_CERT_VAR is defined + tags: + - install + - install:configuration + +- name: copy ssl key from variable + copy: + content: "{{ NGINX_SSL_KEY_VAR }}" + dest: "/etc/ssl/private/{{ NGINX_SSL_KEY|basename }}" + owner: root + group: root + mode: 0640 + when: + - not ssl_key.stat.exists and NGINX_ENABLE_SSL and NGINX_SSL_KEY != 'ssl-cert-snakeoil.key' + - NGINX_SSL_KEY_VAR is defined + no_log: True + tags: + - install + - install:configuration + # removing default link - name: Removing default nginx config and restart (enabled) file: From 8aa0c06d887863c4843bbaeac7fa6426a6b17a76 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 5 Oct 2023 15:41:53 +0500 Subject: [PATCH 556/664] chore: remove config secur refrences from sandbox build script --- util/jenkins/ansible-provision.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 3cb05453d9d..6c7167d43b6 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -124,7 +124,6 @@ if [[ ( -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ) && (! -f $BOTO_CONF fi extra_vars_file="/var/tmp/extra-vars-$$.yml" -sandbox_secure_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml" sandbox_internal_vars_file="${WORKSPACE}/configuration-internal/ansible/vars/developer-sandbox.yml" extra_var_arg="-e@${extra_vars_file}" program_console="false" @@ -132,7 +131,7 @@ program_console="false" if [[ $edx_internal == "true" ]]; then # if this is a an edx server include # the secret var file - extra_var_arg="-e@${sandbox_internal_vars_file} -e@${sandbox_secure_vars_file} -e@${extra_vars_file} -e DECRYPT_CONFIG_PRIVATE_KEY=$WORKSPACE/configuration-secure/ansible/keys/sandbox-remote-config/sandbox/private.key -e ENCRYPTED_CFG_DIR=$WORKSPACE/configuration-internal/sandbox-remote-config/sandbox -e UNENCRYPTED_CFG_DIR=$WORKSPACE" + extra_var_arg="-e@${sandbox_internal_vars_file} -e@${extra_vars_file} -e DECRYPT_CONFIG_PRIVATE_KEY_PATH=$WORKSPACE -e DECRYPT_CONFIG_PRIVATE_KEY=$WORKSPACE/private.key -e ENCRYPTED_CFG_DIR=$WORKSPACE/configuration-internal/sandbox-remote-config/sandbox -e UNENCRYPTED_CFG_DIR=$WORKSPACE" fi if [[ -z $region ]]; then @@ -738,9 +737,9 @@ EOF rm -f "${provision_fluentd_script}" # decrypt lms config file - asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/lms.yml --private_key_path $WORKSPACE/configuration-secure/ansible/keys/sandbox-remote-config/sandbox/private.key --outfile_path $WORKSPACE/lms.yml + asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/lms.yml --private_key_path $WORKSPACE/private.key --outfile_path $WORKSPACE/lms.yml # decrypt cms config file - asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/studio.yml --private_key_path $WORKSPACE/configuration-secure/ansible/keys/sandbox-remote-config/sandbox/private.key --outfile_path $WORKSPACE/cms.yml + asym_crypto_yaml decrypt-encrypted-yaml --secrets_file_path $WORKSPACE/configuration-internal/sandbox-remote-config/sandbox/studio.yml --private_key_path $WORKSPACE/private.key --outfile_path $WORKSPACE/cms.yml sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/lms.yml sed -i "s/deploy_host/${dns_name}.${dns_zone}/g" $WORKSPACE/cms.yml @@ -770,7 +769,8 @@ EOF ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=$WORKSPACE/dockerfiles-internal/edx-platform-private dest=/var/tmp/" -u ubuntu -b set +x - app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" + + app_git_ssh_key=$(aws secretsmanager get-secret-value --secret-id $configuration_secure_secret --query SecretString --output text | jq -r '._local_git_identity') # specify variable names app_hostname="courses" @@ -909,7 +909,7 @@ fi # deploy the edx_ansible play run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu -cat $sandbox_secure_vars_file $sandbox_internal_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean +cat $sandbox_internal_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b ret=$? if [[ $ret -ne 0 ]]; then @@ -947,7 +947,8 @@ fi if [[ $edx_exams == 'true' ]]; then set +x - app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" + + app_git_ssh_key=$(aws secretsmanager get-secret-value --secret-id $configuration_secure_secret --query SecretString --output text | jq -r '._local_git_identity') app_hostname="edx-exams" app_service_name="edx_exams" @@ -970,8 +971,9 @@ fi if [[ $subscriptions == 'true' ]]; then set +x - app_git_ssh_key="$($WORKSPACE/yq '._local_git_identity' $WORKSPACE/configuration-secure/ansible/vars/developer-sandbox.yml)" - + + app_git_ssh_key=$(aws secretsmanager get-secret-value --secret-id $configuration_secure_secret --query SecretString --output text | jq -r '._local_git_identity') + app_hostname="subscriptions" app_service_name="subscriptions" app_name="subscriptions" From 0a62696d182f732c04225b5b66e21e8f947d2c29 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 9 Oct 2023 10:15:25 +0500 Subject: [PATCH 557/664] chore: setting EDXAPP_EDXAPP_SECRET_KEY default value --- CHANGELOG.md | 6 ++++++ playbooks/roles/edxapp/defaults/main.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3504014ac1e..74dbf7aa56e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2023-10-09 + + - Role: edxapp + - Setting default value for `EDXAPP_EDXAPP_SECRET_KEY` as if it is not override in config secure, + helper script template can still be generated + - 2023-09-28 - Role: prospectus - Remove `GATSBY_TURN_ON_SIDEBAR_FILTERS` flag diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 5fc3802f1b0..b0d5cfcdb4d 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -155,7 +155,7 @@ EDXAPP_COMMENTS_SERVICE_URL: 'http://localhost:18080' # EDXAPP_COMMENTS_SERVICE_KEY must match FORUM_API_KEY EDXAPP_COMMENTS_SERVICE_KEY: 'password' -# EDXAPP_EDXAPP_SECRET_KEY: "DUMMY KEY CHANGE BEFORE GOING TO PRODUCTION" # Commented out to ensure secure defaults +EDXAPP_EDXAPP_SECRET_KEY: "DUMMY KEY CHANGE BEFORE GOING TO PRODUCTION" EDXAPP_FERNET_KEYS: - "DUMMY KEY CHANGE BEFORE GOING TO PRODUCTION" From 9401308efde73c57ab798138247cab31b4f8f7a7 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Mon, 9 Oct 2023 15:53:38 +0500 Subject: [PATCH 558/664] chore: update hash of concept xblock --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index b0d5cfcdb4d..478a445a10c 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -557,7 +557,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: # Concept XBlock, in particular, is nowhere near finished and an early prototype. # Profile XBlock is there so we can play with XBlock arguments in the platform, but isn't ready for use outside of # edX. - - name: git+https://github.com/openedx/ConceptXBlock.git@a45a6560c92b6d8b62be1f939ff1d00dfff84e70#egg=concept-xblock + - name: git+https://github.com/openedx/ConceptXBlock.git@75dd86e5fa4c54ab2f04c95c4fd3389ac1f56174#egg=concept-xblock extra_args: -e - name: git+https://github.com/openedx/AudioXBlock.git@20538c6e9bb704801a71ecbb6981f794556dfc45#egg=audio-xblock extra_args: -e From 43c3b06c248a25068dbdb7f7e0901933a0c04980 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 2 Oct 2023 15:52:57 +0500 Subject: [PATCH 559/664] chore: temp removing this to test building sandbox with django42. --- playbooks/roles/edxapp/defaults/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 478a445a10c..a194999a9c5 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -583,9 +583,6 @@ EDXAPP_PRIVATE_REQUIREMENTS: # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e - # "Pathways" learning context plugin for the LabXchange project - - name: git+https://github.com/open-craft/lx-pathway-plugin.git@ba1d470217cd5908cbd8b56075628bd4eacf7b39#egg=lx-pathway-plugin - extra_args: -e # Caliper and xAPI event routing plugin - name: edx-event-routing-backends==5.5.6 From dac8f29449a394fc3ce89d3ccc4a3222e1174e2f Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Mon, 9 Oct 2023 17:54:13 +0500 Subject: [PATCH 560/664] chore: using other fork to delete the table. --- playbooks/roles/edxapp/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index a194999a9c5..6c7d75a9732 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -583,6 +583,9 @@ EDXAPP_PRIVATE_REQUIREMENTS: # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e + # "Pathways" learning context plugin for the LabXchange project + - name: git+https://github.com/irtazaakram/lx-pathway-plugin.git@8fe72de587094f81b7d28123e55ca06f7e0ac1a1#egg=lx-pathway-plugin + extra_args: -e # Caliper and xAPI event routing plugin - name: edx-event-routing-backends==5.5.6 From b6f769f65f0731ec927cfa8e9eac358d1f13a9c7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 10 Oct 2023 09:24:31 +0500 Subject: [PATCH 561/664] chore: remove aws credentials for rapid7 package download --- playbooks/roles/insightvm_agent/defaults/main.yml | 2 -- playbooks/roles/insightvm_agent/tasks/main.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/playbooks/roles/insightvm_agent/defaults/main.yml b/playbooks/roles/insightvm_agent/defaults/main.yml index 48036da7bcd..0067fc986dd 100644 --- a/playbooks/roles/insightvm_agent/defaults/main.yml +++ b/playbooks/roles/insightvm_agent/defaults/main.yml @@ -2,6 +2,4 @@ r7_installer_location: /tmp/rapid7_agent_installer.sh R7_TOKEN: "SET-ME-PLEASE" -R7_IAM_USER_ACCESS_KEY: "SET-ME-PLEASE" -R7_IAM_USER_SECRET_KEY: "SET-ME-PLEASE" R7_BUCKET: "SET-ME-PLEASE (ex. bucket-name)" diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index d4703676360..f522f135f60 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -17,8 +17,6 @@ - name: Pull Rapid7 Agent Installer from S3 aws_s3: - aws_access_key: "{{ R7_IAM_USER_ACCESS_KEY }}" - aws_secret_key: "{{ R7_IAM_USER_SECRET_KEY }}" bucket: "{{ R7_BUCKET }}" object: rapid7/rapid7_agent_installer.sh dest: "{{ r7_installer_location }}" From 123e30abf236fcf90320873a3eed964d08667813 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 10 Oct 2023 13:36:46 -0400 Subject: [PATCH 562/664] fix: skip rapid7 installation if we can't pull installer JIRA:DOS-4175 --- playbooks/roles/insightvm_agent/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index f522f135f60..c2d1244b4b7 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -23,6 +23,8 @@ mode: get overwrite: different ignore_nonexistent_bucket: true + register: pull_rapid7_agent_installer_from_s3_result + ignore_errors: true tags: - manage_rapid7_pull_installer when: not r7_service.stat.exists|bool @@ -33,7 +35,9 @@ mode: "0755" owner: root group: root + when: pull_rapid7_agent_installer_from_s3_result is not failed ignore_errors: "{{ ansible_check_mode }}" + register: ensure_file_permissions_are_set_result tags: - manage_rapid7_file_perms when: not r7_service.stat.exists|bool @@ -44,5 +48,6 @@ tags: - manage_rapid7_agent_install ignore_errors: "{{ ansible_check_mode }}" + when: ensure_file_permissions_are_set_result is not failed and pull_rapid7_agent_installer_from_s3_result is not failed args: creates: /etc/systemd/system/ir_agent.service From 5c899188c39cf10b735ef14ddc71c0a5a76a5d92 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 10 Oct 2023 14:10:37 -0400 Subject: [PATCH 563/664] fix: fix duplicate conditional for running insightvm installer ``` [WARNING]: While constructing a mapping from /godata/pipelines/build_edxapp_ami s/configuration/playbooks/roles/insightvm_agent/tasks/main.yml, line 32, column 3, found a duplicate dict key (when). Using last defined value only. ``` --- playbooks/roles/insightvm_agent/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index c2d1244b4b7..1198566ec70 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -35,12 +35,11 @@ mode: "0755" owner: root group: root - when: pull_rapid7_agent_installer_from_s3_result is not failed ignore_errors: "{{ ansible_check_mode }}" register: ensure_file_permissions_are_set_result tags: - manage_rapid7_file_perms - when: not r7_service.stat.exists|bool + when: pull_rapid7_agent_installer_from_s3_result is not failed and not r7_service.stat.exists|bool - name: Agent Install command: "/tmp/rapid7_agent_installer.sh install_start --token {{ R7_TOKEN }}" From 8401dda08c56a8bce0780878aa2354a3f6c52dca Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 10 Oct 2023 14:12:12 -0400 Subject: [PATCH 564/664] fix: don't run rapid7 installer if service is already running --- playbooks/roles/insightvm_agent/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/insightvm_agent/tasks/main.yml b/playbooks/roles/insightvm_agent/tasks/main.yml index 1198566ec70..faf170086f2 100644 --- a/playbooks/roles/insightvm_agent/tasks/main.yml +++ b/playbooks/roles/insightvm_agent/tasks/main.yml @@ -47,6 +47,6 @@ tags: - manage_rapid7_agent_install ignore_errors: "{{ ansible_check_mode }}" - when: ensure_file_permissions_are_set_result is not failed and pull_rapid7_agent_installer_from_s3_result is not failed + when: ensure_file_permissions_are_set_result is not failed and pull_rapid7_agent_installer_from_s3_result is not failed and not r7_service.stat.exists|bool args: creates: /etc/systemd/system/ir_agent.service From 88e91f05aa780ab351190b6a17758f7db2d044a0 Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Thu, 10 Aug 2023 19:50:47 +0500 Subject: [PATCH 565/664] fix: removed devpi related references from build and playbooks --- docker/build/devpi/Dockerfile | 45 ------------------ docker/build/devpi/docker-entrypoint.sh | 36 -------------- docker/build/notes/Dockerfile | 18 ++++++- docker/plays/devpi.yml | 11 ----- playbooks/roles/devpi/defaults/main.yml | 33 ------------- playbooks/roles/devpi/tasks/main.yml | 63 ------------------------- util/parsefiles_config.yml | 1 - 7 files changed, 17 insertions(+), 190 deletions(-) delete mode 100644 docker/build/devpi/Dockerfile delete mode 100644 docker/build/devpi/docker-entrypoint.sh delete mode 100644 docker/plays/devpi.yml delete mode 100644 playbooks/roles/devpi/defaults/main.yml delete mode 100644 playbooks/roles/devpi/tasks/main.yml diff --git a/docker/build/devpi/Dockerfile b/docker/build/devpi/Dockerfile deleted file mode 100644 index fd26af44d9e..00000000000 --- a/docker/build/devpi/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -# To build this Dockerfile: -# -# From the root of configuration: -# -# docker build -f docker/build/devpi/Dockerfile . -# -# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible -# with the currently checked-out configuration repo. - -ARG BASE_IMAGE_TAG=latest -FROM edxops/focal-common:${BASE_IMAGE_TAG} -LABEL maintainer="edxops" - -ARG ARG_DEVPI_SERVER_VERSION=6.3.1 -ARG ARG_DEVPI_WEB_VERSION=4.0.8 -ARG ARG_DEVPI_CLIENT_VERSION=5.2.3 - -ADD . /edx/app/edx_ansible/edx_ansible - -WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays - -RUN apt-get update - -COPY docker/devstack_common_ansible_overrides.yml /devstack/ansible_overrides.yml - -RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook devpi.yml \ - -c local -i '127.0.0.1,' \ - -t "install,devstack" \ - --extra-vars="@/devstack/ansible_overrides.yml" \ - --extra-vars="DEVPI_SERVER_VERSION=$ARG_DEVPI_SERVER_VERSION" \ - --extra-vars="DEVPI_WEB_VERSION=$ARG_DEVPI_WEB_VERSION" \ - --extra-vars="DEVPI_CLIENT_VERSION=$ARG_DEVPI_CLIENT_VERSION" - -EXPOSE 3141 -VOLUME /data - -COPY docker/build/devpi/docker-entrypoint.sh /docker-entrypoint.sh -RUN chmod +x /docker-entrypoint.sh - -USER root -ENV HOME /data -WORKDIR /data - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["devpi"] diff --git a/docker/build/devpi/docker-entrypoint.sh b/docker/build/devpi/docker-entrypoint.sh deleted file mode 100644 index 83a360c4223..00000000000 --- a/docker/build/devpi/docker-entrypoint.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -function defaults { - : ${DEVPISERVER_SERVERDIR="/data/server"} - : ${DEVPI_CLIENTDIR="/data/client"} - - echo "DEVPISERVER_SERVERDIR is ${DEVPISERVER_SERVERDIR}" - echo "DEVPI_CLIENTDIR is ${DEVPI_CLIENTDIR}" - - export DEVPISERVER_SERVERDIR DEVPI_CLIENTDIR -} - -function initialize_devpi { - echo "[RUN]: Initializing devpi-server..." - DEVPI_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` - devpi-init --root-passwd ${DEVPI_PASSWORD} - echo "[RUN]: devpi-server password set to '${DEVPI_PASSWORD}'" > $DEVPISERVER_SERVERDIR/.serverpassword -} - -defaults - -if [ "$1" = 'devpi' ]; then - source /home/devpi/venvs/devpi_venv/bin/activate - - if [ ! -f $DEVPISERVER_SERVERDIR/.serverversion ]; then - initialize_devpi - fi - - echo "[RUN]: Launching devpi-server..." - exec devpi-server --restrict-modify root --host 0.0.0.0 --port 3141 -fi - -echo "[RUN]: Builtin command not provided [devpi]" -echo "[RUN]: $@" - -exec "$@" diff --git a/docker/build/notes/Dockerfile b/docker/build/notes/Dockerfile index 66619362211..a0fc6742155 100644 --- a/docker/build/notes/Dockerfile +++ b/docker/build/notes/Dockerfile @@ -16,6 +16,9 @@ ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} ENV NOTES_VERSION=${OPENEDX_RELEASE} ENV REPO_OWNER=edx + +ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" + ADD . /edx/app/edx_ansible/edx_ansible WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays @@ -24,7 +27,20 @@ COPY docker/build/notes/ansible_overrides.yml / COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml -RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev +# Ansible Free work start +RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip +RUN apt-get install -y sudo + + + +RUN pip install -r /edx/edx_notes_api/edx_notes_api/requirements/base.txt + + +# Ansible Free work end + + + + RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \ -c local -i '127.0.0.1,' \ diff --git a/docker/plays/devpi.yml b/docker/plays/devpi.yml deleted file mode 100644 index f05210ec798..00000000000 --- a/docker/plays/devpi.yml +++ /dev/null @@ -1,11 +0,0 @@ -# Usage: ansible-playbook devpi.yml -i , -e /admin/edx_admin.yml -e /admin/admin.yml -- name: Configure instance(s) - hosts: all - become: True - gather_facts: True - vars: - serial_count: 1 - serial: "{{ serial_count }}" - roles: - - common - - devpi diff --git a/playbooks/roles/devpi/defaults/main.yml b/playbooks/roles/devpi/defaults/main.yml deleted file mode 100644 index 367d59d3f85..00000000000 --- a/playbooks/roles/devpi/defaults/main.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -## -# Defaults for role devpi - -devpi_group: devpi -devpi_user: devpi -devpi_app_dir: /home/devpi -devpi_venv_dir: devpi_venv -devpi_venv_path: "{{ devpi_app_dir }}/venvs/{{ devpi_venv_dir }}" - -devpi_environment: - PIP_NO_CACHE_DIR: "off" - PIP_INDEX_URL: "https://pypi.python.org/simple" - PIP_TRUSTED_HOST: "127.0.0.1" - VIRTUAL_ENV: "{{ devpi_venv_path }}" - PATH: $VIRTUAL_ENV/bin:$PATH - -# The versions below are required to be passed-in to the role. -DEVPI_REQUIREMENTS: - - name: devpi-server - version: "{{ DEVPI_SERVER_VERSION }}" - - name: devpi-web - version: "{{ DEVPI_WEB_VERSION }}" - - name: devpi-client - version: "{{ DEVPI_CLIENT_VERSION }}" diff --git a/playbooks/roles/devpi/tasks/main.yml b/playbooks/roles/devpi/tasks/main.yml deleted file mode 100644 index 2396cd95927..00000000000 --- a/playbooks/roles/devpi/tasks/main.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# -# edX Configuration -# -# github: https://github.com/openedx/configuration -# wiki: https://openedx.atlassian.net/wiki/display/OpenOPS -# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions -# license: https://github.com/openedx/configuration/blob/master/LICENSE.TXT -# -# -# -# Tasks for role devpi -# -# Overview: -# -# -# Dependencies: -# -# -# Example play: -# -# - -- name: Create the application group - group: - name: "{{ devpi_group }}" - state: present - tags: - - "install" - -- name: Create application user - user: - name: "{{ devpi_user }}" - home: "{{ devpi_app_dir }}" - group: "{{ devpi_group }}" - createhome: no - shell: /bin/false - tags: - - "install" - -- name: Create devpi user dirs - file: - path: "{{ item }}" - state: directory - owner: "{{ devpi_user }}" - group: "{{ devpi_group }}" - with_items: - - "{{ devpi_app_dir }}" - - "{{ devpi_venv_dir }}" - tags: - - "install" - -- name: install python requirements - pip: - name: "{{ item.name }}" - version: "{{ item.version|default(omit) }}" - extra_args: "--exists-action w {{ item.extra_args|default('') }}" - virtualenv: "{{ devpi_venv_path }}" - state: present - with_items: "{{ DEVPI_REQUIREMENTS }}" - become_user: "{{ devpi_user }}" - tags: - - "install" diff --git a/util/parsefiles_config.yml b/util/parsefiles_config.yml index 7b83943242d..99c6ed93aab 100644 --- a/util/parsefiles_config.yml +++ b/util/parsefiles_config.yml @@ -32,7 +32,6 @@ weights: - ecomworker: 4 - notes: 2 - mongo: 1 - - devpi: 1 - analytics_pipeline: 8 - analytics_pipeline_hadoop_datanode: 2 - analytics_pipeline_hadoop_namenode: 3 From 2931d46945214c222c023f9d689ed1efccf90e1b Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 11 Oct 2023 15:48:14 +0500 Subject: [PATCH 566/664] chore: deleting table for `lx-pathway-plugin`. --- playbooks/roles/edxapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 6c7d75a9732..cb4c000503e 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -584,7 +584,7 @@ EDXAPP_PRIVATE_REQUIREMENTS: - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e # "Pathways" learning context plugin for the LabXchange project - - name: git+https://github.com/irtazaakram/lx-pathway-plugin.git@8fe72de587094f81b7d28123e55ca06f7e0ac1a1#egg=lx-pathway-plugin + - name: git+https://github.com/open-craft/lx-pathway-plugin.git@0b695f01c79664e0dc50e5ae9a315fa10a6c0bc5#egg=lx-pathway-plugin extra_args: -e # Caliper and xAPI event routing plugin - name: edx-event-routing-backends==5.5.6 From c285cb9a1b4bb7dcd41ff9e0c40cf4bff6b19461 Mon Sep 17 00:00:00 2001 From: zubairshakoorarbisoft Date: Tue, 15 Aug 2023 13:35:41 +0500 Subject: [PATCH 567/664] fix: devpi_consumer role has been removed froom configuratioon as well. --- docker/build/notes/Dockerfile | 18 +-------------- docker/plays/ecommerce.yml | 1 - docker/plays/edxapp.yml | 1 - .../roles/devpi_consumer/defaults/main.yml | 20 ----------------- playbooks/roles/devpi_consumer/tasks/main.yml | 22 ------------------- .../devpi_consumer/templates/pip.conf.j2 | 11 ---------- 6 files changed, 1 insertion(+), 72 deletions(-) delete mode 100644 playbooks/roles/devpi_consumer/defaults/main.yml delete mode 100644 playbooks/roles/devpi_consumer/tasks/main.yml delete mode 100644 playbooks/roles/devpi_consumer/templates/pip.conf.j2 diff --git a/docker/build/notes/Dockerfile b/docker/build/notes/Dockerfile index a0fc6742155..66619362211 100644 --- a/docker/build/notes/Dockerfile +++ b/docker/build/notes/Dockerfile @@ -16,9 +16,6 @@ ENV OPENEDX_RELEASE=${OPENEDX_RELEASE} ENV NOTES_VERSION=${OPENEDX_RELEASE} ENV REPO_OWNER=edx - -ENV EDX_NOTES_API_VENV="/edx/edx_notes_api/venvs/edx_notes_api" - ADD . /edx/app/edx_ansible/edx_ansible WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays @@ -27,20 +24,7 @@ COPY docker/build/notes/ansible_overrides.yml / COPY docker/build/notes/edx_notes_api.yml /edx/etc/edx_notes_api.yml -# Ansible Free work start -RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev python3-virtualenv python3-pip -RUN apt-get install -y sudo - - - -RUN pip install -r /edx/edx_notes_api/edx_notes_api/requirements/base.txt - - -# Ansible Free work end - - - - +RUN sudo apt-get update && sudo apt-get -y install python3-dev libmysqlclient-dev RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook notes.yml \ -c local -i '127.0.0.1,' \ diff --git a/docker/plays/ecommerce.yml b/docker/plays/ecommerce.yml index 14987139153..ef06356805a 100644 --- a/docker/plays/ecommerce.yml +++ b/docker/plays/ecommerce.yml @@ -6,7 +6,6 @@ serial_count: 1 serial: "{{ serial_count }}" roles: - - devpi_consumer - ecommerce - sqlite_fix - browsers diff --git a/docker/plays/edxapp.yml b/docker/plays/edxapp.yml index ccdc314f9f9..a1504d907dc 100644 --- a/docker/plays/edxapp.yml +++ b/docker/plays/edxapp.yml @@ -7,5 +7,4 @@ serial: "{{ serial_count }}" roles: - common - - devpi_consumer - edxapp diff --git a/playbooks/roles/devpi_consumer/defaults/main.yml b/playbooks/roles/devpi_consumer/defaults/main.yml deleted file mode 100644 index 32295557579..00000000000 --- a/playbooks/roles/devpi_consumer/defaults/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Variables for the devpi_consumer role -# This role allows a host to use the configured devpi server as a primary source for pip -# Defaults are for Docker Devstack -# - -# This should be a directory, pip.conf will be appended -DEVPI_PIP_CONF_PATH: /root/.pip -DEVPI_PIP_CONF_OWNER: root -DEVPI_PIP_CONF_GROUP: root - -DEVPI_HOST: edx.devstack.devpi -DEVPI_PORT: 3141 - -# http or https -# -DEVPI_PROTOCOL: http - -# Use a leading slash, but no trailing slash here -# -DEVPI_INDEX: /root/pypi diff --git a/playbooks/roles/devpi_consumer/tasks/main.yml b/playbooks/roles/devpi_consumer/tasks/main.yml deleted file mode 100644 index 83a4633d0d9..00000000000 --- a/playbooks/roles/devpi_consumer/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -- name: create pip conf directory - file: - path: "{{ DEVPI_PIP_CONF_PATH }}" - state: directory - owner: "{{ DEVPI_PIP_CONF_OWNER }}" - group: "{{ DEVPI_PIP_CONF_GROUP }}" - when: devstack is defined and devstack - tags: - - devstack - - devstack:install - -- name: write pip.conf to devstack - template: - src: "pip.conf.j2" - dest: "{{ DEVPI_PIP_CONF_PATH }}/pip.conf" - owner: "{{ DEVPI_PIP_CONF_OWNER }}" - group: "{{ DEVPI_PIP_CONF_GROUP }}" - mode: 0744 - when: devstack is defined and devstack - tags: - - devstack - - devstack:install diff --git a/playbooks/roles/devpi_consumer/templates/pip.conf.j2 b/playbooks/roles/devpi_consumer/templates/pip.conf.j2 deleted file mode 100644 index 15bd7bb9025..00000000000 --- a/playbooks/roles/devpi_consumer/templates/pip.conf.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[global] -index-url = {{ DEVPI_PROTOCOL }}://{{ DEVPI_HOST }}:{{ DEVPI_PORT }}{{ DEVPI_INDEX }}/+simple/ - -# Enables failover to PyPI if devpi is broken -extra-index-url = https://pypi.python.org/simple - -# This is necessary if you aren't using TLS on the DevPI host -trusted-host = {{ DEVPI_HOST }} - -[search] -index = {{ DEVPI_PROTOCOL }}://{{ DEVPI_HOST }}:{{ DEVPI_PORT }}{{ DEVPI_INDEX }}/ From 8da3ed030db9561bd450b9a3358e99b44442c617 Mon Sep 17 00:00:00 2001 From: Yagnesh Nayi <127923546+Yagnesh1998@users.noreply.github.com> Date: Wed, 11 Oct 2023 20:51:12 +0530 Subject: [PATCH 568/664] feat: Remove JWT_AUTH_REFRESH_COOKIE (#7042) The setting JWT_AUTH_REFRESH_COOKIE is meaningless and unused and is being cleaned up to avoid confusion. See DEPR: https://github.com/openedx/public-engineering/issues/190 --- docker/build/enterprise_catalog/enterprise_catalog.yml | 1 - playbooks/roles/common_vars/defaults/main.yml | 1 - playbooks/roles/ecommerce/defaults/main.yml | 1 - playbooks/roles/edx_django_service/defaults/main.yml | 1 - .../edx_django_service_with_rendered_config/defaults/main.yml | 1 - playbooks/roles/edx_notes_api/defaults/main.yml | 1 - playbooks/roles/edxapp/defaults/main.yml | 2 -- util/helm_values_to_rst_table_util/README.md | 1 - 8 files changed, 9 deletions(-) diff --git a/docker/build/enterprise_catalog/enterprise_catalog.yml b/docker/build/enterprise_catalog/enterprise_catalog.yml index 7ba03dd619d..89cc293c26d 100644 --- a/docker/build/enterprise_catalog/enterprise_catalog.yml +++ b/docker/build/enterprise_catalog/enterprise_catalog.yml @@ -41,7 +41,6 @@ EXTRA_APPS: [] JWT_AUTH: JWT_AUTH_COOKIE_HEADER_PAYLOAD: edx-jwt-cookie-header-payload JWT_AUTH_COOKIE_SIGNATURE: edx-jwt-cookie-signature - JWT_AUTH_REFRESH_COOKIE: edx-jwt-refresh-cookie JWT_ISSUERS: - AUDIENCE: lms-key ISSUER: http://localhost:18000/oauth2 diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index db161b70c92..4b1cfbeff04 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -272,7 +272,6 @@ COMMON_JWT_PUBLIC_SIGNING_JWK_SET: '' COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD: 'edx-jwt-cookie-header-payload' COMMON_JWT_AUTH_COOKIE_SIGNATURE: 'edx-jwt-cookie-signature' -COMMON_JWT_AUTH_REFRESH_COOKIE: 'edx-jwt-refresh-cookie' # To be deprecated, in favor of the above COMMON_JWT_PUBLIC_SIGNING_JWK_SET. COMMON_JWT_SECRET_KEY: 'SET-ME-PLEASE' diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index f297d61e230..f02149026a8 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -226,7 +226,6 @@ ecommerce_config: JWT_PUBLIC_SIGNING_JWK_SET: '{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}' JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' - JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' EDX_DRF_EXTENSIONS: '{{ ECOMMERCE_EDX_DRF_EXTENSIONS }}' diff --git a/playbooks/roles/edx_django_service/defaults/main.yml b/playbooks/roles/edx_django_service/defaults/main.yml index 818cee92921..19eb3a973b5 100644 --- a/playbooks/roles/edx_django_service/defaults/main.yml +++ b/playbooks/roles/edx_django_service/defaults/main.yml @@ -183,7 +183,6 @@ edx_django_service_jwt_auth: JWT_PUBLIC_SIGNING_JWK_SET: '{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}' JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' - JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' edx_django_service_extra_apps: [] diff --git a/playbooks/roles/edx_django_service_with_rendered_config/defaults/main.yml b/playbooks/roles/edx_django_service_with_rendered_config/defaults/main.yml index 03d5e4f3988..bf311d99f7f 100644 --- a/playbooks/roles/edx_django_service_with_rendered_config/defaults/main.yml +++ b/playbooks/roles/edx_django_service_with_rendered_config/defaults/main.yml @@ -166,7 +166,6 @@ edx_django_service_with_rendered_config_jwt_auth: JWT_PUBLIC_SIGNING_JWK_SET: '{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}' JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' - JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' edx_django_service_with_rendered_config_extra_apps: [] diff --git a/playbooks/roles/edx_notes_api/defaults/main.yml b/playbooks/roles/edx_notes_api/defaults/main.yml index bf81f11b35e..3da053356bd 100644 --- a/playbooks/roles/edx_notes_api/defaults/main.yml +++ b/playbooks/roles/edx_notes_api/defaults/main.yml @@ -101,7 +101,6 @@ edx_notes_api_service_config: JWT_PUBLIC_SIGNING_JWK_SET: '{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}' JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' - JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' USERNAME_REPLACEMENT_WORKER: "{{ EDX_NOTES_USERNAME_REPLACEMENT_WORKER }}" # diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 478a445a10c..2dea9dd370b 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -434,7 +434,6 @@ EDXAPP_JWT_SECRET_KEY: "{{ COMMON_JWT_SECRET_KEY }}" EDXAPP_JWT_PUBLIC_SIGNING_JWK_SET: "{{ COMMON_JWT_PUBLIC_SIGNING_JWK_SET|string }}" EDXAPP_JWT_AUTH_COOKIE_HEADER_PAYLOAD: '{{ COMMON_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}' EDXAPP_JWT_AUTH_COOKIE_SIGNATURE: '{{ COMMON_JWT_AUTH_COOKIE_SIGNATURE }}' -EDXAPP_JWT_AUTH_REFRESH_COOKIE: '{{ COMMON_JWT_AUTH_REFRESH_COOKIE }}' # See https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/oauth_dispatch/docs/decisions/0008-use-asymmetric-jwts.rst EDXAPP_JWT_SIGNING_ALGORITHM: !!null @@ -1429,7 +1428,6 @@ generic_env_config: &edxapp_generic_env JWT_PRIVATE_SIGNING_JWK: "{{ EDXAPP_JWT_PRIVATE_SIGNING_JWK|string }}" JWT_AUTH_COOKIE_HEADER_PAYLOAD: "{{ EDXAPP_JWT_AUTH_COOKIE_HEADER_PAYLOAD }}" JWT_AUTH_COOKIE_SIGNATURE: "{{ EDXAPP_JWT_AUTH_COOKIE_SIGNATURE }}" - JWT_AUTH_REFRESH_COOKIE: "{{ EDXAPP_JWT_AUTH_REFRESH_COOKIE }}" # edx-rbac Setting SYSTEM_WIDE_ROLE_CLASSES: "{{ EDXAPP_SYSTEM_WIDE_ROLE_CLASSES }}" diff --git a/util/helm_values_to_rst_table_util/README.md b/util/helm_values_to_rst_table_util/README.md index 28749900622..bbc1c7a999b 100644 --- a/util/helm_values_to_rst_table_util/README.md +++ b/util/helm_values_to_rst_table_util/README.md @@ -40,7 +40,6 @@ Example: app.config.HAYSTACK_CONNECTIONS.default.URL TODO http://notes-elasticsearch-client:9200/ app.config.JWT_AUTH.JWT_AUTH_COOKIE_HEADER_PAYLOAD TODO stage-edx-jwt-cookie-header-payload app.config.JWT_AUTH.JWT_AUTH_COOKIE_SIGNATURE TODO stage-edx-jwt-cookie-signature - app.config.JWT_AUTH.JWT_AUTH_REFRESH_COOKIE TODO stage-edx-jwt-refresh-cookie app.config.JWT_AUTH.JWT_ISSUERS TODO [] app.config.JWT_AUTH.JWT_PUBLIC_SIGNING_JWK_SET TODO app.config.RESULTS_DEFAULT_SIZE TODO 25 From 51e42c841929d9e82951f644229813611e95982b Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:00:29 +0500 Subject: [PATCH 569/664] fix: add pkg-config apt package to fix build --- playbooks/roles/xqueue/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/roles/xqueue/defaults/main.yml b/playbooks/roles/xqueue/defaults/main.yml index 74336a5a199..507665e566c 100644 --- a/playbooks/roles/xqueue/defaults/main.yml +++ b/playbooks/roles/xqueue/defaults/main.yml @@ -180,6 +180,8 @@ xqueue_debian_pkgs: # Needed to be able to create the xqueue mysqldb. # Needed to be able to build wheel for mysqlclient - python3-dev + # Needed for mysqlcient==2.2.0 python pacakge + - pkg-config xqueue_release_specific_debian_pkgs: xenial: - python-mysqldb From fc5c996ebd21f4be13af78feb6819425db656fa9 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 18 Oct 2023 19:06:02 +0500 Subject: [PATCH 570/664] chore: removing `lx-pathway-plugin`. --- playbooks/roles/edxapp/defaults/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 6af82e9f392..cfe876d7089 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -582,9 +582,6 @@ EDXAPP_PRIVATE_REQUIREMENTS: # XBlocks associated with the LabXchange project - name: git+https://github.com/open-craft/labxchange-xblocks.git@a0a8a8dad13199014d4bb29cee416289880bde0b#egg=labxchange-xblocks extra_args: -e - # "Pathways" learning context plugin for the LabXchange project - - name: git+https://github.com/open-craft/lx-pathway-plugin.git@0b695f01c79664e0dc50e5ae9a315fa10a6c0bc5#egg=lx-pathway-plugin - extra_args: -e # Caliper and xAPI event routing plugin - name: edx-event-routing-backends==5.5.6 From 6d9fe216b2ea407a1e5beed6f29e057985bf3f8a Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Thu, 19 Oct 2023 11:24:00 -0400 Subject: [PATCH 571/664] fix: correctly configure retries (#7049) --- playbooks/roles/ad_hoc_reporting/tasks/main.yml | 2 ++ playbooks/roles/mongo_2_6/tasks/main.yml | 2 ++ playbooks/roles/mongo_3_2/tasks/main.yml | 2 ++ playbooks/roles/mongo_3_4/tasks/main.yml | 2 ++ playbooks/roles/mongo_3_6/tasks/main.yml | 2 ++ playbooks/roles/mongo_4_0/tasks/main.yml | 2 ++ playbooks/roles/mongo_4_2/tasks/main.yml | 2 ++ playbooks/roles/mongo_4_4/tasks/main.yml | 2 ++ playbooks/roles/mongo_client/tasks/main.yml | 2 ++ playbooks/roles/python/tasks/main.yml | 2 ++ playbooks/roles/redis/tasks/main.yml | 2 ++ 11 files changed, 22 insertions(+) diff --git a/playbooks/roles/ad_hoc_reporting/tasks/main.yml b/playbooks/roles/ad_hoc_reporting/tasks/main.yml index 4274661bff3..7b679fc8a3d 100644 --- a/playbooks/roles/ad_hoc_reporting/tasks/main.yml +++ b/playbooks/roles/ad_hoc_reporting/tasks/main.yml @@ -32,8 +32,10 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - install:system-requirements + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_2_6/tasks/main.yml b/playbooks/roles/mongo_2_6/tasks/main.yml index c730f1a92d9..c3b6dfe6ed1 100644 --- a/playbooks/roles/mongo_2_6/tasks/main.yml +++ b/playbooks/roles/mongo_2_6/tasks/main.yml @@ -41,10 +41,12 @@ id: "{{ MONGODB_APT_KEY }}" keyserver: "{{ MONGODB_APT_KEYSERVER }}" state: present + register: add_mongo_signing_key retries: 3 tags: - install - install:base + until: add_mongo_signing_key is succeeded - name: Add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_3_2/tasks/main.yml b/playbooks/roles/mongo_3_2/tasks/main.yml index 83672f59043..62981d9cf50 100644 --- a/playbooks/roles/mongo_3_2/tasks/main.yml +++ b/playbooks/roles/mongo_3_2/tasks/main.yml @@ -47,9 +47,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_3_4/tasks/main.yml b/playbooks/roles/mongo_3_4/tasks/main.yml index f16432b6322..10dd2484cdf 100644 --- a/playbooks/roles/mongo_3_4/tasks/main.yml +++ b/playbooks/roles/mongo_3_4/tasks/main.yml @@ -36,9 +36,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_3_6/tasks/main.yml b/playbooks/roles/mongo_3_6/tasks/main.yml index b7512851884..5a1b83db774 100644 --- a/playbooks/roles/mongo_3_6/tasks/main.yml +++ b/playbooks/roles/mongo_3_6/tasks/main.yml @@ -47,9 +47,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_4_0/tasks/main.yml b/playbooks/roles/mongo_4_0/tasks/main.yml index 405b2142b6b..ab41799ba60 100644 --- a/playbooks/roles/mongo_4_0/tasks/main.yml +++ b/playbooks/roles/mongo_4_0/tasks/main.yml @@ -47,9 +47,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_4_2/tasks/main.yml b/playbooks/roles/mongo_4_2/tasks/main.yml index 127a90ae906..a4ede61efb7 100644 --- a/playbooks/roles/mongo_4_2/tasks/main.yml +++ b/playbooks/roles/mongo_4_2/tasks/main.yml @@ -36,9 +36,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_4_4/tasks/main.yml b/playbooks/roles/mongo_4_4/tasks/main.yml index f16432b6322..10dd2484cdf 100644 --- a/playbooks/roles/mongo_4_4/tasks/main.yml +++ b/playbooks/roles/mongo_4_4/tasks/main.yml @@ -36,9 +36,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - "install" - "install:app-requirements" + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/mongo_client/tasks/main.yml b/playbooks/roles/mongo_client/tasks/main.yml index 7f02755ab8f..64c71616169 100644 --- a/playbooks/roles/mongo_client/tasks/main.yml +++ b/playbooks/roles/mongo_client/tasks/main.yml @@ -4,9 +4,11 @@ url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" state: present retries: 3 + register: add_mongo_signing_key tags: - install - install:system-requirements + until: add_mongo_signing_key is succeeded - name: add the mongodb repo to the sources list apt_repository: diff --git a/playbooks/roles/python/tasks/main.yml b/playbooks/roles/python/tasks/main.yml index 490687052e6..9ca64af1723 100644 --- a/playbooks/roles/python/tasks/main.yml +++ b/playbooks/roles/python/tasks/main.yml @@ -16,5 +16,7 @@ - name: Install packages raw: "apt-get install -qq {{ item }}" with_items: "{{ python_packages }}" + register: install_packages retries: 10 delay: 10 + until: install_packages is succeeded diff --git a/playbooks/roles/redis/tasks/main.yml b/playbooks/roles/redis/tasks/main.yml index 15a8d74e1ca..7d46a71baa9 100644 --- a/playbooks/roles/redis/tasks/main.yml +++ b/playbooks/roles/redis/tasks/main.yml @@ -26,9 +26,11 @@ url: "https://packages.redis.io/gpg" state: present retries: 3 + register: add_repo_signing_key tags: - "install" - "install:app-requirements" + until: add_repo_signing_key is succeeded - name: add the redis repo to the sources list apt_repository: From 6cf747d8fc2ac4727e767cfe2023c237bdd3f299 Mon Sep 17 00:00:00 2001 From: Chris Deery <3932645+cdeery@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:33:00 -0400 Subject: [PATCH 572/664] fix: add branch git version variable --- playbooks/profile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/profile.yml b/playbooks/profile.yml index 08dc8824b5b..2698d9820d3 100644 --- a/playbooks/profile.yml +++ b/playbooks/profile.yml @@ -10,6 +10,7 @@ roles: - role: mfe MFE_NAME: profile + MFE_VERSION: '{{ PROFILE_MFE_VERSION }}' - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure From 114a559f1b1b66d7c35cc81e6880e459c12ebc81 Mon Sep 17 00:00:00 2001 From: Chris Deery <3932645+cdeery@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:36:37 -0400 Subject: [PATCH 573/664] fix: add branch version for learner dashboard --- playbooks/learner_dashboard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/learner_dashboard.yml b/playbooks/learner_dashboard.yml index 231b6e7f54c..e4d47f9d189 100644 --- a/playbooks/learner_dashboard.yml +++ b/playbooks/learner_dashboard.yml @@ -10,6 +10,7 @@ roles: - role: mfe MFE_NAME: learner-dashboard + MFE_VERSION: '{{ LEARNER_DASHBOARD_MFE_VERSION }}' - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure From 4de31db721a374c3debcd325ed6ac442d562ef32 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Thu, 19 Oct 2023 18:22:31 +0500 Subject: [PATCH 574/664] chore: added yq to jenkins requirements file --- .../roles/aws/templates/requirements.txt.j2 | 10 ++--- requirements.txt | 22 +++++----- requirements/common_constraints.txt | 5 --- requirements/jenkins.in | 1 + requirements/pip-tools.txt | 23 ++++++---- requirements/pip.txt | 10 ++--- util/elasticsearch/requirements.txt | 4 +- util/jenkins/requirements-cloudflare.txt | 8 ++-- util/jenkins/requirements.txt | 44 +++++++++++-------- util/pingdom/requirements.txt | 8 ++-- util/vpc-tools/requirements.txt | 6 +-- 11 files changed, 76 insertions(+), 65 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index a9a76cd4346..c843d480ab2 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.27.151 +awscli==1.29.66 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.26.151 +boto3==1.28.66 # via -r requirements/aws.in -botocore==1.29.151 +botocore==1.31.66 # via # awscli # boto3 @@ -39,11 +39,11 @@ rsa==4.7.2 # via awscli s3cmd==2.3.0 # via -r requirements/aws.in -s3transfer==0.6.1 +s3transfer==0.7.0 # via # awscli # boto3 six==1.16.0 # via python-dateutil -urllib3==1.26.16 +urllib3==1.26.18 # via botocore diff --git a/requirements.txt b/requirements.txt index 58a64305c50..42154e0d1f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ ansible==2.10.7 # via -r requirements/base.in ansible-base==2.10.17 # via ansible -awscli==1.27.151 +awscli==1.29.66 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -16,25 +16,25 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.26.151 +boto3==1.28.66 # via -r requirements/base.in -botocore==1.29.151 +botocore==1.31.66 # via # awscli # boto3 # s3transfer -certifi==2023.5.7 +certifi==2023.7.22 # via requests -cffi==1.15.1 +cffi==1.16.0 # via # bcrypt # cryptography # pynacl -charset-normalizer==3.1.0 +charset-normalizer==3.3.0 # via requests colorama==0.4.4 # via awscli -cryptography==41.0.1 +cryptography==41.0.4 # via # ansible-base # paramiko @@ -68,7 +68,7 @@ mysqlclient==1.4.6 # via -r requirements/base.in networkx==1.11 # via -r requirements/base.in -packaging==23.1 +packaging==23.2 # via ansible-base paramiko==2.4.2 # via -r requirements/base.in @@ -90,7 +90,7 @@ pynacl==1.5.0 # via paramiko python-dateutil==2.8.2 # via botocore -pyyaml==5.4.1 +pyyaml==6.0.1 # via # -r requirements/base.in # ansible-base @@ -101,7 +101,7 @@ requests==2.31.0 # datadog rsa==4.7.2 # via awscli -s3transfer==0.6.1 +s3transfer==0.7.0 # via # awscli # boto3 @@ -110,7 +110,7 @@ six==1.16.0 # bcrypt # pathlib2 # python-dateutil -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # requests diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 7e39123ff04..afe6aa887fe 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -25,8 +25,3 @@ django-simple-history==3.0.0 # tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. # Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 tox<4.0.0 - -# edx-sphinx-theme is not compatible with latest Sphinx==6.0.0 version -# Pinning Sphinx version unless the compatibility issue gets resolved -# For details, see issue https://github.com/openedx/edx-sphinx-theme/issues/197 -sphinx<6.0.0 diff --git a/requirements/jenkins.in b/requirements/jenkins.in index aab284fe445..580f4262783 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -14,3 +14,4 @@ PyMySQL==0.9.3 python-gnupg redis==2.10.6 splunk-sdk==1.6.16 # older versions have been yanked +yq diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 8620fabd6b5..50d35f22e8d 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,23 +1,30 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -build==0.10.0 +build==1.0.3 # via pip-tools -click==8.1.3 +click==8.1.7 # via pip-tools -packaging==23.1 +importlib-metadata==6.8.0 # via build -pip-tools==6.13.0 +packaging==23.2 + # via build +pip-tools==7.3.0 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via build tomli==2.0.1 - # via build -wheel==0.40.0 + # via + # build + # pip-tools + # pyproject-hooks +wheel==0.41.2 # via pip-tools +zipp==3.17.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip.txt b/requirements/pip.txt index c9cbf009061..2154d29f936 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,14 +1,14 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -wheel==0.40.0 +wheel==0.41.2 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.1.2 +pip==23.3 # via -r requirements/pip.in -setuptools==67.8.0 +setuptools==68.2.2 # via -r requirements/pip.in diff --git a/util/elasticsearch/requirements.txt b/util/elasticsearch/requirements.txt index 188701ce669..41dbdaf1fe3 100644 --- a/util/elasticsearch/requirements.txt +++ b/util/elasticsearch/requirements.txt @@ -8,7 +8,7 @@ deepdiff==3.1.0 # via -r requirements/elasticsearch.in elasticsearch==0.4.5 # via -r requirements/elasticsearch.in -jsonpickle==3.0.1 +jsonpickle==3.0.2 # via deepdiff -urllib3==1.26.16 +urllib3==1.26.18 # via elasticsearch diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 2fbd1f0b7c6..9e1d78c3973 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,15 +4,15 @@ # # make upgrade # -certifi==2023.5.7 +certifi==2023.7.22 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.3.0 # via requests -click==8.1.3 +click==8.1.7 # via -r requirements/cloudflare.in idna==3.4 # via requests requests==2.31.0 # via -r requirements/cloudflare.in -urllib3==2.0.3 +urllib3==2.0.7 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 20b4a05fd7c..99c520d04e3 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -6,7 +6,9 @@ # amqp==5.1.1 # via kombu -awscli==1.27.151 +argcomplete==3.1.2 + # via yq +awscli==1.29.66 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -18,22 +20,22 @@ billiard==4.1.0 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.26.151 +boto3==1.28.66 # via -r requirements/jenkins.in -botocore==1.29.151 +botocore==1.31.66 # via # awscli # boto3 # s3transfer -celery==5.3.0 +celery==5.3.4 # via -r requirements/jenkins.in -certifi==2023.5.7 +certifi==2023.7.22 # via # opsgenie-sdk # requests -charset-normalizer==3.1.0 +charset-normalizer==3.3.0 # via requests -click==8.1.3 +click==8.1.7 # via # -r requirements/jenkins.in # celery @@ -44,7 +46,7 @@ click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery colorama==0.4.4 # via awscli @@ -56,11 +58,11 @@ jmespath==1.0.1 # via # boto3 # botocore -kombu==5.3.0 +kombu==5.3.2 # via celery opsgenie-sdk==0.3.1 # via -r requirements/jenkins.in -prompt-toolkit==3.0.38 +prompt-toolkit==3.0.39 # via click-repl pyasn1==0.5.0 # via rsa @@ -72,16 +74,17 @@ python-dateutil==2.8.2 # celery # opsgenie-sdk # s3cmd -python-gnupg==0.5.0 +python-gnupg==0.5.1 # via -r requirements/jenkins.in python-magic==0.4.27 # via s3cmd -pytz==2023.3 +pytz==2023.3.post1 # via opsgenie-sdk -pyyaml==5.4.1 +pyyaml==6.0.1 # via # -r requirements/jenkins.in # awscli + # yq redis==2.10.6 # via -r requirements/jenkins.in requests==2.31.0 @@ -90,24 +93,25 @@ rsa==4.7.2 # via awscli s3cmd==2.3.0 # via -r requirements/jenkins.in -s3transfer==0.6.1 +s3transfer==0.7.0 # via # awscli # boto3 six==1.16.0 # via - # click-repl # opsgenie-sdk # python-dateutil splunk-sdk==1.6.16 # via -r requirements/jenkins.in -typing-extensions==4.6.3 +tomlkit==0.12.1 + # via yq +typing-extensions==4.8.0 # via kombu tzdata==2023.3 # via # backports-zoneinfo # celery -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # opsgenie-sdk @@ -117,8 +121,12 @@ vine==5.0.0 # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via prompt-toolkit +xmltodict==0.13.0 + # via yq +yq==3.2.3 + # via -r requirements/jenkins.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 9ee7f1503b8..3c17a367372 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,19 +4,19 @@ # # make upgrade # -certifi==2023.5.7 +certifi==2023.7.22 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.3.0 # via requests click==6.7 # via -r requirements/pingdom.in idna==3.4 # via requests -pyyaml==6.0 +pyyaml==6.0.1 # via -r requirements/pingdom.in requests==2.31.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==2.0.3 +urllib3==2.0.7 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index edbfba21f4c..5e75124e8b6 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,9 +6,9 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2023.5.7 +certifi==2023.7.22 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.3.0 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in @@ -16,5 +16,5 @@ idna==3.4 # via requests requests==2.31.0 # via -r requirements/vpc-tools.in -urllib3==2.0.3 +urllib3==2.0.7 # via requests From d9cb43be7ab9ee459362da1f9f982e3c7f260836 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Wed, 25 Oct 2023 13:26:34 +0500 Subject: [PATCH 575/664] chore: update connect_timeout for memcache server --- playbooks/roles/edxapp/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index cfe876d7089..26a4b2aa59c 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1330,6 +1330,7 @@ generic_cache_config: &default_generic_cache no_delay: true ignore_exc: true use_pooling: true + connect_timeout: 0.5 edxapp_revisions_config: EDX_PLATFORM_REVISION: "{{ EDX_PLATFORM_VERSION }}" From b837c1a2dd186e57b56ef9bfca755166dd0325c5 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 26 Oct 2023 09:52:21 +0500 Subject: [PATCH 576/664] chore: update mongo mms agent --- playbooks/roles/mongo_mms/defaults/main.yml | 2 +- playbooks/roles/mongo_mms/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/mongo_mms/defaults/main.yml b/playbooks/roles/mongo_mms/defaults/main.yml index a2fc6e3bc34..2acb92fcd2b 100644 --- a/playbooks/roles/mongo_mms/defaults/main.yml +++ b/playbooks/roles/mongo_mms/defaults/main.yml @@ -8,6 +8,6 @@ mongodb_agent_dir: "/data" mongodb_agent_user: "mongodb" mongodb_agent: - agent: mongodb-mms-automation-agent-manager - version: "13.5.0.8451-1" + version: "13.7.0.8514-1" config: "/etc/mongodb-mms/automation-agent.config" dir: "automation" diff --git a/playbooks/roles/mongo_mms/tasks/main.yml b/playbooks/roles/mongo_mms/tasks/main.yml index 721308d88f5..bab13647838 100644 --- a/playbooks/roles/mongo_mms/tasks/main.yml +++ b/playbooks/roles/mongo_mms/tasks/main.yml @@ -14,7 +14,7 @@ - name: download mongodb agent get_url: url: "{{ base_url }}/{{ item.dir }}/{{ item.agent }}_{{ item.version }}_{{ pkg_arch }}.{{ os_version }}.{{ pkg_format }}" - dest: "/tmp/{{ item.agent }}-{{ item.version }}.{{ pkg_format }}" + dest: "/tmp/{{ item.agent }}_{{ item.version }}.{{ pkg_format }}" register: download_mongodb_deb with_items: "{{ mongodb_agent }}" From a4916ef13c0d5b213bf8987263637f2856c2af49 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Thu, 26 Oct 2023 10:11:32 +0500 Subject: [PATCH 577/664] chore: added jq to jenkins requirements file --- playbooks/roles/aws/templates/requirements.txt.j2 | 6 +++--- requirements.txt | 10 +++++----- requirements/jenkins.in | 1 + requirements/pip.txt | 2 +- util/jenkins/requirements-cloudflare.txt | 2 +- util/jenkins/requirements.txt | 10 ++++++---- util/pingdom/requirements.txt | 2 +- util/vpc-tools/requirements.txt | 2 +- 8 files changed, 19 insertions(+), 16 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index c843d480ab2..17add96d6db 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.29.66 +awscli==1.29.70 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.28.66 +boto3==1.28.70 # via -r requirements/aws.in -botocore==1.31.66 +botocore==1.31.70 # via # awscli # boto3 diff --git a/requirements.txt b/requirements.txt index 42154e0d1f1..24cc324665a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ ansible==2.10.7 # via -r requirements/base.in ansible-base==2.10.17 # via ansible -awscli==1.29.66 +awscli==1.29.70 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -16,9 +16,9 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.28.66 +boto3==1.28.70 # via -r requirements/base.in -botocore==1.31.66 +botocore==1.31.70 # via # awscli # boto3 @@ -30,11 +30,11 @@ cffi==1.16.0 # bcrypt # cryptography # pynacl -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests colorama==0.4.4 # via awscli -cryptography==41.0.4 +cryptography==41.0.5 # via # ansible-base # paramiko diff --git a/requirements/jenkins.in b/requirements/jenkins.in index 580f4262783..0f5728b084b 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -15,3 +15,4 @@ python-gnupg redis==2.10.6 splunk-sdk==1.6.16 # older versions have been yanked yq +jq diff --git a/requirements/pip.txt b/requirements/pip.txt index 2154d29f936..0c788d61e50 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.41.2 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3 +pip==23.3.1 # via -r requirements/pip.in setuptools==68.2.2 # via -r requirements/pip.in diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 9e1d78c3973..2c7f48bb9d7 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -6,7 +6,7 @@ # certifi==2023.7.22 # via requests -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests click==8.1.7 # via -r requirements/cloudflare.in diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 99c520d04e3..1f91a34b58d 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -8,7 +8,7 @@ amqp==5.1.1 # via kombu argcomplete==3.1.2 # via yq -awscli==1.29.66 +awscli==1.29.70 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -20,9 +20,9 @@ billiard==4.1.0 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.28.66 +boto3==1.28.70 # via -r requirements/jenkins.in -botocore==1.31.66 +botocore==1.31.70 # via # awscli # boto3 @@ -33,7 +33,7 @@ certifi==2023.7.22 # via # opsgenie-sdk # requests -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests click==8.1.7 # via @@ -58,6 +58,8 @@ jmespath==1.0.1 # via # boto3 # botocore +jq==1.6.0 + # via -r requirements/jenkins.in kombu==5.3.2 # via celery opsgenie-sdk==0.3.1 diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 3c17a367372..84f610ad228 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -6,7 +6,7 @@ # certifi==2023.7.22 # via requests -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests click==6.7 # via -r requirements/pingdom.in diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 5e75124e8b6..c965b4fce70 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -8,7 +8,7 @@ boto==2.49.0 # via -r requirements/vpc-tools.in certifi==2023.7.22 # via requests -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in From 51598da8ab311ea5adb4659e9f1280ad857c1d49 Mon Sep 17 00:00:00 2001 From: Syed Imran Hassan <45480841+syedimranhassan@users.noreply.github.com> Date: Thu, 26 Oct 2023 12:46:53 +0500 Subject: [PATCH 578/664] chore: added jq pkg for jenkins job --- playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index 1d2aa6350a5..c9ac444901b 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -74,6 +74,7 @@ JENKINS_DATA_ENGINEERING_EXTRA_PKGS: - libgraphviz-dev - libxmlsec1-dev - zip + - jq # See https://www.vaultproject.io/downloads for latest version. DE_JENKINS_VAULT_CLI_VERSION: 1.9.4 From 6a9ade5d579ae1324fe2ab563063cc8586a56a52 Mon Sep 17 00:00:00 2001 From: ohnickmoy Date: Thu, 26 Oct 2023 12:43:29 -0400 Subject: [PATCH 579/664] fix: encrypt sandbox volumes --- playbooks/roles/launch_ec2/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/launch_ec2/tasks/main.yml b/playbooks/roles/launch_ec2/tasks/main.yml index df95a730ab6..33ccab3c2a1 100644 --- a/playbooks/roles/launch_ec2/tasks/main.yml +++ b/playbooks/roles/launch_ec2/tasks/main.yml @@ -52,6 +52,7 @@ volume_size: "{{ root_ebs_size }}" delete_on_termination: true volume_type: "gp2" + encrypted: true zone: "{{ zone }}" instance_profile_name: "{{ instance_profile_name }}" user_data: "{{ user_data }}" From 9bceb7652bb3a69f538bedec7c45560c9a999715 Mon Sep 17 00:00:00 2001 From: Nadeem Shahzad Date: Mon, 30 Oct 2023 10:42:23 +0500 Subject: [PATCH 580/664] Revert "chore: add APPLE_ROOT_CA environment variable in ecommerce (DOS-3642)" --- playbooks/roles/ecommerce/defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index f02149026a8..91217930af2 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -298,7 +298,6 @@ ecommerce_gunicorn_port: "8130" ecommerce_environment: ECOMMERCE_CFG: "{{ COMMON_CFG_DIR }}/{{ ecommerce_service_name }}.yml" - APPLE_ROOT_CA: "{{ COMMON_APP_DIR }}/ecommerce/ecommerce/ecommerce/extensions/iap/AppleRootCA-G3.cer" ecommerce_create_demo_data: false From be211eb6b03712ef04590f5edebfd93fd89565b9 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Mon, 30 Oct 2023 08:53:47 -0600 Subject: [PATCH 581/664] fix: add retries to unattended-upgrade --- playbooks/roles/security/tasks/security-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/security/tasks/security-ubuntu.yml b/playbooks/roles/security/tasks/security-ubuntu.yml index 1929c59688f..d2b9a69eb7d 100644 --- a/playbooks/roles/security/tasks/security-ubuntu.yml +++ b/playbooks/roles/security/tasks/security-ubuntu.yml @@ -70,5 +70,5 @@ - unattended-upgrade --dry-run - unattended-upgrade register: ubuntu_security - retries: 3 + retries: 10 until: ubuntu_security is succeeded From f9765faa9eef576e3d7e8a6fb0413e83e9c5ced2 Mon Sep 17 00:00:00 2001 From: k8 Date: Wed, 1 Nov 2023 16:31:25 -0400 Subject: [PATCH 582/664] chore: remove prospectus gatsby key JIRA:CLOUDSEC-12 --- playbooks/roles/prospectus/templates/.env.environment.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/prospectus/templates/.env.environment.j2 b/playbooks/roles/prospectus/templates/.env.environment.j2 index 72c005f9375..fcc8ba4aed8 100644 --- a/playbooks/roles/prospectus/templates/.env.environment.j2 +++ b/playbooks/roles/prospectus/templates/.env.environment.j2 @@ -11,7 +11,6 @@ GATSBY_CONTENTFUL_ACCESS_TOKEN={{ PROSPECTUS_CONTENTFUL_ACCESS_TOKEN }} GATSBY_SEGMENT_WRITE_KEY={{ PROSPECTUS_SEGMENT_WRITE_KEY }} PROSPECTUS_ALGOLIA_BROWSE_KEY={{ PROSPECTUS_ALGOLIA_BROWSE_KEY }} PROSPECTUS_ALGOLIA_ADMIN_KEY={{ PROSPECTUS_ALGOLIA_ADMIN_KEY }} -GATSBY_ZENDESK_KEY={{ PROSPECTUS_GATSBY_ZENDESK_KEY }} GATSBY_XPERT_STG_API_URL={{ PROSPECTUS_GATSBY_XPERT_STG_API_URL }} GATSBY_XPERT_UAT_API_URL={{ PROSPECTUS_GATSBY_XPERT_UAT_API_URL }} GATSBY_XPERT_PROD_API_URL={{ PROSPECTUS_GATSBY_XPERT_PROD_API_URL }} From 2c93fad532de3b019252be405aceef40ddd8749d Mon Sep 17 00:00:00 2001 From: Saad Ali Date: Mon, 6 Nov 2023 17:53:50 +0500 Subject: [PATCH 583/664] fix: Add a task postgresql_privs to set privileges (#7063) --- playbooks/create_pgsql_db_and_users.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/playbooks/create_pgsql_db_and_users.yml b/playbooks/create_pgsql_db_and_users.yml index 10a0ee780be..52a45628fbd 100644 --- a/playbooks/create_pgsql_db_and_users.yml +++ b/playbooks/create_pgsql_db_and_users.yml @@ -25,7 +25,9 @@ # db: "example" # state: "present" # password: "user-password" -# privileges: "ALL" +# privs: "ALL" +# db_objects: "ALL_DEFAULT" +# db_object_type: "default_privs" # <<: *default_connection - name: Create databases and users @@ -50,12 +52,11 @@ tags: - dbs - - name: create PostgreSQL users and assign privileges + - name: Create PostgreSQL users postgresql_user: db: "{{ item.db }}" name: "{{ item.name }}" state: "{{ item.state | default('present') }}" - priv: "{{ item.privileges }}" password: "{{ item.password }}" login_host: "{{ item.login_host }}" login_user: "{{ item.login_user }}" @@ -63,3 +64,17 @@ with_items: "{{ database_users }}" tags: - users + + - name: Assign privileges + postgresql_privs: + roles: "{{ item.name }}" + db: "{{ item.db }}" + privs: "{{ item.privs }}" + objs: "{{ item.db_objects }}" + type: "{{ item.db_object_type }}" + login_host: "{{ item.login_host }}" + login_user: "{{ item.login_user }}" + login_password: "{{ item.login_password }}" + with_items: "{{ database_users }}" + tags: + - privileges From 82ac6a2f11d88b0b5a256c4fc7014975bf4896a3 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Thu, 16 Nov 2023 12:55:55 -0700 Subject: [PATCH 584/664] feat: add launch permission modification capability --- playbooks/continuous_delivery/create_ami.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index c7e97cffcb2..f2b73d8642a 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -79,6 +79,14 @@ }" register: ami_register + - name: Allow AMI to be launched by another account + ec2_ami: + image_id: "{{ ami_register.image_id }}" + state: present + launch_permissions: + user_ids: "{{ allowed_accounts }}" + when: allowed_accounts is defined + - name: Add any tags that are on the instance to the AMI ec2_tag: region: "{{ ec2_region }}" From e923ae736bb01b46b64cdaa24adc4a64b2c7abf9 Mon Sep 17 00:00:00 2001 From: Rebecca Graber Date: Mon, 20 Nov 2023 10:12:25 -0500 Subject: [PATCH 585/664] fix: replace pycrypto with pycryptodome (#7068) --- requirements.txt | 2 +- requirements/base.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 24cc324665a..b7b41d6ea6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -82,7 +82,7 @@ pyasn1==0.5.0 # rsa pycparser==2.21 # via cffi -pycrypto==2.6.1 +pycryptodome==3.19.0 # via -r requirements/base.in pymongo==3.9.0 # via -r requirements/base.in diff --git a/requirements/base.in b/requirements/base.in index a1eaa57a6a1..e38281a4e19 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -15,7 +15,7 @@ networkx==1.11 paramiko==2.4.2 pathlib2==2.3.0 prettytable==0.7.2 -pycrypto==2.6.1 +pycryptodome pymongo==3.9.0 # Needed for the mongo_* modules (playbooks/library/mongo_*) PyYAML requests From 5044d961797442b0c801974695020188bb760822 Mon Sep 17 00:00:00 2001 From: k8 Date: Thu, 7 Dec 2023 12:45:52 -0500 Subject: [PATCH 586/664] chore: migrate edx minos to edx JIRA:CLOUDSEC-12 --- playbooks/roles/minos/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/minos/defaults/main.yml b/playbooks/roles/minos/defaults/main.yml index 813d4d7b0a8..736bee8a087 100644 --- a/playbooks/roles/minos/defaults/main.yml +++ b/playbooks/roles/minos/defaults/main.yml @@ -31,7 +31,7 @@ minos_cfg_file: "{{ COMMON_CFG_DIR }}/minos.yml" minos_voter_cfg: "{{ COMMON_CFG_DIR }}/{{ minos_service_name }}/conf.d/" minos_git_ssh: "/tmp/git.sh" minos_git_identity: "{{ minos_app_dir }}/minos-git-identity" -minos_edx_server_tools_repo: "git@github.com/edx-ops/edx-minos.git" +minos_edx_server_tools_repo: "git@github.com/edx/edx-minos.git" minos_edx_server_tools_branch: "master" MINOS_EDX_SERVER_TOOLS_VERSION: "0.4" minos_requirement: "git+ssh://{{ minos_edx_server_tools_repo }}@{{ minos_edx_server_tools_branch }}#egg=edx-minos" From 2605d6cc914650f69337b50ffec0edb786397a8d Mon Sep 17 00:00:00 2001 From: Joseph Mulloy Date: Fri, 8 Dec 2023 14:12:59 -0500 Subject: [PATCH 587/664] fix: Move hashed filenames to common path GSRE-991 Move filenames that are hashed based on content to a common bucket path in S3 so that we don't have search glitches from 404s on deploys --- .../app/nginx/sites-available/prospectus.j2 | 24 ++++++++++++++++--- playbooks/roles/prospectus/tasks/main.yml | 10 +++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index be8ade41525..5f64e56dab1 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -81,16 +81,34 @@ server { {% endif %} } - # Cache js/css for a long time at the edge, they are versioned in their names - location ~ \.(js|css)$ { + # Look for hashed .css, .js and .map files in bucket/static_hashed to prevent 404 when cloudflare cache is cleared + # Regex only looks in root path, which are the only .css, .js and .map files with content hash filenames + location ~* ^/[^/]+\.(css|js|map)$ { + # Cache js/css for a long time at the edge, they are versioned in their names add_header 'Cache-Control' 'public, max-age=31536000, immutable'; {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} - proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/{{ PROSPECTUS_S3_HOSTING_PREFIX }}$request_uri; + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/static_hashed$request_uri; # Hide client headers from S3 to prevent request headers too big error proxy_pass_request_headers off; {% endif %} } + {% if PROSPECTUS_S3_HOSTING_PROXY_ENABLED %} + # Look for /page-data/sq/d/*.js files in bucket/static_hashed to prevent 404 when cloudflare cache is cleared + location /page-data/sq/d/ { + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/static_hashed$request_uri; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; + } + + # Look for /static/* files in bucket/static_hashed to prevent 404 when cloudflare cache is cleared + location /static/ { + proxy_pass {{ PROSPECTUS_S3_HOSTING_BUCKET_URL }}/static_hashed$request_uri; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; + } + {% endif %} + # images sometimes change, we want to cache them for an hour at the edge to reduce bandwidth. location /images/ { diff --git a/playbooks/roles/prospectus/tasks/main.yml b/playbooks/roles/prospectus/tasks/main.yml index f178f759a82..8c35966d5d2 100644 --- a/playbooks/roles/prospectus/tasks/main.yml +++ b/playbooks/roles/prospectus/tasks/main.yml @@ -241,7 +241,15 @@ - name: Upload prospectus to S3 become_user: "{{ prospectus_user }}" - shell: 'aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ PROSPECTUS_S3_HOSTING_PREFIX | default(PROSPECTUS_VERSION, true) }}' + shell: "aws s3 sync --quiet {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/{{ PROSPECTUS_S3_HOSTING_PREFIX | default(PROSPECTUS_VERSION, true) }}" + when: PROSPECTUS_S3_UPLOAD_ENABLED|bool + tags: + - install + - install:system-requirements + + - name: Upload prospectus hashed static files to S3 + become_user: "{{ prospectus_user }}" + shell: "aws s3 cp --recursive {{ PROSPECTUS_DATA_DIR }} s3://{{ PROSPECTUS_S3_HOSTING_BUCKET }}/static_hashed --exclude '*' --include '/*.css' --include '/*.js' --include '/*.map' --exclude '*/*' --include 'static/*' --include 'page-data/sq/d/*'" when: PROSPECTUS_S3_UPLOAD_ENABLED|bool tags: - install From 828bd2013503f51b3aae8fa7c021fee716204858 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Mon, 11 Dec 2023 12:49:21 +0500 Subject: [PATCH 588/664] chore: Update list mysql script to check specific regions --- .../list_mysql_process/list_mysql_process.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/util/jenkins/list_mysql_process/list_mysql_process.py b/util/jenkins/list_mysql_process/list_mysql_process.py index f6293ef2199..d1ccedb2c5d 100644 --- a/util/jenkins/list_mysql_process/list_mysql_process.py +++ b/util/jenkins/list_mysql_process/list_mysql_process.py @@ -26,7 +26,7 @@ def describe_db_instances(self): return self.client.describe_db_instances() -def rds_extractor(environment): +def rds_extractor(environment, regioninclude): """ Return list of all RDS instances across all the regions Returns: @@ -45,16 +45,22 @@ def rds_extractor(environment): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) + if regioninclude: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in regioninclude]} for region in regions_list["Regions"]: - client = RDSBotoWrapper(region_name=region["RegionName"]) - response = client.describe_db_instances() - for instance in response.get('DBInstances'): - if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: - temp_dict = {} - temp_dict["name"] = instance["DBInstanceIdentifier"] - temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") - temp_dict["Port"] = instance.get("Port") - rds_list.append(temp_dict) + try: + client = RDSBotoWrapper(region_name=region["RegionName"]) + response = client.describe_db_instances() + for instance in response.get('DBInstances'): + if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: + temp_dict = {} + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") + temp_dict["Port"] = instance.get("Port") + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) return rds_list @@ -112,7 +118,8 @@ def check_queries_running(rds_list, username, password): @click.option('--password', envvar='PASSWORD', required=True) @click.option('--environment', required=True, help='Use to identify the environment') @click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') -def controller(username, password, environment, rdsignore): +@click.option('--regioninclude', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def controller(username, password, environment, rdsignore, regioninclude): """ Control execution of all other functions Arguments: @@ -125,7 +132,7 @@ def controller(username, password, environment, rdsignore): environment (str): Get this from cli args """ - rds_list = rds_extractor(environment) + rds_list = rds_extractor(environment, regioninclude) filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) process_list = check_queries_running(filtered_rds_list, username, password) if len(process_list) > 0: From 720b9c48d96d6febe811b4f68e4ab68b1a33072a Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 12 Dec 2023 12:23:36 +0500 Subject: [PATCH 589/664] chore: Update check primray key job to run for whitelisted regions --- .../primary_keys/check_primary_keys.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/util/jenkins/primary_keys/check_primary_keys.py b/util/jenkins/primary_keys/check_primary_keys.py index e7f6673038e..61ea5b87f81 100644 --- a/util/jenkins/primary_keys/check_primary_keys.py +++ b/util/jenkins/primary_keys/check_primary_keys.py @@ -118,7 +118,7 @@ def send_an_email(to_addr, from_addr, primary_keys_message, region): ) -def get_rds_from_all_regions(): +def get_rds_from_all_regions(whitelistregions): """ Gets a list of RDS instances across all the regions and deployments in AWS @@ -142,17 +142,23 @@ def get_rds_from_all_regions(): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} for region in regions_list["Regions"]: - print("Getting RDS instances in region {}".format(region["RegionName"])) - rds_client = RDSBotoWrapper(region_name=region["RegionName"]) - response = rds_client.describe_db_instances() - for instance in response.get('DBInstances'): - if "test" not in instance["DBInstanceIdentifier"]: - temp_dict = dict() - temp_dict["name"] = instance["DBInstanceIdentifier"] - temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") - temp_dict["Port"] = instance.get("Port") - rds_list.append(temp_dict) + try: + print("Getting RDS instances in region {}".format(region["RegionName"])) + rds_client = RDSBotoWrapper(region_name=region["RegionName"]) + response = rds_client.describe_db_instances() + for instance in response.get('DBInstances'): + if "test" not in instance["DBInstanceIdentifier"]: + temp_dict = dict() + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") + temp_dict["Port"] = instance.get("Port") + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) return rds_list @@ -334,7 +340,8 @@ def get_metrics_and_calcuate_diff(namespace, metric_name, dimension, value, curr @click.option('--recipient', multiple=True, help='Recipient Email address') @click.option('--sender', multiple=True, help='Sender email address') @click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') -def controller(username, password, environment, deploy, region, recipient, sender, rdsignore): +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def controller(username, password, environment, deploy, region, recipient, sender, rdsignore, whitelistregions): """ calls other function and calculate the results :param username: username for the RDS. @@ -342,7 +349,7 @@ def controller(username, password, environment, deploy, region, recipient, sende :return: None """ # get list of all the RDSes across all the regions and deployments - rds_list = get_rds_from_all_regions() + rds_list = get_rds_from_all_regions(whitelistregions) filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) table_list = check_primary_keys(filtered_rds_list, username, password, environment, deploy) if len(table_list) > 0: From 8ee862041a9303fe1b2c70b78d9c0adb96a1957d Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 12 Dec 2023 12:57:49 +0500 Subject: [PATCH 590/664] chore: Update check dead lock script to run for whitelisted regions --- .../export_dead_locks/export_dead_locks.py | 37 +++++++++++-------- .../list_mysql_process/list_mysql_process.py | 12 +++--- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/util/jenkins/export_dead_locks/export_dead_locks.py b/util/jenkins/export_dead_locks/export_dead_locks.py index a2d048c5899..4bdc74ce18f 100644 --- a/util/jenkins/export_dead_locks/export_dead_locks.py +++ b/util/jenkins/export_dead_locks/export_dead_locks.py @@ -30,7 +30,7 @@ def describe_db_instances(self): return self.client.describe_db_instances() -def rds_extractor(environment): +def rds_extractor(environment, whitelistregions): """ Return list of all RDS instances across all the regions Returns: @@ -49,19 +49,25 @@ def rds_extractor(environment): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} for region in regions_list["Regions"]: - rds_client = RDSBotoWrapper(region_name=region["RegionName"]) - response = rds_client.describe_db_instances() - for instance in response.get('DBInstances'): - if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: - temp_dict = {} - temp_dict["name"] = instance["DBInstanceIdentifier"] - temp_dict["ARN"] = instance["DBInstanceArn"] - temp_dict["Region"] = region["RegionName"] - temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") - temp_dict["Username"] = instance.get("MasterUsername") - temp_dict["Port"] = instance.get("Port") - rds_list.append(temp_dict) + try: + rds_client = RDSBotoWrapper(region_name=region["RegionName"]) + response = rds_client.describe_db_instances() + for instance in response.get('DBInstances'): + if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: + temp_dict = {} + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["ARN"] = instance["DBInstanceArn"] + temp_dict["Region"] = region["RegionName"] + temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") + temp_dict["Username"] = instance.get("MasterUsername") + temp_dict["Port"] = instance.get("Port") + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) return rds_list @@ -110,8 +116,9 @@ def rds_controller(rds_list, username, password, hostname, splunkusername, splun @click.option('--port', required=True, help='Use to identify the splunk port') @click.option('--indexname', required=True, help='Use to identify the splunk index name') @click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') -def main(username, password, environment, hostname, splunkusername, splunkpassword, port, indexname, rdsignore): - rds_list = rds_extractor(environment) +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def main(username, password, environment, hostname, splunkusername, splunkpassword, port, indexname, rdsignore, whitelistregions): + rds_list = rds_extractor(environment, whitelistregions) filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) rds_controller(filtered_rds_list, username, password, hostname, splunkusername, splunkpassword, port, indexname) diff --git a/util/jenkins/list_mysql_process/list_mysql_process.py b/util/jenkins/list_mysql_process/list_mysql_process.py index d1ccedb2c5d..2dd092c0889 100644 --- a/util/jenkins/list_mysql_process/list_mysql_process.py +++ b/util/jenkins/list_mysql_process/list_mysql_process.py @@ -26,7 +26,7 @@ def describe_db_instances(self): return self.client.describe_db_instances() -def rds_extractor(environment, regioninclude): +def rds_extractor(environment, whitelistregions): """ Return list of all RDS instances across all the regions Returns: @@ -45,8 +45,8 @@ def rds_extractor(environment, regioninclude): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) - if regioninclude: - regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in regioninclude]} + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} for region in regions_list["Regions"]: try: client = RDSBotoWrapper(region_name=region["RegionName"]) @@ -118,8 +118,8 @@ def check_queries_running(rds_list, username, password): @click.option('--password', envvar='PASSWORD', required=True) @click.option('--environment', required=True, help='Use to identify the environment') @click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') -@click.option('--regioninclude', '-r', multiple=True, help='Regions to check, can be specified multiple times') -def controller(username, password, environment, rdsignore, regioninclude): +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def controller(username, password, environment, rdsignore, whitelistregions): """ Control execution of all other functions Arguments: @@ -132,7 +132,7 @@ def controller(username, password, environment, rdsignore, regioninclude): environment (str): Get this from cli args """ - rds_list = rds_extractor(environment, regioninclude) + rds_list = rds_extractor(environment, whitelistregions) filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) process_list = check_queries_running(filtered_rds_list, username, password) if len(process_list) > 0: From 279d5c9836383665fa34c45789a522bcb6faf476 Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 12 Dec 2023 15:38:00 +0500 Subject: [PATCH 591/664] chore: Update slow logs script to run for whitelisted regions --- .../export_slow_query_logs.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/util/jenkins/export_slow_logs/export_slow_query_logs.py b/util/jenkins/export_slow_logs/export_slow_query_logs.py index fa26b72b86a..9cb8813dd9b 100644 --- a/util/jenkins/export_slow_logs/export_slow_query_logs.py +++ b/util/jenkins/export_slow_logs/export_slow_query_logs.py @@ -45,7 +45,7 @@ def describe_db_instances(self): return self.client.describe_db_instances() -def rds_extractor(environment): +def rds_extractor(environment, whitelistregions): """ Return list of all RDS instances across all the regions Returns: @@ -64,19 +64,25 @@ def rds_extractor(environment): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} for region in regions_list["Regions"]: - client = RDSBotoWrapper(region_name=region["RegionName"]) - response = client.describe_db_instances() - for instance in response.get('DBInstances'): - if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: - temp_dict = {} - temp_dict["name"] = instance["DBInstanceIdentifier"] - temp_dict["ARN"] = instance["DBInstanceArn"] - temp_dict["Region"] = region["RegionName"] - temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") - temp_dict["Username"] = instance.get("MasterUsername") - temp_dict["Port"] = instance.get("Port") - rds_list.append(temp_dict) + try: + client = RDSBotoWrapper(region_name=region["RegionName"]) + response = client.describe_db_instances() + for instance in response.get('DBInstances'): + if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: + temp_dict = {} + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["ARN"] = instance["DBInstanceArn"] + temp_dict["Region"] = region["RegionName"] + temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") + temp_dict["Username"] = instance.get("MasterUsername") + temp_dict["Port"] = instance.get("Port") + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) return rds_list @@ -137,8 +143,9 @@ def rds_controller(rds_list, username, password): @click.option('--password', envvar='PASSWORD', required=True) @click.option('--environment', required=True, help='Use to identify the environment') @click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') -def main(username, password, environment, rdsignore): - rds_list = rds_extractor(environment) +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def main(username, password, environment, rdsignore, whitelistregions): + rds_list = rds_extractor(environment, whitelistregions) filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) rds_controller(filtered_rds_list, username, password) From 853a7cb8de66373c85d59a0e484ded0549e316af Mon Sep 17 00:00:00 2001 From: syedimranhassan Date: Tue, 12 Dec 2023 16:28:08 +0500 Subject: [PATCH 592/664] chore: Update RDS alarms script to run for whitelisted regions --- .../rds_alarms_checker/missing_rds_alarms.py | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/util/jenkins/rds_alarms_checker/missing_rds_alarms.py b/util/jenkins/rds_alarms_checker/missing_rds_alarms.py index e7518d6c22d..151a50183d5 100644 --- a/util/jenkins/rds_alarms_checker/missing_rds_alarms.py +++ b/util/jenkins/rds_alarms_checker/missing_rds_alarms.py @@ -34,7 +34,7 @@ def describe_alarms(self, **kwargs): return self.client.describe_alarms(**kwargs) -def rds_extractor(): +def rds_extractor(whitelistregions): """ Return list of all RDS instances across all the regions Returns: @@ -53,16 +53,22 @@ def rds_extractor(): except ClientError as e: print(f"Unable to connect to AWS with error :{e}") sys.exit(1) + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} for region in regions_list["Regions"]: - client = RDSBotoWrapper(region_name=region["RegionName"]) - response = client.describe_db_instances() - for instance in response.get('DBInstances'): - if "test" not in instance["DBInstanceIdentifier"]: - temp_dict = {} - temp_dict["name"] = instance["DBInstanceIdentifier"] - temp_dict["ARN"] = instance["DBInstanceArn"] - temp_dict["Region"] = region["RegionName"] - rds_list.append(temp_dict) + try: + client = RDSBotoWrapper(region_name=region["RegionName"]) + response = client.describe_db_instances() + for instance in response.get('DBInstances'): + if "test" not in instance["DBInstanceIdentifier"]: + temp_dict = {} + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["ARN"] = instance["DBInstanceArn"] + temp_dict["Region"] = region["RegionName"] + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) return rds_list @@ -79,11 +85,12 @@ def cloudwatch_alarm_checker(alarmprefix, region): @click.command() @click.option('--ignore', type=(str), multiple=True, help='RDS Instances to ignore') -def controller(ignore): +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def controller(ignore, whitelistregions): """ Control execution of all other functions """ - rds = rds_extractor() + rds = rds_extractor(whitelistregions) missing_alarm = [] # List of RDS we don't care about ignore_rds_list = list(ignore) From ae749b176fa997e4883152bb5b7ed917db30eb91 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 17 Dec 2023 20:39:31 -0500 Subject: [PATCH 593/664] chore: Updating Python Requirements --- .../roles/aws/templates/requirements.txt.j2 | 12 +++--- requirements.txt | 18 ++++----- requirements/common_constraints.txt | 4 -- requirements/pip-tools.txt | 4 +- requirements/pip.txt | 6 +-- util/jenkins/requirements-cloudflare.txt | 8 ++-- util/jenkins/requirements.txt | 40 +++++++++---------- util/pingdom/requirements.txt | 8 ++-- util/vpc-tools/requirements.txt | 8 ++-- 9 files changed, 52 insertions(+), 56 deletions(-) diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 17add96d6db..2b3b3b0edda 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.29.70 +awscli==1.32.2 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.28.70 +boto3==1.34.2 # via -r requirements/aws.in -botocore==1.31.70 +botocore==1.34.2 # via # awscli # boto3 @@ -23,7 +23,7 @@ jmespath==1.0.1 # via # boto3 # botocore -pyasn1==0.5.0 +pyasn1==0.5.1 # via rsa python-dateutil==2.8.2 # via @@ -37,9 +37,9 @@ pyyaml==5.3.1 # awscli rsa==4.7.2 # via awscli -s3cmd==2.3.0 +s3cmd==2.4.0 # via -r requirements/aws.in -s3transfer==0.7.0 +s3transfer==0.9.0 # via # awscli # boto3 diff --git a/requirements.txt b/requirements.txt index b7b41d6ea6f..5293f81318b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ ansible==2.10.7 # via -r requirements/base.in ansible-base==2.10.17 # via ansible -awscli==1.29.70 +awscli==1.32.2 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -16,25 +16,25 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.28.70 +boto3==1.34.2 # via -r requirements/base.in -botocore==1.31.70 +botocore==1.34.2 # via # awscli # boto3 # s3transfer -certifi==2023.7.22 +certifi==2023.11.17 # via requests cffi==1.16.0 # via # bcrypt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests colorama==0.4.4 # via awscli -cryptography==41.0.5 +cryptography==41.0.7 # via # ansible-base # paramiko @@ -50,7 +50,7 @@ docutils==0.16 # via awscli ecdsa==0.13.3 # via -r requirements/base.in -idna==3.4 +idna==3.6 # via requests jinja2==2.8 # via @@ -76,7 +76,7 @@ pathlib2==2.3.0 # via -r requirements/base.in prettytable==0.7.2 # via -r requirements/base.in -pyasn1==0.5.0 +pyasn1==0.5.1 # via # paramiko # rsa @@ -101,7 +101,7 @@ requests==2.31.0 # datadog rsa==4.7.2 # via awscli -s3transfer==0.7.0 +s3transfer==0.9.0 # via # awscli # boto3 diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index afe6aa887fe..15aafb293da 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -21,7 +21,3 @@ elasticsearch<7.14.0 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected django-simple-history==3.0.0 - -# tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. -# Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 -tox<4.0.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 50d35f22e8d..93a9cee28cb 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via build packaging==23.2 # via build @@ -21,7 +21,7 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.2 +wheel==0.42.0 # via pip-tools zipp==3.17.0 # via importlib-metadata diff --git a/requirements/pip.txt b/requirements/pip.txt index 0c788d61e50..d798b87b36c 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.41.2 +wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3.1 +pip==23.3.2 # via -r requirements/pip.in -setuptools==68.2.2 +setuptools==69.0.2 # via -r requirements/pip.in diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index 2c7f48bb9d7..9419d9fd3d1 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,15 +4,15 @@ # # make upgrade # -certifi==2023.7.22 +certifi==2023.11.17 # via requests -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/cloudflare.in -idna==3.4 +idna==3.6 # via requests requests==2.31.0 # via -r requirements/cloudflare.in -urllib3==2.0.7 +urllib3==2.1.0 # via requests diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 1f91a34b58d..4ddf612445b 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -4,11 +4,11 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via kombu -argcomplete==3.1.2 +argcomplete==3.2.1 # via yq -awscli==1.29.70 +awscli==1.32.2 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,24 +16,24 @@ backports-zoneinfo[tzdata]==0.2.1 # via # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.28.70 +boto3==1.34.2 # via -r requirements/jenkins.in -botocore==1.31.70 +botocore==1.34.2 # via # awscli # boto3 # s3transfer -celery==5.3.4 +celery==5.3.6 # via -r requirements/jenkins.in -certifi==2023.7.22 +certifi==2023.11.17 # via # opsgenie-sdk # requests -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -52,7 +52,7 @@ colorama==0.4.4 # via awscli docutils==0.16 # via awscli -idna==3.4 +idna==3.6 # via requests jmespath==1.0.1 # via @@ -60,13 +60,13 @@ jmespath==1.0.1 # botocore jq==1.6.0 # via -r requirements/jenkins.in -kombu==5.3.2 +kombu==5.3.4 # via celery opsgenie-sdk==0.3.1 # via -r requirements/jenkins.in -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.43 # via click-repl -pyasn1==0.5.0 +pyasn1==0.5.1 # via rsa pymysql==0.9.3 # via -r requirements/jenkins.in @@ -76,7 +76,7 @@ python-dateutil==2.8.2 # celery # opsgenie-sdk # s3cmd -python-gnupg==0.5.1 +python-gnupg==0.5.2 # via -r requirements/jenkins.in python-magic==0.4.27 # via s3cmd @@ -93,9 +93,9 @@ requests==2.31.0 # via opsgenie-sdk rsa==4.7.2 # via awscli -s3cmd==2.3.0 +s3cmd==2.4.0 # via -r requirements/jenkins.in -s3transfer==0.7.0 +s3transfer==0.9.0 # via # awscli # boto3 @@ -105,9 +105,9 @@ six==1.16.0 # python-dateutil splunk-sdk==1.6.16 # via -r requirements/jenkins.in -tomlkit==0.12.1 +tomlkit==0.12.3 # via yq -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via kombu tzdata==2023.3 # via @@ -118,12 +118,12 @@ urllib3==1.26.18 # botocore # opsgenie-sdk # requests -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.8 +wcwidth==0.2.12 # via prompt-toolkit xmltodict==0.13.0 # via yq diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 84f610ad228..3e9b2a4bbba 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,13 +4,13 @@ # # make upgrade # -certifi==2023.7.22 +certifi==2023.11.17 # via requests -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests click==6.7 # via -r requirements/pingdom.in -idna==3.4 +idna==3.6 # via requests pyyaml==6.0.1 # via -r requirements/pingdom.in @@ -18,5 +18,5 @@ requests==2.31.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in -urllib3==2.0.7 +urllib3==2.1.0 # via requests diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index c965b4fce70..a966598293d 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,15 +6,15 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2023.7.22 +certifi==2023.11.17 # via requests -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests docopt==0.6.2 # via -r requirements/vpc-tools.in -idna==3.4 +idna==3.6 # via requests requests==2.31.0 # via -r requirements/vpc-tools.in -urllib3==2.0.7 +urllib3==2.1.0 # via requests From 431235c2fe03c4d0ec0f844be10f289f947abb14 Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Mon, 18 Dec 2023 16:13:25 +0500 Subject: [PATCH 594/664] chore: remove key tasks from mysql8 installation --- playbooks/roles/mysql/tasks/mysql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mysql/tasks/mysql.yml b/playbooks/roles/mysql/tasks/mysql.yml index 13b5b4732aa..b1ec751d242 100644 --- a/playbooks/roles/mysql/tasks/mysql.yml +++ b/playbooks/roles/mysql/tasks/mysql.yml @@ -59,7 +59,7 @@ apt_key: keyserver: "{{ MYSQL_APT_KEYSERVER }}" id: "{{ MYSQL_APT_KEY }}" - when: ansible_distribution_release == 'focal' + when: ansible_distribution_release == 'focal' and not mysql_8_0_install - name: add the mysql-5.7 repo to the sources list apt_repository: From eefd61c859e31a66bd825ac381bdd7a2fac8b629 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Fri, 10 Dec 2021 10:52:56 -0500 Subject: [PATCH 595/664] fix: add retries to jdk download this endpoint evidently sometimes throws spurious 403s, so bandaid --- playbooks/roles/oraclejdk/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/roles/oraclejdk/tasks/main.yml b/playbooks/roles/oraclejdk/tasks/main.yml index 2ef46eb0106..0749a483d87 100644 --- a/playbooks/roles/oraclejdk/tasks/main.yml +++ b/playbooks/roles/oraclejdk/tasks/main.yml @@ -22,6 +22,10 @@ headers: Cookie: oraclelicense=accept-securebackup-cookie dest: "/var/tmp/{{ oraclejdk_file }}" + retries: 3 + delay: 10 + register: oracle_jdk_download_retry + until: oracle_jdk_download_retry is succeeded - name: Create jvm dir file: From 9f0271e3e7d34cc2509cbbe782b35edabdbaeba0 Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Tue, 19 Dec 2023 14:19:59 -0500 Subject: [PATCH 596/664] fix: update reviewing team for requirements upgrades --- .github/workflows/upgrade-python-requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml index fcdad5c826e..e951fc09388 100644 --- a/.github/workflows/upgrade-python-requirements.yml +++ b/.github/workflows/upgrade-python-requirements.yml @@ -13,7 +13,7 @@ jobs: call-upgrade-python-requirements-workflow: with: branch: ${{ github.event.inputs.branch }} - team_reviewers: "devops" + team_reviewers: "2u-sre" email_address: tools-jenkins-isre@2u-internal.opsgenie.net send_success_notification: false secrets: From 6050cd18d9505ae0128be595df466040b619b49c Mon Sep 17 00:00:00 2001 From: Matt Hughes Date: Wed, 20 Dec 2023 15:30:22 -0500 Subject: [PATCH 597/664] fix: failure trying to create specific kind of user via ansible specifically when running the create db and users runbook which is used to set up users for mysql databases as documented here: https://2u-internal.atlassian.net/wiki/spaces/SRE/pages/19252888/How+to+Create+and+Update+Databases+and+Database+Users --- playbooks/create_db_and_users.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/create_db_and_users.yml b/playbooks/create_db_and_users.yml index 5324f5048c0..58001c9a052 100644 --- a/playbooks/create_db_and_users.yml +++ b/playbooks/create_db_and_users.yml @@ -102,7 +102,7 @@ - name: assign privileges to AWS RDS IAM users shell: | - mysql -u"{{ item.login_user }}" -p"{{ item.login_password }}" -h"{{ item.login_host }}" -e "GRANT {{ item.privileges }} to '{{ item.name }}'@'{{ item.host }}'REQUIRE SSL" + mysql -u"{{ item.login_user }}" -p"{{ item.login_password }}" -h"{{ item.login_host }}" -e "GRANT {{ item.privileges }} to '{{ item.name }}'@'{{ item.host }}'" when: item.mysql_plugin is defined and item.state == 'present' and item.mysql_plugin == 'AWSAuthenticationPlugin' with_items: "{{ database_users }}" tags: From 6b4591a1f5c5efbfbcbbe927ce6949f08e297015 Mon Sep 17 00:00:00 2001 From: katebygrace Date: Fri, 5 Jan 2024 15:22:38 -0500 Subject: [PATCH 598/664] chore: move sysadmin JIRA:CLOUDSEC-12 --- jenkins/extend-sandbox-termination.py | 89 ++++++++ jenkins/requirements/base.in | 10 + jenkins/requirements/base.txt | 56 +++++ jenkins/requirements/constraints.txt | 12 ++ jenkins/requirements/dev.in | 10 + jenkins/requirements/dev.txt | 295 ++++++++++++++++++++++++++ jenkins/requirements/doc.in | 9 + jenkins/requirements/doc.txt | 212 ++++++++++++++++++ jenkins/requirements/pip-tools.in | 4 + jenkins/requirements/pip-tools.txt | 20 ++ jenkins/requirements/private.readme | 15 ++ jenkins/requirements/quality.in | 9 + jenkins/requirements/quality.txt | 205 ++++++++++++++++++ jenkins/requirements/test.in | 8 + jenkins/requirements/test.txt | 119 +++++++++++ jenkins/requirements/travis.in | 4 + jenkins/requirements/travis.txt | 20 ++ jenkins/saml-ssl-expiration-check.py | 40 ++++ jenkins/ssl-expiration-check.py | 106 +++++++++ 19 files changed, 1243 insertions(+) create mode 100644 jenkins/extend-sandbox-termination.py create mode 100644 jenkins/requirements/base.in create mode 100644 jenkins/requirements/base.txt create mode 100644 jenkins/requirements/constraints.txt create mode 100644 jenkins/requirements/dev.in create mode 100644 jenkins/requirements/dev.txt create mode 100644 jenkins/requirements/doc.in create mode 100644 jenkins/requirements/doc.txt create mode 100644 jenkins/requirements/pip-tools.in create mode 100644 jenkins/requirements/pip-tools.txt create mode 100644 jenkins/requirements/private.readme create mode 100644 jenkins/requirements/quality.in create mode 100644 jenkins/requirements/quality.txt create mode 100644 jenkins/requirements/test.in create mode 100644 jenkins/requirements/test.txt create mode 100644 jenkins/requirements/travis.in create mode 100644 jenkins/requirements/travis.txt create mode 100644 jenkins/saml-ssl-expiration-check.py create mode 100644 jenkins/ssl-expiration-check.py diff --git a/jenkins/extend-sandbox-termination.py b/jenkins/extend-sandbox-termination.py new file mode 100644 index 00000000000..6a5740970c9 --- /dev/null +++ b/jenkins/extend-sandbox-termination.py @@ -0,0 +1,89 @@ +__author__ = 'arbab' +''' +This script will be used to modify/extend the termination date on the sandbox. +''' +import boto +from datetime import datetime +from datetime import timedelta +import logging +import argparse + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description="Modify/extend the termination date on the sandbox.") + + parser.add_argument('-n', '--noop', action='store_true', + help="don't actually run the commands", default=False) + + parser.add_argument('-p', '--profile', default=None, + help="AWS profile to use when connecting.") + + extend_group = parser.add_mutually_exclusive_group(required=True) + + extend_group.add_argument('-d', '--day', default=None, + help="number of days", type=int) + + extend_group.add_argument('-a', '--always', default=False, + help="Do not terminate this Sandbox") + + group = parser.add_mutually_exclusive_group(required=True) + + group.add_argument('-u', '--username', default=None, + help="GitHub username") + + group.add_argument('-c', '--custom', default=None, + help="Custom name, if the sandbox was not created with the default options") + + group.add_argument('-i', '--instance-id', default=None, + help="Sandbox Instance ID") + + args = parser.parse_args() + + ec2 = boto.connect_ec2(profile_name=args.profile) + + days_to_increase = args.day + + if args.username: + sandbox_name = args.username + '-sandbox' + reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name}) + if args.custom: + sandbox_name = args.custom + reservations = ec2.get_all_instances(filters={"tag:Name": sandbox_name}) + if args.instance_id: + instance_id = args.instance_id + reservations = ec2.get_all_instances(instance_ids=[instance_id]) + + instance = reservations[0].instances[0] + + if args.noop: + logger.info("Sandbox ID:{} with Name: {} and Owner: {} will extend by {} days".format( + instance.id, + instance.tags['Name'], + instance.tags['owner'], + days_to_increase + ) + ) + elif args.always: + instance.add_tag('do_not_terminate', 'true') + logger.info("Sandbox ID:{} with Name: {} and Owner: {} will not be terminate".format( + instance.id, + instance.tags['Name'], + instance.tags['owner'], + ) + ) + else: + # modified the terminate time + terminate_time = datetime.strptime(str(instance.tags['instance_termination_time']), "%m-%d-%Y %H:%M:%S") + terminate_time = terminate_time + timedelta(days=days_to_increase) + instance.add_tag('instance_termination_time', terminate_time.strftime("%m-%d-%Y %H:%M:%S")) + logger.info("Sandbox ID:{} with Name: {} and Owner: {} has been extended by {} days".format( + instance.id, + instance.tags['Name'], + instance.tags['owner'], + days_to_increase + ) + ) \ No newline at end of file diff --git a/jenkins/requirements/base.in b/jenkins/requirements/base.in new file mode 100644 index 00000000000..5f7f34712b2 --- /dev/null +++ b/jenkins/requirements/base.in @@ -0,0 +1,10 @@ +# Core requirements for using this application +-c constraints.txt +boto +boto3 +python-gnupg +pyyaml +pymysql +pymongo +requests +pyOpenSSL diff --git a/jenkins/requirements/base.txt b/jenkins/requirements/base.txt new file mode 100644 index 00000000000..a5f3f34912c --- /dev/null +++ b/jenkins/requirements/base.txt @@ -0,0 +1,56 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +boto==2.49.0 + # via -r requirements/base.in +boto3==1.20.26 + # via -r requirements/base.in +botocore==1.23.26 + # via + # boto3 + # s3transfer +certifi==2021.10.8 + # via requests +cffi==1.15.0 + # via cryptography +charset-normalizer==2.0.9 + # via requests +cryptography==3.4.8 + # via + # -c requirements/constraints.txt + # pyopenssl +idna==3.3 + # via requests +jmespath==0.10.0 + # via + # boto3 + # botocore +pycparser==2.21 + # via cffi +pymongo==4.0.1 + # via -r requirements/base.in +pymysql==1.0.2 + # via -r requirements/base.in +pyopenssl==21.0.0 + # via -r requirements/base.in +python-dateutil==2.8.2 + # via botocore +python-gnupg==0.4.8 + # via -r requirements/base.in +pyyaml==6.0 + # via -r requirements/base.in +requests==2.26.0 + # via -r requirements/base.in +s3transfer==0.5.0 + # via boto3 +six==1.16.0 + # via + # pyopenssl + # python-dateutil +urllib3==1.26.7 + # via + # botocore + # requests diff --git a/jenkins/requirements/constraints.txt b/jenkins/requirements/constraints.txt new file mode 100644 index 00000000000..4386c3e6d9d --- /dev/null +++ b/jenkins/requirements/constraints.txt @@ -0,0 +1,12 @@ +# Version constraints for pip-installation. +# +# This file doesn't install any packages. It specifies version constraints +# that will be applied if a package is needed. +# +# When pinning something here, please provide an explanation of why. Ideally, +# link to other information that will help people in the future to remove the +# pin when possible. Writing an issue against the offending project and +# linking to it here is good. + +# Pin cryptography because it broke saml-ssl-expiration-check.py, unpin when fixed +cryptography<=3.4.8 diff --git a/jenkins/requirements/dev.in b/jenkins/requirements/dev.in new file mode 100644 index 00000000000..2e8c93c698b --- /dev/null +++ b/jenkins/requirements/dev.in @@ -0,0 +1,10 @@ +# Additional requirements for development of this application +-c constraints.txt + +-r pip-tools.txt # pip-tools and its dependencies, for managing requirements files +-r quality.txt # Core and quality check dependencies +-r travis.txt # tox and related dependencies + +diff-cover # Changeset diff test coverage +edx-i18n-tools # For i18n_tool dummy +tox-battery # Makes tox aware of requirements file changes diff --git a/jenkins/requirements/dev.txt b/jenkins/requirements/dev.txt new file mode 100644 index 00000000000..dc5768fbabf --- /dev/null +++ b/jenkins/requirements/dev.txt @@ -0,0 +1,295 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +asgiref==3.4.1 + # via django +astroid==2.9.0 + # via + # -r requirements/quality.txt + # pylint + # pylint-celery +attrs==21.2.0 + # via + # -r requirements/quality.txt + # pytest +backports.entry-points-selectable==1.1.1 + # via virtualenv +boto==2.49.0 + # via -r requirements/quality.txt +boto3==1.20.26 + # via -r requirements/quality.txt +botocore==1.23.26 + # via + # -r requirements/quality.txt + # boto3 + # s3transfer +certifi==2021.10.8 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # requests +cffi==1.15.0 + # via + # -r requirements/quality.txt + # cryptography +chardet==4.0.0 + # via diff-cover +charset-normalizer==2.0.9 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # requests +click==8.0.3 + # via + # -r requirements/pip-tools.txt + # -r requirements/quality.txt + # click-log + # code-annotations + # edx-lint + # pip-tools +click-log==0.3.2 + # via + # -r requirements/quality.txt + # edx-lint +code-annotations==1.2.0 + # via + # -r requirements/quality.txt + # edx-lint +codecov==2.1.12 + # via -r requirements/travis.txt +coverage[toml]==6.2 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # codecov + # pytest-cov +cryptography==3.4.8 + # via + # -c requirements/constraints.txt + # -r requirements/quality.txt + # pyopenssl +diff-cover==6.4.4 + # via -r requirements/dev.in +distlib==0.3.4 + # via virtualenv +django==3.2.10 + # via edx-i18n-tools +edx-i18n-tools==0.8.1 + # via -r requirements/dev.in +edx-lint==5.2.1 + # via -r requirements/quality.txt +filelock==3.4.0 + # via + # tox + # virtualenv +idna==3.3 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # requests +iniconfig==1.1.1 + # via + # -r requirements/quality.txt + # pytest +isort==5.10.1 + # via + # -r requirements/quality.txt + # pylint +jinja2==3.0.3 + # via + # -r requirements/quality.txt + # code-annotations + # diff-cover +jmespath==0.10.0 + # via + # -r requirements/quality.txt + # boto3 + # botocore +lazy-object-proxy==1.7.1 + # via + # -r requirements/quality.txt + # astroid +markupsafe==2.0.1 + # via + # -r requirements/quality.txt + # jinja2 +mccabe==0.6.1 + # via + # -r requirements/quality.txt + # pylint +packaging==21.3 + # via + # -r requirements/quality.txt + # pytest + # tox +path==16.2.0 + # via edx-i18n-tools +pbr==5.8.0 + # via + # -r requirements/quality.txt + # stevedore +pep517==0.12.0 + # via + # -r requirements/pip-tools.txt + # pip-tools +pip-tools==6.4.0 + # via -r requirements/pip-tools.txt +platformdirs==2.4.0 + # via + # -r requirements/quality.txt + # pylint + # virtualenv +pluggy==1.0.0 + # via + # -r requirements/quality.txt + # diff-cover + # pytest + # tox +polib==1.1.1 + # via edx-i18n-tools +py==1.11.0 + # via + # -r requirements/quality.txt + # pytest + # tox +pycodestyle==2.8.0 + # via -r requirements/quality.txt +pycparser==2.21 + # via + # -r requirements/quality.txt + # cffi +pydocstyle==6.1.1 + # via -r requirements/quality.txt +pygments==2.10.0 + # via diff-cover +pylint==2.12.2 + # via + # -r requirements/quality.txt + # edx-lint + # pylint-celery + # pylint-django + # pylint-plugin-utils +pylint-celery==0.3 + # via + # -r requirements/quality.txt + # edx-lint +pylint-django==2.4.4 + # via + # -r requirements/quality.txt + # edx-lint +pylint-plugin-utils==0.6 + # via + # -r requirements/quality.txt + # pylint-celery + # pylint-django +pymongo==4.0.1 + # via -r requirements/quality.txt +pymysql==1.0.2 + # via -r requirements/quality.txt +pyopenssl==21.0.0 + # via -r requirements/quality.txt +pyparsing==3.0.6 + # via + # -r requirements/quality.txt + # packaging +pytest==6.2.5 + # via + # -r requirements/quality.txt + # pytest-cov + # pytest-django +pytest-cov==3.0.0 + # via -r requirements/quality.txt +pytest-django==4.5.2 + # via -r requirements/quality.txt +python-dateutil==2.8.2 + # via + # -r requirements/quality.txt + # botocore +python-gnupg==0.4.8 + # via -r requirements/quality.txt +python-slugify==5.0.2 + # via + # -r requirements/quality.txt + # code-annotations +pytz==2021.3 + # via django +pyyaml==6.0 + # via + # -r requirements/quality.txt + # code-annotations + # edx-i18n-tools +requests==2.26.0 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # codecov +s3transfer==0.5.0 + # via + # -r requirements/quality.txt + # boto3 +six==1.16.0 + # via + # -r requirements/quality.txt + # edx-lint + # pyopenssl + # python-dateutil + # tox + # virtualenv +snowballstemmer==2.2.0 + # via + # -r requirements/quality.txt + # pydocstyle +sqlparse==0.4.2 + # via django +stevedore==3.5.0 + # via + # -r requirements/quality.txt + # code-annotations +text-unidecode==1.3 + # via + # -r requirements/quality.txt + # python-slugify +toml==0.10.2 + # via + # -r requirements/quality.txt + # pylint + # pytest + # tox +tomli==2.0.0 + # via + # -r requirements/pip-tools.txt + # -r requirements/quality.txt + # coverage + # pep517 +tox==3.24.4 + # via tox-battery +tox-battery==0.6.1 + # via -r requirements/dev.in +typing-extensions==4.0.1 + # via + # -r requirements/quality.txt + # astroid + # pylint +urllib3==1.26.7 + # via + # -r requirements/quality.txt + # -r requirements/travis.txt + # botocore + # requests +virtualenv==20.10.0 + # via tox +wheel==0.37.1 + # via + # -r requirements/pip-tools.txt + # pip-tools +wrapt==1.13.3 + # via + # -r requirements/quality.txt + # astroid + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/jenkins/requirements/doc.in b/jenkins/requirements/doc.in new file mode 100644 index 00000000000..690e8e1d5b1 --- /dev/null +++ b/jenkins/requirements/doc.in @@ -0,0 +1,9 @@ +# Requirements for documentation validation +-c constraints.txt + +-r test.txt # Core and testing dependencies for this package + +doc8 # reStructuredText style checker +edx_sphinx_theme # edX theme for Sphinx output +readme_renderer # Validates README.rst for usage on PyPI +Sphinx # Documentation builder diff --git a/jenkins/requirements/doc.txt b/jenkins/requirements/doc.txt new file mode 100644 index 00000000000..2942cd63331 --- /dev/null +++ b/jenkins/requirements/doc.txt @@ -0,0 +1,212 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +alabaster==0.7.12 + # via sphinx +attrs==21.2.0 + # via + # -r requirements/test.txt + # pytest +babel==2.9.1 + # via sphinx +bleach==4.1.0 + # via readme-renderer +boto==2.49.0 + # via -r requirements/test.txt +boto3==1.20.26 + # via -r requirements/test.txt +botocore==1.23.26 + # via + # -r requirements/test.txt + # boto3 + # s3transfer +certifi==2021.10.8 + # via + # -r requirements/test.txt + # requests +cffi==1.15.0 + # via + # -r requirements/test.txt + # cryptography +charset-normalizer==2.0.9 + # via + # -r requirements/test.txt + # requests +click==8.0.3 + # via + # -r requirements/test.txt + # code-annotations +code-annotations==1.2.0 + # via -r requirements/test.txt +coverage[toml]==6.2 + # via + # -r requirements/test.txt + # pytest-cov +cryptography==3.4.8 + # via + # -c requirements/constraints.txt + # -r requirements/test.txt + # pyopenssl +doc8==0.10.1 + # via -r requirements/doc.in +docutils==0.17.1 + # via + # doc8 + # readme-renderer + # restructuredtext-lint + # sphinx +edx-sphinx-theme==3.0.0 + # via -r requirements/doc.in +idna==3.3 + # via + # -r requirements/test.txt + # requests +imagesize==1.3.0 + # via sphinx +iniconfig==1.1.1 + # via + # -r requirements/test.txt + # pytest +jinja2==3.0.3 + # via + # -r requirements/test.txt + # code-annotations + # sphinx +jmespath==0.10.0 + # via + # -r requirements/test.txt + # boto3 + # botocore +markupsafe==2.0.1 + # via + # -r requirements/test.txt + # jinja2 +packaging==21.3 + # via + # -r requirements/test.txt + # bleach + # pytest + # sphinx +pbr==5.8.0 + # via + # -r requirements/test.txt + # stevedore +pluggy==1.0.0 + # via + # -r requirements/test.txt + # pytest +py==1.11.0 + # via + # -r requirements/test.txt + # pytest +pycparser==2.21 + # via + # -r requirements/test.txt + # cffi +pygments==2.10.0 + # via + # doc8 + # readme-renderer + # sphinx +pymongo==4.0.1 + # via -r requirements/test.txt +pymysql==1.0.2 + # via -r requirements/test.txt +pyopenssl==21.0.0 + # via -r requirements/test.txt +pyparsing==3.0.6 + # via + # -r requirements/test.txt + # packaging +pytest==6.2.5 + # via + # -r requirements/test.txt + # pytest-cov + # pytest-django +pytest-cov==3.0.0 + # via -r requirements/test.txt +pytest-django==4.5.2 + # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # botocore +python-gnupg==0.4.8 + # via -r requirements/test.txt +python-slugify==5.0.2 + # via + # -r requirements/test.txt + # code-annotations +pytz==2021.3 + # via babel +pyyaml==6.0 + # via + # -r requirements/test.txt + # code-annotations +readme-renderer==32.0 + # via -r requirements/doc.in +requests==2.26.0 + # via + # -r requirements/test.txt + # sphinx +restructuredtext-lint==1.3.2 + # via doc8 +s3transfer==0.5.0 + # via + # -r requirements/test.txt + # boto3 +six==1.16.0 + # via + # -r requirements/test.txt + # bleach + # edx-sphinx-theme + # pyopenssl + # python-dateutil +snowballstemmer==2.2.0 + # via sphinx +sphinx==4.3.2 + # via + # -r requirements/doc.in + # edx-sphinx-theme +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +stevedore==3.5.0 + # via + # -r requirements/test.txt + # code-annotations + # doc8 +text-unidecode==1.3 + # via + # -r requirements/test.txt + # python-slugify +toml==0.10.2 + # via + # -r requirements/test.txt + # pytest +tomli==2.0.0 + # via + # -r requirements/test.txt + # coverage +urllib3==1.26.7 + # via + # -r requirements/test.txt + # botocore + # requests +webencodings==0.5.1 + # via bleach + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/jenkins/requirements/pip-tools.in b/jenkins/requirements/pip-tools.in new file mode 100644 index 00000000000..3f1b64ae937 --- /dev/null +++ b/jenkins/requirements/pip-tools.in @@ -0,0 +1,4 @@ +# Just the dependencies to run pip-tools, mainly for the "upgrade" make target +-c constraints.txt + +pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/jenkins/requirements/pip-tools.txt b/jenkins/requirements/pip-tools.txt new file mode 100644 index 00000000000..f4b492113e3 --- /dev/null +++ b/jenkins/requirements/pip-tools.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +click==8.0.3 + # via pip-tools +pep517==0.12.0 + # via pip-tools +pip-tools==6.4.0 + # via -r requirements/pip-tools.in +tomli==2.0.0 + # via pep517 +wheel==0.37.1 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/jenkins/requirements/private.readme b/jenkins/requirements/private.readme new file mode 100644 index 00000000000..5600a1075bc --- /dev/null +++ b/jenkins/requirements/private.readme @@ -0,0 +1,15 @@ +# If there are any Python packages you want to keep in your virtualenv beyond +# those listed in the official requirements files, create a "private.in" file +# and list them there. Generate the corresponding "private.txt" file pinning +# all of their indirect dependencies to specific versions as follows: + +# pip-compile private.in + +# This allows you to use "pip-sync" without removing these packages: + +# pip-sync requirements/*.txt + +# "private.in" and "private.txt" aren't checked into git to avoid merge +# conflicts, and the presence of this file allows "private.*" to be +# included in scripted pip-sync usage without requiring that those files be +# created first. diff --git a/jenkins/requirements/quality.in b/jenkins/requirements/quality.in new file mode 100644 index 00000000000..0bd84a6468c --- /dev/null +++ b/jenkins/requirements/quality.in @@ -0,0 +1,9 @@ +# Requirements for code quality checks +-c constraints.txt + +-r test.txt # Core and testing dependencies for this package + +edx-lint # edX pylint rules and plugins +isort # to standardize order of imports +pycodestyle # PEP 8 compliance validation +pydocstyle # PEP 257 compliance validation diff --git a/jenkins/requirements/quality.txt b/jenkins/requirements/quality.txt new file mode 100644 index 00000000000..86f5ed5fab6 --- /dev/null +++ b/jenkins/requirements/quality.txt @@ -0,0 +1,205 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +astroid==2.9.0 + # via + # pylint + # pylint-celery +attrs==21.2.0 + # via + # -r requirements/test.txt + # pytest +boto==2.49.0 + # via -r requirements/test.txt +boto3==1.20.26 + # via -r requirements/test.txt +botocore==1.23.26 + # via + # -r requirements/test.txt + # boto3 + # s3transfer +certifi==2021.10.8 + # via + # -r requirements/test.txt + # requests +cffi==1.15.0 + # via + # -r requirements/test.txt + # cryptography +charset-normalizer==2.0.9 + # via + # -r requirements/test.txt + # requests +click==8.0.3 + # via + # -r requirements/test.txt + # click-log + # code-annotations + # edx-lint +click-log==0.3.2 + # via edx-lint +code-annotations==1.2.0 + # via + # -r requirements/test.txt + # edx-lint +coverage[toml]==6.2 + # via + # -r requirements/test.txt + # pytest-cov +cryptography==3.4.8 + # via + # -c requirements/constraints.txt + # -r requirements/test.txt + # pyopenssl +edx-lint==5.2.1 + # via -r requirements/quality.in +idna==3.3 + # via + # -r requirements/test.txt + # requests +iniconfig==1.1.1 + # via + # -r requirements/test.txt + # pytest +isort==5.10.1 + # via + # -r requirements/quality.in + # pylint +jinja2==3.0.3 + # via + # -r requirements/test.txt + # code-annotations +jmespath==0.10.0 + # via + # -r requirements/test.txt + # boto3 + # botocore +lazy-object-proxy==1.7.1 + # via astroid +markupsafe==2.0.1 + # via + # -r requirements/test.txt + # jinja2 +mccabe==0.6.1 + # via pylint +packaging==21.3 + # via + # -r requirements/test.txt + # pytest +pbr==5.8.0 + # via + # -r requirements/test.txt + # stevedore +platformdirs==2.4.0 + # via pylint +pluggy==1.0.0 + # via + # -r requirements/test.txt + # pytest +py==1.11.0 + # via + # -r requirements/test.txt + # pytest +pycodestyle==2.8.0 + # via -r requirements/quality.in +pycparser==2.21 + # via + # -r requirements/test.txt + # cffi +pydocstyle==6.1.1 + # via -r requirements/quality.in +pylint==2.12.2 + # via + # edx-lint + # pylint-celery + # pylint-django + # pylint-plugin-utils +pylint-celery==0.3 + # via edx-lint +pylint-django==2.4.4 + # via edx-lint +pylint-plugin-utils==0.6 + # via + # pylint-celery + # pylint-django +pymongo==4.0.1 + # via -r requirements/test.txt +pymysql==1.0.2 + # via -r requirements/test.txt +pyopenssl==21.0.0 + # via -r requirements/test.txt +pyparsing==3.0.6 + # via + # -r requirements/test.txt + # packaging +pytest==6.2.5 + # via + # -r requirements/test.txt + # pytest-cov + # pytest-django +pytest-cov==3.0.0 + # via -r requirements/test.txt +pytest-django==4.5.2 + # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # botocore +python-gnupg==0.4.8 + # via -r requirements/test.txt +python-slugify==5.0.2 + # via + # -r requirements/test.txt + # code-annotations +pyyaml==6.0 + # via + # -r requirements/test.txt + # code-annotations +requests==2.26.0 + # via -r requirements/test.txt +s3transfer==0.5.0 + # via + # -r requirements/test.txt + # boto3 +six==1.16.0 + # via + # -r requirements/test.txt + # edx-lint + # pyopenssl + # python-dateutil +snowballstemmer==2.2.0 + # via pydocstyle +stevedore==3.5.0 + # via + # -r requirements/test.txt + # code-annotations +text-unidecode==1.3 + # via + # -r requirements/test.txt + # python-slugify +toml==0.10.2 + # via + # -r requirements/test.txt + # pylint + # pytest +tomli==2.0.0 + # via + # -r requirements/test.txt + # coverage +typing-extensions==4.0.1 + # via + # astroid + # pylint +urllib3==1.26.7 + # via + # -r requirements/test.txt + # botocore + # requests +wrapt==1.13.3 + # via astroid + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/jenkins/requirements/test.in b/jenkins/requirements/test.in new file mode 100644 index 00000000000..6797160bffa --- /dev/null +++ b/jenkins/requirements/test.in @@ -0,0 +1,8 @@ +# Requirements for test runs. +-c constraints.txt + +-r base.txt # Core dependencies for this package + +pytest-cov # pytest extension for code coverage statistics +pytest-django # pytest extension for better Django support +code-annotations # provides commands used by the pii_check make target. diff --git a/jenkins/requirements/test.txt b/jenkins/requirements/test.txt new file mode 100644 index 00000000000..5838cfa4633 --- /dev/null +++ b/jenkins/requirements/test.txt @@ -0,0 +1,119 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +attrs==21.2.0 + # via pytest +boto==2.49.0 + # via -r requirements/base.txt +boto3==1.20.26 + # via -r requirements/base.txt +botocore==1.23.26 + # via + # -r requirements/base.txt + # boto3 + # s3transfer +certifi==2021.10.8 + # via + # -r requirements/base.txt + # requests +cffi==1.15.0 + # via + # -r requirements/base.txt + # cryptography +charset-normalizer==2.0.9 + # via + # -r requirements/base.txt + # requests +click==8.0.3 + # via code-annotations +code-annotations==1.2.0 + # via -r requirements/test.in +coverage[toml]==6.2 + # via pytest-cov +cryptography==3.4.8 + # via + # -c requirements/constraints.txt + # -r requirements/base.txt + # pyopenssl +idna==3.3 + # via + # -r requirements/base.txt + # requests +iniconfig==1.1.1 + # via pytest +jinja2==3.0.3 + # via code-annotations +jmespath==0.10.0 + # via + # -r requirements/base.txt + # boto3 + # botocore +markupsafe==2.0.1 + # via jinja2 +packaging==21.3 + # via pytest +pbr==5.8.0 + # via stevedore +pluggy==1.0.0 + # via pytest +py==1.11.0 + # via pytest +pycparser==2.21 + # via + # -r requirements/base.txt + # cffi +pymongo==4.0.1 + # via -r requirements/base.txt +pymysql==1.0.2 + # via -r requirements/base.txt +pyopenssl==21.0.0 + # via -r requirements/base.txt +pyparsing==3.0.6 + # via packaging +pytest==6.2.5 + # via + # pytest-cov + # pytest-django +pytest-cov==3.0.0 + # via -r requirements/test.in +pytest-django==4.5.2 + # via -r requirements/test.in +python-dateutil==2.8.2 + # via + # -r requirements/base.txt + # botocore +python-gnupg==0.4.8 + # via -r requirements/base.txt +python-slugify==5.0.2 + # via code-annotations +pyyaml==6.0 + # via + # -r requirements/base.txt + # code-annotations +requests==2.26.0 + # via -r requirements/base.txt +s3transfer==0.5.0 + # via + # -r requirements/base.txt + # boto3 +six==1.16.0 + # via + # -r requirements/base.txt + # pyopenssl + # python-dateutil +stevedore==3.5.0 + # via code-annotations +text-unidecode==1.3 + # via python-slugify +toml==0.10.2 + # via pytest +tomli==2.0.0 + # via coverage +urllib3==1.26.7 + # via + # -r requirements/base.txt + # botocore + # requests diff --git a/jenkins/requirements/travis.in b/jenkins/requirements/travis.in new file mode 100644 index 00000000000..f1ad3f9fa03 --- /dev/null +++ b/jenkins/requirements/travis.in @@ -0,0 +1,4 @@ +# Requirements for running tests in Travis +-c constraints.txt + +codecov # Code coverage reporting diff --git a/jenkins/requirements/travis.txt b/jenkins/requirements/travis.txt new file mode 100644 index 00000000000..25c2d8d0623 --- /dev/null +++ b/jenkins/requirements/travis.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +certifi==2021.10.8 + # via requests +charset-normalizer==2.0.9 + # via requests +codecov==2.1.12 + # via -r requirements/travis.in +coverage==6.2 + # via codecov +idna==3.3 + # via requests +requests==2.26.0 + # via codecov +urllib3==1.26.7 + # via requests diff --git a/jenkins/saml-ssl-expiration-check.py b/jenkins/saml-ssl-expiration-check.py new file mode 100644 index 00000000000..124e7ec3cf7 --- /dev/null +++ b/jenkins/saml-ssl-expiration-check.py @@ -0,0 +1,40 @@ +import argparse +import logging +import OpenSSL +from datetime import datetime, timedelta +import sys +import yaml +from os.path import basename + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('-e', '--region', default='us-east-1', required=True, + help="AWS Region") + + parser.add_argument('-d', '--days', type=int, + help="Alert if SSL certificate will expire within these days", default=90) + parser.add_argument('-i','--file', + help="input YAML file to parse and get SAML cert") + + + args = parser.parse_args() + + time_now = datetime.now() + ssl_expire_check = time_now + timedelta(days=args.days) + saml_cert_file = args.file + expired_ssl = basename(saml_cert_file).strip('.yml') + + with open(saml_cert_file) as f: + secure_config = yaml.safe_load(f) + cert = secure_config['EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT'] + x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) + cert_expire_date = datetime.strptime(x509.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%S%fZ").date() + + if ssl_expire_check.date() > cert_expire_date: + logger.info("{} SAML certificate will be expired on {}".format(expired_ssl,cert_expire_date)) + sys.exit(1) diff --git a/jenkins/ssl-expiration-check.py b/jenkins/ssl-expiration-check.py new file mode 100644 index 00000000000..3effa5c1fed --- /dev/null +++ b/jenkins/ssl-expiration-check.py @@ -0,0 +1,106 @@ +import boto3 +import argparse +import logging +import ssl +import OpenSSL +import smtplib +from datetime import date, datetime, timedelta +from socket import socket +from pprint import pformat + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def send_an_email(toaddr, fromaddr, expired_ssl_message, not_check_message, region): + client = boto3.client('ses', region_name=region) + + message = """ + +

Hello,

+ +

Certificates that are associated with these load-balancers will be expired in next 30 days:

+ +

{expired_ssl_message}

+ +

These ELBs have SSL certificate but don't have any instance associated to them:

+ +

{not_check_message}

+ + """.format(expired_ssl_message=expired_ssl_message, not_check_message=not_check_message) + client.send_email( + Source=fromaddr, + Destination={ + 'ToAddresses': [ + toaddr + ] + }, + Message={ + 'Subject': { + 'Data': 'These Certificates will be expired in the next 30 days', + 'Charset': 'utf-8' + }, + 'Body': { + 'Html':{ + 'Data': message, + 'Charset': 'utf-8' + } + } + } + ) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser( + description="Find the SSL Certificates that will expire after X days.") + + parser.add_argument('-e', '--region', default='us-east-1', required=True, + help="AWS Region for getting the records", type=str) + + parser.add_argument('-d', '--days', type=int, + help="Alert if SSL certificate will expire within these days", default=30) + + email_args = parser.add_argument_group("Email Arguments", + "Args for sending email.") + + email_args.add_argument('-r', '--recipient', type=str, + help='Recipient email address') + + email_args.add_argument('-f', '--from-email', type=str, + help="Sender email address for email notifications. " + "Email notifications will be disabled if not provided") + + args = parser.parse_args() + + expire_ssl = [] + time_now = datetime.now() + ssl_expire_check = time_now + timedelta(days=args.days) + + elb_conn = boto3.client('elb', region_name=args.region) + elbs = elb_conn.describe_load_balancers()['LoadBalancerDescriptions'] + + elbs_with_ssl = [elb for elb in elbs for listener in elb['ListenerDescriptions'] if (listener['Listener']['LoadBalancerPort'] == 443)] + + elbs_to_check = [(elb['LoadBalancerName'],elb['DNSName']) for elb in elbs_with_ssl if elb['Instances']] + + elbs_not_need_to_check = [elb['DNSName'] for elb in elbs_with_ssl if not elb['Instances']] + + for elb in elbs_to_check: + elb_tags = elb_conn.describe_tags(LoadBalancerNames=[elb[0]])['TagDescriptions'][0]['Tags'] + for tag in elb_tags: + if 'kubernetes.io' in tag["Key"]: + break + else: + print("Checking {}".format(elb[1])) + cert = ssl.get_server_certificate((elb[1], 443)) + x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) + cert_expire_date = datetime.strptime(x509.get_notAfter().decode(), "%Y%m%d%H%M%S%fZ").date() + if ssl_expire_check.date() > cert_expire_date: + print("Expires {}".format(cert_expire_date)) + expire_ssl.append((elb[1],cert_expire_date)) + + if expire_ssl or elbs_not_need_to_check: + expired_ssl_message = pformat(expire_ssl) + not_check_message = pformat(elbs_not_need_to_check) + print(not_check_message) + if args.from_email and args.recipient: + send_an_email(args.recipient, args.from_email, expired_ssl_message, not_check_message, args.region) From 231d7358e7e850751cd53bfc35ab198b8d140d21 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 8 Jan 2024 15:33:31 +0500 Subject: [PATCH 599/664] chore: add extend sandbox termination period script --- jenkins/requirements/base.in | 10 - jenkins/requirements/base.txt | 56 ---- jenkins/requirements/constraints.txt | 12 - jenkins/requirements/dev.in | 10 - jenkins/requirements/dev.txt | 295 ------------------ jenkins/requirements/doc.in | 9 - jenkins/requirements/doc.txt | 212 ------------- jenkins/requirements/pip-tools.in | 4 - jenkins/requirements/pip-tools.txt | 20 -- jenkins/requirements/private.readme | 15 - jenkins/requirements/quality.in | 9 - jenkins/requirements/quality.txt | 205 ------------ jenkins/requirements/test.in | 8 - jenkins/requirements/test.txt | 119 ------- jenkins/requirements/travis.in | 4 - jenkins/requirements/travis.txt | 20 -- jenkins/saml-ssl-expiration-check.py | 40 --- jenkins/ssl-expiration-check.py | 106 ------- .../jenkins}/extend-sandbox-termination.py | 3 +- 19 files changed, 1 insertion(+), 1156 deletions(-) delete mode 100644 jenkins/requirements/base.in delete mode 100644 jenkins/requirements/base.txt delete mode 100644 jenkins/requirements/constraints.txt delete mode 100644 jenkins/requirements/dev.in delete mode 100644 jenkins/requirements/dev.txt delete mode 100644 jenkins/requirements/doc.in delete mode 100644 jenkins/requirements/doc.txt delete mode 100644 jenkins/requirements/pip-tools.in delete mode 100644 jenkins/requirements/pip-tools.txt delete mode 100644 jenkins/requirements/private.readme delete mode 100644 jenkins/requirements/quality.in delete mode 100644 jenkins/requirements/quality.txt delete mode 100644 jenkins/requirements/test.in delete mode 100644 jenkins/requirements/test.txt delete mode 100644 jenkins/requirements/travis.in delete mode 100644 jenkins/requirements/travis.txt delete mode 100644 jenkins/saml-ssl-expiration-check.py delete mode 100644 jenkins/ssl-expiration-check.py rename {jenkins => util/jenkins}/extend-sandbox-termination.py (98%) diff --git a/jenkins/requirements/base.in b/jenkins/requirements/base.in deleted file mode 100644 index 5f7f34712b2..00000000000 --- a/jenkins/requirements/base.in +++ /dev/null @@ -1,10 +0,0 @@ -# Core requirements for using this application --c constraints.txt -boto -boto3 -python-gnupg -pyyaml -pymysql -pymongo -requests -pyOpenSSL diff --git a/jenkins/requirements/base.txt b/jenkins/requirements/base.txt deleted file mode 100644 index a5f3f34912c..00000000000 --- a/jenkins/requirements/base.txt +++ /dev/null @@ -1,56 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -boto==2.49.0 - # via -r requirements/base.in -boto3==1.20.26 - # via -r requirements/base.in -botocore==1.23.26 - # via - # boto3 - # s3transfer -certifi==2021.10.8 - # via requests -cffi==1.15.0 - # via cryptography -charset-normalizer==2.0.9 - # via requests -cryptography==3.4.8 - # via - # -c requirements/constraints.txt - # pyopenssl -idna==3.3 - # via requests -jmespath==0.10.0 - # via - # boto3 - # botocore -pycparser==2.21 - # via cffi -pymongo==4.0.1 - # via -r requirements/base.in -pymysql==1.0.2 - # via -r requirements/base.in -pyopenssl==21.0.0 - # via -r requirements/base.in -python-dateutil==2.8.2 - # via botocore -python-gnupg==0.4.8 - # via -r requirements/base.in -pyyaml==6.0 - # via -r requirements/base.in -requests==2.26.0 - # via -r requirements/base.in -s3transfer==0.5.0 - # via boto3 -six==1.16.0 - # via - # pyopenssl - # python-dateutil -urllib3==1.26.7 - # via - # botocore - # requests diff --git a/jenkins/requirements/constraints.txt b/jenkins/requirements/constraints.txt deleted file mode 100644 index 4386c3e6d9d..00000000000 --- a/jenkins/requirements/constraints.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Version constraints for pip-installation. -# -# This file doesn't install any packages. It specifies version constraints -# that will be applied if a package is needed. -# -# When pinning something here, please provide an explanation of why. Ideally, -# link to other information that will help people in the future to remove the -# pin when possible. Writing an issue against the offending project and -# linking to it here is good. - -# Pin cryptography because it broke saml-ssl-expiration-check.py, unpin when fixed -cryptography<=3.4.8 diff --git a/jenkins/requirements/dev.in b/jenkins/requirements/dev.in deleted file mode 100644 index 2e8c93c698b..00000000000 --- a/jenkins/requirements/dev.in +++ /dev/null @@ -1,10 +0,0 @@ -# Additional requirements for development of this application --c constraints.txt - --r pip-tools.txt # pip-tools and its dependencies, for managing requirements files --r quality.txt # Core and quality check dependencies --r travis.txt # tox and related dependencies - -diff-cover # Changeset diff test coverage -edx-i18n-tools # For i18n_tool dummy -tox-battery # Makes tox aware of requirements file changes diff --git a/jenkins/requirements/dev.txt b/jenkins/requirements/dev.txt deleted file mode 100644 index dc5768fbabf..00000000000 --- a/jenkins/requirements/dev.txt +++ /dev/null @@ -1,295 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -asgiref==3.4.1 - # via django -astroid==2.9.0 - # via - # -r requirements/quality.txt - # pylint - # pylint-celery -attrs==21.2.0 - # via - # -r requirements/quality.txt - # pytest -backports.entry-points-selectable==1.1.1 - # via virtualenv -boto==2.49.0 - # via -r requirements/quality.txt -boto3==1.20.26 - # via -r requirements/quality.txt -botocore==1.23.26 - # via - # -r requirements/quality.txt - # boto3 - # s3transfer -certifi==2021.10.8 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # requests -cffi==1.15.0 - # via - # -r requirements/quality.txt - # cryptography -chardet==4.0.0 - # via diff-cover -charset-normalizer==2.0.9 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # requests -click==8.0.3 - # via - # -r requirements/pip-tools.txt - # -r requirements/quality.txt - # click-log - # code-annotations - # edx-lint - # pip-tools -click-log==0.3.2 - # via - # -r requirements/quality.txt - # edx-lint -code-annotations==1.2.0 - # via - # -r requirements/quality.txt - # edx-lint -codecov==2.1.12 - # via -r requirements/travis.txt -coverage[toml]==6.2 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # codecov - # pytest-cov -cryptography==3.4.8 - # via - # -c requirements/constraints.txt - # -r requirements/quality.txt - # pyopenssl -diff-cover==6.4.4 - # via -r requirements/dev.in -distlib==0.3.4 - # via virtualenv -django==3.2.10 - # via edx-i18n-tools -edx-i18n-tools==0.8.1 - # via -r requirements/dev.in -edx-lint==5.2.1 - # via -r requirements/quality.txt -filelock==3.4.0 - # via - # tox - # virtualenv -idna==3.3 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # requests -iniconfig==1.1.1 - # via - # -r requirements/quality.txt - # pytest -isort==5.10.1 - # via - # -r requirements/quality.txt - # pylint -jinja2==3.0.3 - # via - # -r requirements/quality.txt - # code-annotations - # diff-cover -jmespath==0.10.0 - # via - # -r requirements/quality.txt - # boto3 - # botocore -lazy-object-proxy==1.7.1 - # via - # -r requirements/quality.txt - # astroid -markupsafe==2.0.1 - # via - # -r requirements/quality.txt - # jinja2 -mccabe==0.6.1 - # via - # -r requirements/quality.txt - # pylint -packaging==21.3 - # via - # -r requirements/quality.txt - # pytest - # tox -path==16.2.0 - # via edx-i18n-tools -pbr==5.8.0 - # via - # -r requirements/quality.txt - # stevedore -pep517==0.12.0 - # via - # -r requirements/pip-tools.txt - # pip-tools -pip-tools==6.4.0 - # via -r requirements/pip-tools.txt -platformdirs==2.4.0 - # via - # -r requirements/quality.txt - # pylint - # virtualenv -pluggy==1.0.0 - # via - # -r requirements/quality.txt - # diff-cover - # pytest - # tox -polib==1.1.1 - # via edx-i18n-tools -py==1.11.0 - # via - # -r requirements/quality.txt - # pytest - # tox -pycodestyle==2.8.0 - # via -r requirements/quality.txt -pycparser==2.21 - # via - # -r requirements/quality.txt - # cffi -pydocstyle==6.1.1 - # via -r requirements/quality.txt -pygments==2.10.0 - # via diff-cover -pylint==2.12.2 - # via - # -r requirements/quality.txt - # edx-lint - # pylint-celery - # pylint-django - # pylint-plugin-utils -pylint-celery==0.3 - # via - # -r requirements/quality.txt - # edx-lint -pylint-django==2.4.4 - # via - # -r requirements/quality.txt - # edx-lint -pylint-plugin-utils==0.6 - # via - # -r requirements/quality.txt - # pylint-celery - # pylint-django -pymongo==4.0.1 - # via -r requirements/quality.txt -pymysql==1.0.2 - # via -r requirements/quality.txt -pyopenssl==21.0.0 - # via -r requirements/quality.txt -pyparsing==3.0.6 - # via - # -r requirements/quality.txt - # packaging -pytest==6.2.5 - # via - # -r requirements/quality.txt - # pytest-cov - # pytest-django -pytest-cov==3.0.0 - # via -r requirements/quality.txt -pytest-django==4.5.2 - # via -r requirements/quality.txt -python-dateutil==2.8.2 - # via - # -r requirements/quality.txt - # botocore -python-gnupg==0.4.8 - # via -r requirements/quality.txt -python-slugify==5.0.2 - # via - # -r requirements/quality.txt - # code-annotations -pytz==2021.3 - # via django -pyyaml==6.0 - # via - # -r requirements/quality.txt - # code-annotations - # edx-i18n-tools -requests==2.26.0 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # codecov -s3transfer==0.5.0 - # via - # -r requirements/quality.txt - # boto3 -six==1.16.0 - # via - # -r requirements/quality.txt - # edx-lint - # pyopenssl - # python-dateutil - # tox - # virtualenv -snowballstemmer==2.2.0 - # via - # -r requirements/quality.txt - # pydocstyle -sqlparse==0.4.2 - # via django -stevedore==3.5.0 - # via - # -r requirements/quality.txt - # code-annotations -text-unidecode==1.3 - # via - # -r requirements/quality.txt - # python-slugify -toml==0.10.2 - # via - # -r requirements/quality.txt - # pylint - # pytest - # tox -tomli==2.0.0 - # via - # -r requirements/pip-tools.txt - # -r requirements/quality.txt - # coverage - # pep517 -tox==3.24.4 - # via tox-battery -tox-battery==0.6.1 - # via -r requirements/dev.in -typing-extensions==4.0.1 - # via - # -r requirements/quality.txt - # astroid - # pylint -urllib3==1.26.7 - # via - # -r requirements/quality.txt - # -r requirements/travis.txt - # botocore - # requests -virtualenv==20.10.0 - # via tox -wheel==0.37.1 - # via - # -r requirements/pip-tools.txt - # pip-tools -wrapt==1.13.3 - # via - # -r requirements/quality.txt - # astroid - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/jenkins/requirements/doc.in b/jenkins/requirements/doc.in deleted file mode 100644 index 690e8e1d5b1..00000000000 --- a/jenkins/requirements/doc.in +++ /dev/null @@ -1,9 +0,0 @@ -# Requirements for documentation validation --c constraints.txt - --r test.txt # Core and testing dependencies for this package - -doc8 # reStructuredText style checker -edx_sphinx_theme # edX theme for Sphinx output -readme_renderer # Validates README.rst for usage on PyPI -Sphinx # Documentation builder diff --git a/jenkins/requirements/doc.txt b/jenkins/requirements/doc.txt deleted file mode 100644 index 2942cd63331..00000000000 --- a/jenkins/requirements/doc.txt +++ /dev/null @@ -1,212 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -alabaster==0.7.12 - # via sphinx -attrs==21.2.0 - # via - # -r requirements/test.txt - # pytest -babel==2.9.1 - # via sphinx -bleach==4.1.0 - # via readme-renderer -boto==2.49.0 - # via -r requirements/test.txt -boto3==1.20.26 - # via -r requirements/test.txt -botocore==1.23.26 - # via - # -r requirements/test.txt - # boto3 - # s3transfer -certifi==2021.10.8 - # via - # -r requirements/test.txt - # requests -cffi==1.15.0 - # via - # -r requirements/test.txt - # cryptography -charset-normalizer==2.0.9 - # via - # -r requirements/test.txt - # requests -click==8.0.3 - # via - # -r requirements/test.txt - # code-annotations -code-annotations==1.2.0 - # via -r requirements/test.txt -coverage[toml]==6.2 - # via - # -r requirements/test.txt - # pytest-cov -cryptography==3.4.8 - # via - # -c requirements/constraints.txt - # -r requirements/test.txt - # pyopenssl -doc8==0.10.1 - # via -r requirements/doc.in -docutils==0.17.1 - # via - # doc8 - # readme-renderer - # restructuredtext-lint - # sphinx -edx-sphinx-theme==3.0.0 - # via -r requirements/doc.in -idna==3.3 - # via - # -r requirements/test.txt - # requests -imagesize==1.3.0 - # via sphinx -iniconfig==1.1.1 - # via - # -r requirements/test.txt - # pytest -jinja2==3.0.3 - # via - # -r requirements/test.txt - # code-annotations - # sphinx -jmespath==0.10.0 - # via - # -r requirements/test.txt - # boto3 - # botocore -markupsafe==2.0.1 - # via - # -r requirements/test.txt - # jinja2 -packaging==21.3 - # via - # -r requirements/test.txt - # bleach - # pytest - # sphinx -pbr==5.8.0 - # via - # -r requirements/test.txt - # stevedore -pluggy==1.0.0 - # via - # -r requirements/test.txt - # pytest -py==1.11.0 - # via - # -r requirements/test.txt - # pytest -pycparser==2.21 - # via - # -r requirements/test.txt - # cffi -pygments==2.10.0 - # via - # doc8 - # readme-renderer - # sphinx -pymongo==4.0.1 - # via -r requirements/test.txt -pymysql==1.0.2 - # via -r requirements/test.txt -pyopenssl==21.0.0 - # via -r requirements/test.txt -pyparsing==3.0.6 - # via - # -r requirements/test.txt - # packaging -pytest==6.2.5 - # via - # -r requirements/test.txt - # pytest-cov - # pytest-django -pytest-cov==3.0.0 - # via -r requirements/test.txt -pytest-django==4.5.2 - # via -r requirements/test.txt -python-dateutil==2.8.2 - # via - # -r requirements/test.txt - # botocore -python-gnupg==0.4.8 - # via -r requirements/test.txt -python-slugify==5.0.2 - # via - # -r requirements/test.txt - # code-annotations -pytz==2021.3 - # via babel -pyyaml==6.0 - # via - # -r requirements/test.txt - # code-annotations -readme-renderer==32.0 - # via -r requirements/doc.in -requests==2.26.0 - # via - # -r requirements/test.txt - # sphinx -restructuredtext-lint==1.3.2 - # via doc8 -s3transfer==0.5.0 - # via - # -r requirements/test.txt - # boto3 -six==1.16.0 - # via - # -r requirements/test.txt - # bleach - # edx-sphinx-theme - # pyopenssl - # python-dateutil -snowballstemmer==2.2.0 - # via sphinx -sphinx==4.3.2 - # via - # -r requirements/doc.in - # edx-sphinx-theme -sphinxcontrib-applehelp==1.0.2 - # via sphinx -sphinxcontrib-devhelp==1.0.2 - # via sphinx -sphinxcontrib-htmlhelp==2.0.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==1.0.3 - # via sphinx -sphinxcontrib-serializinghtml==1.1.5 - # via sphinx -stevedore==3.5.0 - # via - # -r requirements/test.txt - # code-annotations - # doc8 -text-unidecode==1.3 - # via - # -r requirements/test.txt - # python-slugify -toml==0.10.2 - # via - # -r requirements/test.txt - # pytest -tomli==2.0.0 - # via - # -r requirements/test.txt - # coverage -urllib3==1.26.7 - # via - # -r requirements/test.txt - # botocore - # requests -webencodings==0.5.1 - # via bleach - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/jenkins/requirements/pip-tools.in b/jenkins/requirements/pip-tools.in deleted file mode 100644 index 3f1b64ae937..00000000000 --- a/jenkins/requirements/pip-tools.in +++ /dev/null @@ -1,4 +0,0 @@ -# Just the dependencies to run pip-tools, mainly for the "upgrade" make target --c constraints.txt - -pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/jenkins/requirements/pip-tools.txt b/jenkins/requirements/pip-tools.txt deleted file mode 100644 index f4b492113e3..00000000000 --- a/jenkins/requirements/pip-tools.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -click==8.0.3 - # via pip-tools -pep517==0.12.0 - # via pip-tools -pip-tools==6.4.0 - # via -r requirements/pip-tools.in -tomli==2.0.0 - # via pep517 -wheel==0.37.1 - # via pip-tools - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/jenkins/requirements/private.readme b/jenkins/requirements/private.readme deleted file mode 100644 index 5600a1075bc..00000000000 --- a/jenkins/requirements/private.readme +++ /dev/null @@ -1,15 +0,0 @@ -# If there are any Python packages you want to keep in your virtualenv beyond -# those listed in the official requirements files, create a "private.in" file -# and list them there. Generate the corresponding "private.txt" file pinning -# all of their indirect dependencies to specific versions as follows: - -# pip-compile private.in - -# This allows you to use "pip-sync" without removing these packages: - -# pip-sync requirements/*.txt - -# "private.in" and "private.txt" aren't checked into git to avoid merge -# conflicts, and the presence of this file allows "private.*" to be -# included in scripted pip-sync usage without requiring that those files be -# created first. diff --git a/jenkins/requirements/quality.in b/jenkins/requirements/quality.in deleted file mode 100644 index 0bd84a6468c..00000000000 --- a/jenkins/requirements/quality.in +++ /dev/null @@ -1,9 +0,0 @@ -# Requirements for code quality checks --c constraints.txt - --r test.txt # Core and testing dependencies for this package - -edx-lint # edX pylint rules and plugins -isort # to standardize order of imports -pycodestyle # PEP 8 compliance validation -pydocstyle # PEP 257 compliance validation diff --git a/jenkins/requirements/quality.txt b/jenkins/requirements/quality.txt deleted file mode 100644 index 86f5ed5fab6..00000000000 --- a/jenkins/requirements/quality.txt +++ /dev/null @@ -1,205 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -astroid==2.9.0 - # via - # pylint - # pylint-celery -attrs==21.2.0 - # via - # -r requirements/test.txt - # pytest -boto==2.49.0 - # via -r requirements/test.txt -boto3==1.20.26 - # via -r requirements/test.txt -botocore==1.23.26 - # via - # -r requirements/test.txt - # boto3 - # s3transfer -certifi==2021.10.8 - # via - # -r requirements/test.txt - # requests -cffi==1.15.0 - # via - # -r requirements/test.txt - # cryptography -charset-normalizer==2.0.9 - # via - # -r requirements/test.txt - # requests -click==8.0.3 - # via - # -r requirements/test.txt - # click-log - # code-annotations - # edx-lint -click-log==0.3.2 - # via edx-lint -code-annotations==1.2.0 - # via - # -r requirements/test.txt - # edx-lint -coverage[toml]==6.2 - # via - # -r requirements/test.txt - # pytest-cov -cryptography==3.4.8 - # via - # -c requirements/constraints.txt - # -r requirements/test.txt - # pyopenssl -edx-lint==5.2.1 - # via -r requirements/quality.in -idna==3.3 - # via - # -r requirements/test.txt - # requests -iniconfig==1.1.1 - # via - # -r requirements/test.txt - # pytest -isort==5.10.1 - # via - # -r requirements/quality.in - # pylint -jinja2==3.0.3 - # via - # -r requirements/test.txt - # code-annotations -jmespath==0.10.0 - # via - # -r requirements/test.txt - # boto3 - # botocore -lazy-object-proxy==1.7.1 - # via astroid -markupsafe==2.0.1 - # via - # -r requirements/test.txt - # jinja2 -mccabe==0.6.1 - # via pylint -packaging==21.3 - # via - # -r requirements/test.txt - # pytest -pbr==5.8.0 - # via - # -r requirements/test.txt - # stevedore -platformdirs==2.4.0 - # via pylint -pluggy==1.0.0 - # via - # -r requirements/test.txt - # pytest -py==1.11.0 - # via - # -r requirements/test.txt - # pytest -pycodestyle==2.8.0 - # via -r requirements/quality.in -pycparser==2.21 - # via - # -r requirements/test.txt - # cffi -pydocstyle==6.1.1 - # via -r requirements/quality.in -pylint==2.12.2 - # via - # edx-lint - # pylint-celery - # pylint-django - # pylint-plugin-utils -pylint-celery==0.3 - # via edx-lint -pylint-django==2.4.4 - # via edx-lint -pylint-plugin-utils==0.6 - # via - # pylint-celery - # pylint-django -pymongo==4.0.1 - # via -r requirements/test.txt -pymysql==1.0.2 - # via -r requirements/test.txt -pyopenssl==21.0.0 - # via -r requirements/test.txt -pyparsing==3.0.6 - # via - # -r requirements/test.txt - # packaging -pytest==6.2.5 - # via - # -r requirements/test.txt - # pytest-cov - # pytest-django -pytest-cov==3.0.0 - # via -r requirements/test.txt -pytest-django==4.5.2 - # via -r requirements/test.txt -python-dateutil==2.8.2 - # via - # -r requirements/test.txt - # botocore -python-gnupg==0.4.8 - # via -r requirements/test.txt -python-slugify==5.0.2 - # via - # -r requirements/test.txt - # code-annotations -pyyaml==6.0 - # via - # -r requirements/test.txt - # code-annotations -requests==2.26.0 - # via -r requirements/test.txt -s3transfer==0.5.0 - # via - # -r requirements/test.txt - # boto3 -six==1.16.0 - # via - # -r requirements/test.txt - # edx-lint - # pyopenssl - # python-dateutil -snowballstemmer==2.2.0 - # via pydocstyle -stevedore==3.5.0 - # via - # -r requirements/test.txt - # code-annotations -text-unidecode==1.3 - # via - # -r requirements/test.txt - # python-slugify -toml==0.10.2 - # via - # -r requirements/test.txt - # pylint - # pytest -tomli==2.0.0 - # via - # -r requirements/test.txt - # coverage -typing-extensions==4.0.1 - # via - # astroid - # pylint -urllib3==1.26.7 - # via - # -r requirements/test.txt - # botocore - # requests -wrapt==1.13.3 - # via astroid - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/jenkins/requirements/test.in b/jenkins/requirements/test.in deleted file mode 100644 index 6797160bffa..00000000000 --- a/jenkins/requirements/test.in +++ /dev/null @@ -1,8 +0,0 @@ -# Requirements for test runs. --c constraints.txt - --r base.txt # Core dependencies for this package - -pytest-cov # pytest extension for code coverage statistics -pytest-django # pytest extension for better Django support -code-annotations # provides commands used by the pii_check make target. diff --git a/jenkins/requirements/test.txt b/jenkins/requirements/test.txt deleted file mode 100644 index 5838cfa4633..00000000000 --- a/jenkins/requirements/test.txt +++ /dev/null @@ -1,119 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -attrs==21.2.0 - # via pytest -boto==2.49.0 - # via -r requirements/base.txt -boto3==1.20.26 - # via -r requirements/base.txt -botocore==1.23.26 - # via - # -r requirements/base.txt - # boto3 - # s3transfer -certifi==2021.10.8 - # via - # -r requirements/base.txt - # requests -cffi==1.15.0 - # via - # -r requirements/base.txt - # cryptography -charset-normalizer==2.0.9 - # via - # -r requirements/base.txt - # requests -click==8.0.3 - # via code-annotations -code-annotations==1.2.0 - # via -r requirements/test.in -coverage[toml]==6.2 - # via pytest-cov -cryptography==3.4.8 - # via - # -c requirements/constraints.txt - # -r requirements/base.txt - # pyopenssl -idna==3.3 - # via - # -r requirements/base.txt - # requests -iniconfig==1.1.1 - # via pytest -jinja2==3.0.3 - # via code-annotations -jmespath==0.10.0 - # via - # -r requirements/base.txt - # boto3 - # botocore -markupsafe==2.0.1 - # via jinja2 -packaging==21.3 - # via pytest -pbr==5.8.0 - # via stevedore -pluggy==1.0.0 - # via pytest -py==1.11.0 - # via pytest -pycparser==2.21 - # via - # -r requirements/base.txt - # cffi -pymongo==4.0.1 - # via -r requirements/base.txt -pymysql==1.0.2 - # via -r requirements/base.txt -pyopenssl==21.0.0 - # via -r requirements/base.txt -pyparsing==3.0.6 - # via packaging -pytest==6.2.5 - # via - # pytest-cov - # pytest-django -pytest-cov==3.0.0 - # via -r requirements/test.in -pytest-django==4.5.2 - # via -r requirements/test.in -python-dateutil==2.8.2 - # via - # -r requirements/base.txt - # botocore -python-gnupg==0.4.8 - # via -r requirements/base.txt -python-slugify==5.0.2 - # via code-annotations -pyyaml==6.0 - # via - # -r requirements/base.txt - # code-annotations -requests==2.26.0 - # via -r requirements/base.txt -s3transfer==0.5.0 - # via - # -r requirements/base.txt - # boto3 -six==1.16.0 - # via - # -r requirements/base.txt - # pyopenssl - # python-dateutil -stevedore==3.5.0 - # via code-annotations -text-unidecode==1.3 - # via python-slugify -toml==0.10.2 - # via pytest -tomli==2.0.0 - # via coverage -urllib3==1.26.7 - # via - # -r requirements/base.txt - # botocore - # requests diff --git a/jenkins/requirements/travis.in b/jenkins/requirements/travis.in deleted file mode 100644 index f1ad3f9fa03..00000000000 --- a/jenkins/requirements/travis.in +++ /dev/null @@ -1,4 +0,0 @@ -# Requirements for running tests in Travis --c constraints.txt - -codecov # Code coverage reporting diff --git a/jenkins/requirements/travis.txt b/jenkins/requirements/travis.txt deleted file mode 100644 index 25c2d8d0623..00000000000 --- a/jenkins/requirements/travis.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# make upgrade -# -certifi==2021.10.8 - # via requests -charset-normalizer==2.0.9 - # via requests -codecov==2.1.12 - # via -r requirements/travis.in -coverage==6.2 - # via codecov -idna==3.3 - # via requests -requests==2.26.0 - # via codecov -urllib3==1.26.7 - # via requests diff --git a/jenkins/saml-ssl-expiration-check.py b/jenkins/saml-ssl-expiration-check.py deleted file mode 100644 index 124e7ec3cf7..00000000000 --- a/jenkins/saml-ssl-expiration-check.py +++ /dev/null @@ -1,40 +0,0 @@ -import argparse -import logging -import OpenSSL -from datetime import datetime, timedelta -import sys -import yaml -from os.path import basename - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument('-e', '--region', default='us-east-1', required=True, - help="AWS Region") - - parser.add_argument('-d', '--days', type=int, - help="Alert if SSL certificate will expire within these days", default=90) - parser.add_argument('-i','--file', - help="input YAML file to parse and get SAML cert") - - - args = parser.parse_args() - - time_now = datetime.now() - ssl_expire_check = time_now + timedelta(days=args.days) - saml_cert_file = args.file - expired_ssl = basename(saml_cert_file).strip('.yml') - - with open(saml_cert_file) as f: - secure_config = yaml.safe_load(f) - cert = secure_config['EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT'] - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) - cert_expire_date = datetime.strptime(x509.get_notAfter().decode('utf-8'), "%Y%m%d%H%M%S%fZ").date() - - if ssl_expire_check.date() > cert_expire_date: - logger.info("{} SAML certificate will be expired on {}".format(expired_ssl,cert_expire_date)) - sys.exit(1) diff --git a/jenkins/ssl-expiration-check.py b/jenkins/ssl-expiration-check.py deleted file mode 100644 index 3effa5c1fed..00000000000 --- a/jenkins/ssl-expiration-check.py +++ /dev/null @@ -1,106 +0,0 @@ -import boto3 -import argparse -import logging -import ssl -import OpenSSL -import smtplib -from datetime import date, datetime, timedelta -from socket import socket -from pprint import pformat - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def send_an_email(toaddr, fromaddr, expired_ssl_message, not_check_message, region): - client = boto3.client('ses', region_name=region) - - message = """ - -

Hello,

- -

Certificates that are associated with these load-balancers will be expired in next 30 days:

- -

{expired_ssl_message}

- -

These ELBs have SSL certificate but don't have any instance associated to them:

- -

{not_check_message}

- - """.format(expired_ssl_message=expired_ssl_message, not_check_message=not_check_message) - client.send_email( - Source=fromaddr, - Destination={ - 'ToAddresses': [ - toaddr - ] - }, - Message={ - 'Subject': { - 'Data': 'These Certificates will be expired in the next 30 days', - 'Charset': 'utf-8' - }, - 'Body': { - 'Html':{ - 'Data': message, - 'Charset': 'utf-8' - } - } - } - ) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser( - description="Find the SSL Certificates that will expire after X days.") - - parser.add_argument('-e', '--region', default='us-east-1', required=True, - help="AWS Region for getting the records", type=str) - - parser.add_argument('-d', '--days', type=int, - help="Alert if SSL certificate will expire within these days", default=30) - - email_args = parser.add_argument_group("Email Arguments", - "Args for sending email.") - - email_args.add_argument('-r', '--recipient', type=str, - help='Recipient email address') - - email_args.add_argument('-f', '--from-email', type=str, - help="Sender email address for email notifications. " - "Email notifications will be disabled if not provided") - - args = parser.parse_args() - - expire_ssl = [] - time_now = datetime.now() - ssl_expire_check = time_now + timedelta(days=args.days) - - elb_conn = boto3.client('elb', region_name=args.region) - elbs = elb_conn.describe_load_balancers()['LoadBalancerDescriptions'] - - elbs_with_ssl = [elb for elb in elbs for listener in elb['ListenerDescriptions'] if (listener['Listener']['LoadBalancerPort'] == 443)] - - elbs_to_check = [(elb['LoadBalancerName'],elb['DNSName']) for elb in elbs_with_ssl if elb['Instances']] - - elbs_not_need_to_check = [elb['DNSName'] for elb in elbs_with_ssl if not elb['Instances']] - - for elb in elbs_to_check: - elb_tags = elb_conn.describe_tags(LoadBalancerNames=[elb[0]])['TagDescriptions'][0]['Tags'] - for tag in elb_tags: - if 'kubernetes.io' in tag["Key"]: - break - else: - print("Checking {}".format(elb[1])) - cert = ssl.get_server_certificate((elb[1], 443)) - x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert) - cert_expire_date = datetime.strptime(x509.get_notAfter().decode(), "%Y%m%d%H%M%S%fZ").date() - if ssl_expire_check.date() > cert_expire_date: - print("Expires {}".format(cert_expire_date)) - expire_ssl.append((elb[1],cert_expire_date)) - - if expire_ssl or elbs_not_need_to_check: - expired_ssl_message = pformat(expire_ssl) - not_check_message = pformat(elbs_not_need_to_check) - print(not_check_message) - if args.from_email and args.recipient: - send_an_email(args.recipient, args.from_email, expired_ssl_message, not_check_message, args.region) diff --git a/jenkins/extend-sandbox-termination.py b/util/jenkins/extend-sandbox-termination.py similarity index 98% rename from jenkins/extend-sandbox-termination.py rename to util/jenkins/extend-sandbox-termination.py index 6a5740970c9..d92126ce1e5 100644 --- a/jenkins/extend-sandbox-termination.py +++ b/util/jenkins/extend-sandbox-termination.py @@ -1,4 +1,3 @@ -__author__ = 'arbab' ''' This script will be used to modify/extend the termination date on the sandbox. ''' @@ -86,4 +85,4 @@ instance.tags['owner'], days_to_increase ) - ) \ No newline at end of file + ) From 5efb3b69d25ea55f457d1b130c0427e384ca8dde Mon Sep 17 00:00:00 2001 From: Adam Blackwell Date: Thu, 25 Jun 2020 15:55:36 -0400 Subject: [PATCH 600/664] docs: Update pull request template. Also removing link to https://openedx.atlassian.net/wiki/spaces/SRE/pages/28967861/Ops+Ansible+Testing+Checklist since --- .github/PULL_REQUEST_TEMPLATE.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b5e6622ff4f..e55706fa566 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,10 +3,7 @@ Configuration Pull Request Make sure that the following steps are done before merging: - - [ ] A SRE team member has approved the PR if it is code shared across multiple services and you don't own all of the services. - - [ ] Are you adding any new default values that need to be overridden when this change goes live? If so: - - [ ] Update the appropriate internal repo (be sure to update for all our environments) - - [ ] If you are updating a secure value rather than an internal one, file a SRE ticket with details. - - [ ] Add an entry to the CHANGELOG. - - [ ] If you are making a complicated change, have you performed the proper testing specified on the [Ops Ansible Testing Checklist](https://openedx.atlassian.net/wiki/spaces/SRE/pages/28967861/Ops+Ansible+Testing+Checklist)? Adding a new variable does not require the full list (although testing on a sandbox is a great idea to ensure it links with your downstream code changes). - - [ ] Think about how this change will affect Open edX operators. Have you updated the wiki page for the next Open edX release? + - [ ] Have a Site Reliability Engineer review the PR if you don't own all of the services impacted. + - [ ] If you are adding any new default values that need to be overridden when this change goes live, update internal repos and add an entry to the top of the CHANGELOG. + - [ ] Performed the appropriate testing. + - [ ] Think about how this change will affect Open edX operators and update the wiki page for the next Open edX release if needed From 7f97b24f2f6066c5b5da162b157e647edb2343f7 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Thu, 25 Jan 2024 16:55:47 -0500 Subject: [PATCH 601/664] fix: add retry to fetch tags (#7091) --- playbooks/continuous_delivery/create_ami.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/continuous_delivery/create_ami.yml b/playbooks/continuous_delivery/create_ami.yml index f2b73d8642a..db5e2e0ec9f 100644 --- a/playbooks/continuous_delivery/create_ami.yml +++ b/playbooks/continuous_delivery/create_ami.yml @@ -52,6 +52,8 @@ resource: "{{ instance_id }}" state: list register: instance_tags + retries: 3 + until: instance_tags is succeeded - name: Stop instance ec2: From c363f1697014d3ac08ca795f360e645363766b18 Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Mon, 29 Jan 2024 13:08:22 -0500 Subject: [PATCH 602/664] chore: upgrade to node18 (was node16) for mfe role (#7093) --- playbooks/roles/mfe/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 7e1002a30a1..9481839551f 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -15,7 +15,7 @@ MFE_NODEENV_DIR: '{{ MFE_HOME }}/nodeenvs/{{ MFE_NAME }}' MFE_NODEENV_BIN: '{{ MFE_NODEENV_DIR }}/bin' MFE_NODE_MODULES_DIR: '{{ MFE_CODE_DIR }}/node_modules' MFE_NODE_BIN: '{{ MFE_NODE_MODULES_DIR }}/.bin' -MFE_NODE_VERSION: '16.13.2' +MFE_NODE_VERSION: '18.17.0' MFE_NPM_VERSION: '8.1.2' MFE_DEBIAN_PKGS_DEFAULT: From ee8a900fcd6e290a945912ef53f4b2fabbd73860 Mon Sep 17 00:00:00 2001 From: Piotr Surowiec Date: Tue, 30 Jan 2024 23:44:07 +0100 Subject: [PATCH 603/664] fix: allow defining MFE-specific env overrides while preserving extra defaults (#7090) --- CHANGELOG.md | 5 +++++ playbooks/roles/mfe/defaults/main.yml | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74dbf7aa56e..9f49f49c162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2024-01-25 + - Role: mfe + - Added `MFE_ENVIRONMENT_DEFAULT_EXTRA` to allow operators to add extra environment variables to all MFEs when + deploying them with the `mfe_deployer` role. + - 2023-10-09 - Role: edxapp diff --git a/playbooks/roles/mfe/defaults/main.yml b/playbooks/roles/mfe/defaults/main.yml index 9481839551f..f47a6b588b6 100644 --- a/playbooks/roles/mfe/defaults/main.yml +++ b/playbooks/roles/mfe/defaults/main.yml @@ -139,8 +139,10 @@ MFE_ENVIRONMENT_DEFAULT: MFE_STANDALONE_NGINX: true +# This variable can be overridden to include extra defaults for all MFEs deployed with the `mfe_deployer` role. +MFE_ENVIRONMENT_DEFAULT_EXTRA: {} # NOTE: This should be overridden by inheriting MFE-specific role. MFE_ENVIRONMENT_EXTRA: {} -MFE_ENVIRONMENT: '{{ MFE_ENVIRONMENT_DEFAULT | combine(MFE_ENVIRONMENT_EXTRA) }}' +MFE_ENVIRONMENT: '{{ MFE_ENVIRONMENT_DEFAULT | combine(MFE_ENVIRONMENT_DEFAULT_EXTRA) | combine(MFE_ENVIRONMENT_EXTRA) }}' MFE_NPM_OVERRIDES: [] From fede510085f43b3859b1f291d1369fb41640f9b7 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 7 Feb 2024 11:48:33 +0500 Subject: [PATCH 604/664] chore: update mongo 4.4 minor version --- playbooks/roles/mongo_4_4/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/mongo_4_4/defaults/main.yml b/playbooks/roles/mongo_4_4/defaults/main.yml index ea9b0ca391d..a154a14c1fe 100644 --- a/playbooks/roles/mongo_4_4/defaults/main.yml +++ b/playbooks/roles/mongo_4_4/defaults/main.yml @@ -4,7 +4,7 @@ mongo_logappend: true mongo_logrotate: reopen MONGO_VERSION_MAJOR_MINOR: "4.4" -MONGO_VERSION_PATCH: "6" +MONGO_VERSION_PATCH: "28" PYMONGO_VERSION: "3.11.2" MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" mongo_port: "27017" From 5545ff7f792687f0e79f135345769c382e47d8ba Mon Sep 17 00:00:00 2001 From: Hassan Date: Wed, 7 Feb 2024 10:32:36 -0600 Subject: [PATCH 605/664] chore: Enable TLSv1.2 on DE jenkins. --- playbooks/roles/jenkins_data_engineering/defaults/main.yml | 2 +- playbooks/roles/jenkins_data_engineering_new/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/jenkins_data_engineering/defaults/main.yml b/playbooks/roles/jenkins_data_engineering/defaults/main.yml index 7edb6adf351..80510583a7f 100644 --- a/playbooks/roles/jenkins_data_engineering/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering/defaults/main.yml @@ -24,7 +24,7 @@ de_jenkins_seed_job_source: '{{ role_path }}/../jenkins_data_engineering/files/x de_jenkins_user_uid: 900 de_jenkins_group_gid: 900 DE_JENKINS_VERSION: jenkins_2.89.4 -de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16g -Djenkins.install.runSetupWizard=false' +de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16g -Djenkins.install.runSetupWizard=false -Dmail.smtp.starttls.enable=true -Dmail.smtp.ssl.protocols=TLSv1.2' jenkins_base_environment_variables: - NAME: 'AUTOMATION_USER' diff --git a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml index c9ac444901b..974164e8cbf 100644 --- a/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml +++ b/playbooks/roles/jenkins_data_engineering_new/defaults/main.yml @@ -29,7 +29,7 @@ de_jenkins_seed_job_source: '{{ role_path }}/../jenkins_data_engineering_new/fil de_jenkins_user_uid: 900 de_jenkins_group_gid: 900 DE_JENKINS_VERSION: jenkins_2.222.3 -de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16g -Djenkins.install.runSetupWizard=false' +de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx16g -Djenkins.install.runSetupWizard=false -Dmail.smtp.starttls.enable=true -Dmail.smtp.ssl.protocols=TLSv1.2' de_jenkins_snap_pkgs: - hub From 635f185dd667c7fe2fee89f7be64146d4bf3384a Mon Sep 17 00:00:00 2001 From: Eugene Dyudyunov Date: Fri, 9 Feb 2024 02:07:27 +0200 Subject: [PATCH 606/664] fix: retirement pipeline pip version (#7046) * fix: retirement pipeline pip version With configured version of the pip (19.0.3) it is impossible to install the cryptography package with version higher than 39.X.X (starting from the palm release it is 40+, source: https://github.com/openedx/tubular/blob/open-release/palm.3/requirements/base.txt#L47) In this PR I use the COMMON_PIP_VERSION. --- playbooks/roles/user_retirement_pipeline/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/user_retirement_pipeline/defaults/main.yml b/playbooks/roles/user_retirement_pipeline/defaults/main.yml index aaf7fffedb1..2ab1beb533e 100644 --- a/playbooks/roles/user_retirement_pipeline/defaults/main.yml +++ b/playbooks/roles/user_retirement_pipeline/defaults/main.yml @@ -26,7 +26,7 @@ retirement_service_venv_dir: "{{ retirement_service_home }}/venv" retirement_service_user_shell: "/bin/bash" retirement_service_script_path: "{{ retirement_service_app_dir }}/scripts" -retirement_service_pip_version: "19.0.3" +retirement_service_pip_version: "{{ COMMON_PIP_VERSION }}" retirement_service_environment: PATH: '{{ retirement_service_venv_dir }}/bin:{{ ansible_env.PATH }}' From c8b480a15e951cc18ce5d96e716606dc540ff4ac Mon Sep 17 00:00:00 2001 From: salman2013 Date: Mon, 12 Feb 2024 19:17:21 +0500 Subject: [PATCH 607/664] chore: remove bok-choy reference --- vagrant/base/analyticstack/Vagrantfile | 2 -- vagrant/release/analyticstack/Vagrantfile | 2 -- 2 files changed, 4 deletions(-) diff --git a/vagrant/base/analyticstack/Vagrantfile b/vagrant/base/analyticstack/Vagrantfile index 8fa50e401b9..def342a330b 100644 --- a/vagrant/base/analyticstack/Vagrantfile +++ b/vagrant/base/analyticstack/Vagrantfile @@ -42,8 +42,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8000, host: 8000 # LMS config.vm.network :forwarded_port, guest: 8001, host: 8001 # Studio config.vm.network :forwarded_port, guest: 8002, host: 8002 # Ecommerce - config.vm.network :forwarded_port, guest: 8003, host: 8003 # LMS for Bok Choy - config.vm.network :forwarded_port, guest: 8031, host: 8031 # Studio for Bok Choy config.vm.network :forwarded_port, guest: 8120, host: 8120 # edX Notes Service config.vm.network :forwarded_port, guest: 8765, host: 8765 config.vm.network :forwarded_port, guest: 9200, host: 9200 diff --git a/vagrant/release/analyticstack/Vagrantfile b/vagrant/release/analyticstack/Vagrantfile index f80a61c60ff..7c5ed276c3e 100644 --- a/vagrant/release/analyticstack/Vagrantfile +++ b/vagrant/release/analyticstack/Vagrantfile @@ -126,8 +126,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.network :forwarded_port, guest: 8000, host: 8000 # LMS config.vm.network :forwarded_port, guest: 8001, host: 8001 # Studio config.vm.network :forwarded_port, guest: 8002, host: 8002 # Ecommerce - config.vm.network :forwarded_port, guest: 8003, host: 8003 # LMS for Bok Choy - config.vm.network :forwarded_port, guest: 8031, host: 8031 # Studio for Bok Choy config.vm.network :forwarded_port, guest: 8120, host: 8120 # edX Notes Service config.vm.network :forwarded_port, guest: 8765, host: 8765 config.vm.network :forwarded_port, guest: 9200, host: 9200 # Elasticsearch From b9a757b6c6e22b432af80ca598756165048e4574 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Tue, 13 Feb 2024 17:18:16 -0500 Subject: [PATCH 608/664] temp: hardcode newrelic fluent-bit version (#7104) The 2.2.2 version of fluent-bit is not available, but the 2.0.8 version is. We are temporarily hard-coding until we find a better fix. % curl --head https://download.newrelic.com/infrastructure_agent/linux/apt/pool/main/f/fluent-bit/fluent-bit_2.2.2_ubuntu-bionic_amd64.deb HTTP/2 404 x-amz-error-code: NoSuchKey x-amz-error-message: The specified key does not exist. date: Tue, 13 Feb 2024 20:10:12 GMT % curl --head https://download.newrelic.com/infrastructure_agent/linux/apt/pool/main/f/fluent-bit_2.2.2_ubuntu-bionic_amd64.deb HTTP/2 200 last-modified: Tue, 13 Feb 2024 18:41:25 GMT x-amz-version-id: pxrI_0iO4EqL0GcVWWrUgCxIn84ob6t4 date: Tue, 13 Feb 2024 20:10:23 GMT --- playbooks/roles/newrelic_infrastructure/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/newrelic_infrastructure/defaults/main.yml b/playbooks/roles/newrelic_infrastructure/defaults/main.yml index 4833da32155..9ccc774766b 100644 --- a/playbooks/roles/newrelic_infrastructure/defaults/main.yml +++ b/playbooks/roles/newrelic_infrastructure/defaults/main.yml @@ -33,6 +33,7 @@ NEWRELIC_INFRASTRUCTURE_AMAZON_REPO: 'https://download.newrelic.com/infrastructu newrelic_infrastructure_debian_pkgs: - newrelic-infra + - fluent-bit=2.0.8 newrelic_infrastructure_redhat_pkgs: - newrelic-infra From 33a382583298fbf59d9491bc8bf90d3524740ea4 Mon Sep 17 00:00:00 2001 From: Phillip Shiu Date: Tue, 13 Feb 2024 21:06:49 -0500 Subject: [PATCH 609/664] Revert "temp: hardcode newrelic fluent-bit version (#7104)" This reverts commit b9a757b6c6e22b432af80ca598756165048e4574. --- playbooks/roles/newrelic_infrastructure/defaults/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/newrelic_infrastructure/defaults/main.yml b/playbooks/roles/newrelic_infrastructure/defaults/main.yml index 9ccc774766b..4833da32155 100644 --- a/playbooks/roles/newrelic_infrastructure/defaults/main.yml +++ b/playbooks/roles/newrelic_infrastructure/defaults/main.yml @@ -33,7 +33,6 @@ NEWRELIC_INFRASTRUCTURE_AMAZON_REPO: 'https://download.newrelic.com/infrastructu newrelic_infrastructure_debian_pkgs: - newrelic-infra - - fluent-bit=2.0.8 newrelic_infrastructure_redhat_pkgs: - newrelic-infra From 6d4c7f4e04fbd3e144e19c280e845dc8030fee91 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 14 Feb 2024 16:51:17 +0500 Subject: [PATCH 610/664] chore: update the Datadog role to be compatible with the latest agent --- playbooks/roles/datadog/defaults/main.yml | 29 ++++++++++----- playbooks/roles/datadog/tasks/main.yml | 37 +++++++++++++++---- .../roles/datadog/templates/conf.yaml.j2 | 9 +++++ 3 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 playbooks/roles/datadog/templates/conf.yaml.j2 diff --git a/playbooks/roles/datadog/defaults/main.yml b/playbooks/roles/datadog/defaults/main.yml index 57674af76ac..79c86170a29 100644 --- a/playbooks/roles/datadog/defaults/main.yml +++ b/playbooks/roles/datadog/defaults/main.yml @@ -1,15 +1,24 @@ --- DATADOG_API_KEY: "SPECIFY_KEY_HERE" -DATADOG_AGENT_VERSION: '1:5.10.1-1' +DATADOG_SITE: datadoghq.com -datadog_apt_key: "0x382E94DE" +DATADOG_AGENT_VERSION: '1:7.50.3-1' + +datadog_user: "dd-agent" + +# Variables for Datadog MongoDB Monitoring +datadog_mongo_host: 127.0.0.1 +datadog_mongo_port: 27017 +datadog_mongo_username: datadog +datadog_mongo_password: password +datadog_mongo_db: admin +datadog_authsource_db: admin + +datadog_apt_key: "0xAD9589B7" datadog_debian_pkgs: - - apparmor-utils - - build-essential - - curl - - g++ - - gcc - - ipython - - pkg-config - - rsyslog + - apt-transport-https + - curl + - gnupg + +DATADOG_MONGODB_MONITORING: true diff --git a/playbooks/roles/datadog/tasks/main.yml b/playbooks/roles/datadog/tasks/main.yml index 6cad2fa9d59..df18605728f 100644 --- a/playbooks/roles/datadog/tasks/main.yml +++ b/playbooks/roles/datadog/tasks/main.yml @@ -22,7 +22,7 @@ - name: Add apt key for datadog apt_key: - id: "382E94DE" + id: "33EE313BAD9589B7" url: "{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ datadog_apt_key }}" state: present tags: @@ -30,7 +30,7 @@ - name: Install apt repository for datadog apt_repository: - repo: 'deb http://apt.datadoghq.com/ stable main' + repo: 'deb http://apt.datadoghq.com/ stable 7' state: present update_cache: yes tags: @@ -43,23 +43,46 @@ - datadog - name: Bootstrap config - shell: cp datadog.conf.example datadog.conf + shell: cp datadog.yaml.example datadog.yaml args: - chdir: /etc/dd-agent/ - creates: /etc/dd-agent/datadog.conf + chdir: /etc/datadog-agent/ + creates: /etc/datadog-agent/datadog.yaml tags: - datadog - name: Update api-key lineinfile: - dest: "/etc/dd-agent/datadog.conf" + dest: "/etc/datadog-agent/datadog.yaml" regexp: "^api_key:.*" - line: "api_key:{{ DATADOG_API_KEY }}" + line: "api_key: {{ DATADOG_API_KEY }}" notify: - restart the datadog service tags: - datadog +- name: Update site + lineinfile: + dest: "/etc/datadog-agent/datadog.yaml" + regexp: "^# site:.*" + line: "site: {{ DATADOG_SITE }}" + notify: + - restart the datadog service + tags: + - datadog + +- name: Write MongoDB monitoring config + template: + src: "conf.yaml.j2" + dest: "/etc/datadog-agent/conf.d/mongo.d/conf.yaml" + owner: "{{ datadog_user }}" + group: "{{ datadog_user }}" + mode: 0644 + notify: + - restart the datadog service + tags: + - datadog + when: DATADOG_MONGODB_MONITORING | default(false) | bool + - name: Ensure started and enabled service: name: datadog-agent diff --git a/playbooks/roles/datadog/templates/conf.yaml.j2 b/playbooks/roles/datadog/templates/conf.yaml.j2 new file mode 100644 index 00000000000..c19a1eecc9b --- /dev/null +++ b/playbooks/roles/datadog/templates/conf.yaml.j2 @@ -0,0 +1,9 @@ +init_config: +instances: + - hosts: + - {{ datadog_mongo_host }}:{{ datadog_mongo_port }} + username: {{ datadog_mongo_username }} + password: {{ datadog_mongo_password }} + database: {{ datadog_mongo_db }} + options: + authSource: {{ datadog_authsource_db }} From 03034147fd6bd50986a894e544b16f2711deaa6f Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 16 Feb 2024 12:08:41 +0500 Subject: [PATCH 611/664] chore: remove r7 agent from playbooks --- playbooks/analyticsapi.yml | 2 -- playbooks/conductor.yml | 2 -- playbooks/discovery.yml | 2 -- playbooks/ecommerce.yml | 2 -- playbooks/ecomworker.yml | 2 -- playbooks/edxapp.yml | 2 -- playbooks/enterprise_catalog.yml | 2 -- playbooks/forum.yml | 2 -- playbooks/insights.yml | 2 -- playbooks/mongo_4_2.yml | 2 -- playbooks/mongo_mms.yml | 2 -- playbooks/prospectus.yml | 2 -- playbooks/tools_jenkins.yml | 2 -- playbooks/xqueue.yml | 2 -- playbooks/xqwatcher.yml | 2 -- 15 files changed, 30 deletions(-) diff --git a/playbooks/analyticsapi.yml b/playbooks/analyticsapi.yml index a8d1f07c6f7..5551301b698 100644 --- a/playbooks/analyticsapi.yml +++ b/playbooks/analyticsapi.yml @@ -24,5 +24,3 @@ - role: hermes HERMES_TARGET_SERVICE: 'analytics_api' when: ANALYTICS_API_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/conductor.yml b/playbooks/conductor.yml index d02facbe0bb..11522c137fa 100644 --- a/playbooks/conductor.yml +++ b/playbooks/conductor.yml @@ -23,5 +23,3 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/discovery.yml b/playbooks/discovery.yml index 68cfcf42496..eb0717c0823 100644 --- a/playbooks/discovery.yml +++ b/playbooks/discovery.yml @@ -24,6 +24,4 @@ - role: hermes HERMES_TARGET_SERVICE: 'discovery' when: DISCOVERY_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/ecommerce.yml b/playbooks/ecommerce.yml index 27cc9be3a92..9f36e4163c3 100644 --- a/playbooks/ecommerce.yml +++ b/playbooks/ecommerce.yml @@ -24,5 +24,3 @@ - role: hermes HERMES_TARGET_SERVICE: 'ecommerce' when: ECOMMERCE_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/ecomworker.yml b/playbooks/ecomworker.yml index 5a9fe7de6a9..f7c288770bb 100644 --- a/playbooks/ecomworker.yml +++ b/playbooks/ecomworker.yml @@ -20,6 +20,4 @@ - role: hermes HERMES_TARGET_SERVICE: 'ecomworker' when: ECOMMERCE_WORKER_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/edxapp.yml b/playbooks/edxapp.yml index ea509022c26..2364f0f585a 100644 --- a/playbooks/edxapp.yml +++ b/playbooks/edxapp.yml @@ -50,5 +50,3 @@ filename: '{{ hermes_download_dir }}/studio.yml' command: "sudo /bin/cp {{ hermes_download_dir }}/studio.yml {{ COMMON_CFG_DIR }}/studio.yml && sudo /edx/app/edxapp/reload_cms_config.sh" secret_key_files: "{{ HERMES_PRIVATE_KEYS_DICT | map('regex_replace','^(.*)$','/edx/app/hermes/hermes-\\1') | join(',') if HERMES_PRIVATE_KEYS_DICT is defined else None }}" - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/enterprise_catalog.yml b/playbooks/enterprise_catalog.yml index f708fadeb7c..c773022b4dd 100644 --- a/playbooks/enterprise_catalog.yml +++ b/playbooks/enterprise_catalog.yml @@ -19,5 +19,3 @@ - role: hermes HERMES_TARGET_SERVICE: 'enterprise_catalog' when: ENTERPRISE_CATALOG_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/forum.yml b/playbooks/forum.yml index 22df4f5f10e..b3ee89ab9b7 100644 --- a/playbooks/forum.yml +++ b/playbooks/forum.yml @@ -21,5 +21,3 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/insights.yml b/playbooks/insights.yml index ecc32becc90..f780a5df4d6 100644 --- a/playbooks/insights.yml +++ b/playbooks/insights.yml @@ -24,5 +24,3 @@ - role: hermes HERMES_TARGET_SERVICE: 'insights' when: INSIGHTS_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/mongo_4_2.yml b/playbooks/mongo_4_2.yml index 7f0c02c072e..50e75177f24 100644 --- a/playbooks/mongo_4_2.yml +++ b/playbooks/mongo_4_2.yml @@ -27,5 +27,3 @@ when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/mongo_mms.yml b/playbooks/mongo_mms.yml index 89ad585bc32..f22c031ebbe 100644 --- a/playbooks/mongo_mms.yml +++ b/playbooks/mongo_mms.yml @@ -13,5 +13,3 @@ when: COMMON_ENABLE_DATADOG - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/prospectus.yml b/playbooks/prospectus.yml index eb9ebfbf386..4c2983e052e 100644 --- a/playbooks/prospectus.yml +++ b/playbooks/prospectus.yml @@ -32,5 +32,3 @@ when: COMMON_ENABLE_SPLUNKFORWARDER|bool and PROSPECTUS_ENABLE_POST_BUILD|bool - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE|bool and PROSPECTUS_ENABLE_POST_BUILD|bool - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT|bool and PROSPECTUS_ENABLE_POST_BUILD|bool diff --git a/playbooks/tools_jenkins.yml b/playbooks/tools_jenkins.yml index b4fc1808a50..57848d95a7d 100644 --- a/playbooks/tools_jenkins.yml +++ b/playbooks/tools_jenkins.yml @@ -37,5 +37,3 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/xqueue.yml b/playbooks/xqueue.yml index f90c7ece45e..4e23fcb06b2 100644 --- a/playbooks/xqueue.yml +++ b/playbooks/xqueue.yml @@ -21,5 +21,3 @@ - role: hermes HERMES_TARGET_SERVICE: 'xqueue' when: XQUEUE_HERMES_ENABLED - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT diff --git a/playbooks/xqwatcher.yml b/playbooks/xqwatcher.yml index 42f052d913f..8798dda77cc 100644 --- a/playbooks/xqwatcher.yml +++ b/playbooks/xqwatcher.yml @@ -19,5 +19,3 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: datadog-uninstall when: not COMMON_ENABLE_DATADOG - - role: insightvm_agent - when: COMMON_ENABLE_INSIGHTVM_AGENT From 2f6f7f0e2a98592fd5a407ce31eba4b4bf177f2c Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 21 Feb 2024 09:09:07 -0500 Subject: [PATCH 612/664] feat: use edx-atlas to pull translations. This adds a step to add translations pulls to the edxapp pipeline. --- playbooks/roles/edxapp/tasks/deploy.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 5e148e6b0ae..df79918c331 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -412,6 +412,15 @@ when: - celery_worker is not defined +# Translations steps +- name: Pull translations via Atlas + shell: ". {{ edxapp_app_dir }}/edxapp_env && make OPENEDX_ATLAS_PULL=true pull_translations" + args: + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + tags: + - assets + - name: compile JS translations shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} compilejsi18n" args: From 0359a53ec9b6274b5c55a7e0f17244ca375523b4 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 21 Feb 2024 13:02:47 -0500 Subject: [PATCH 613/664] feat: Attempt to use edx-atlas to pull translations. Previous attempt did not have access to the proper configs. I think this is the correct spot. --- .../tasks/main.yml | 10 ++++++++++ playbooks/roles/edxapp/tasks/deploy.yml | 18 ------------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml index 2ce3960ed56..3e44d2b1deb 100644 --- a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml +++ b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml @@ -233,6 +233,16 @@ - assets - assets:gather +# Translations step +- name: Pull translations via Atlas + shell: make OPENEDX_ATLAS_PULL=true pull_translations + args: + chdir: "{{ edx_django_service_with_rendered_config_code_dir }}" + become_user: "{{ edx_django_service_with_rendered_config_user }}" + environment: "{{ edx_django_service_with_rendered_config_environment }}" + tags: + - assets + - name: restart the application supervisorctl: state: restarted diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index df79918c331..9c30c62103f 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -412,24 +412,6 @@ when: - celery_worker is not defined -# Translations steps -- name: Pull translations via Atlas - shell: ". {{ edxapp_app_dir }}/edxapp_env && make OPENEDX_ATLAS_PULL=true pull_translations" - args: - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - tags: - - assets - -- name: compile JS translations - shell: ". {{ edxapp_app_dir }}/edxapp_env && {{ edxapp_venv_bin }}/python ./manage.py lms --settings={{ edxapp_settings }} compilejsi18n" - args: - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - when: EDXAPP_COMPILE_JSI18N and celery_worker is not defined - tags: - - assets - # creates the supervisor jobs for the # service variants configured, runs # gather_assets and db migrations From fe5f9043db0ad102c92df7c1f51d77c0cf6e743c Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 21 Feb 2024 13:54:46 -0500 Subject: [PATCH 614/664] feat: Try to build translations somewhere else. I believe this is the correct place to build translations for edxapp pipelines. --- .../tasks/main.yml | 10 ---------- .../roles/edxapp/tasks/service_variant_config.yml | 2 +- .../edxapp/templates/edx/bin/edxapp-update-assets.j2 | 4 ++++ 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml index 3e44d2b1deb..2ce3960ed56 100644 --- a/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml +++ b/playbooks/roles/edx_django_service_with_rendered_config/tasks/main.yml @@ -233,16 +233,6 @@ - assets - assets:gather -# Translations step -- name: Pull translations via Atlas - shell: make OPENEDX_ATLAS_PULL=true pull_translations - args: - chdir: "{{ edx_django_service_with_rendered_config_code_dir }}" - become_user: "{{ edx_django_service_with_rendered_config_user }}" - environment: "{{ edx_django_service_with_rendered_config_environment }}" - tags: - - assets - - name: restart the application supervisorctl: state: restarted diff --git a/playbooks/roles/edxapp/tasks/service_variant_config.yml b/playbooks/roles/edxapp/tasks/service_variant_config.yml index c9b169d32b2..9bc6e411509 100644 --- a/playbooks/roles/edxapp/tasks/service_variant_config.yml +++ b/playbooks/roles/edxapp/tasks/service_variant_config.yml @@ -313,7 +313,7 @@ - assets # Gather assets using paver if possible -- name: "gather static assets with paver" +- name: "gather static assets with paver and pull translations" command: "{{ COMMON_BIN_DIR }}/edxapp-update-assets" when: celery_worker is not defined and not devstack tags: diff --git a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 index fe3dc8fc89d..fdeb2e2f823 100644 --- a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 +++ b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 @@ -11,3 +11,7 @@ sudo -E -H -u {{ edxapp_user }} \ env "PATH=$PATH" \ {{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings $EDX_PLATFORM_SETTINGS {% endif %} + +sudo -E -H -u {{ edxapp_user }} \ + env "OPENEDX_ATLAS_PULL=true" \ + make pull_translations From d3b33f621361006a03089c52cbc202f1ec3130fa Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 21 Feb 2024 14:47:27 -0500 Subject: [PATCH 615/664] fix: Run make pull translations without overriding the PATH env. --- .../roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 index fdeb2e2f823..838c30b2b9d 100644 --- a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 +++ b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 @@ -12,6 +12,6 @@ sudo -E -H -u {{ edxapp_user }} \ {{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings $EDX_PLATFORM_SETTINGS {% endif %} -sudo -E -H -u {{ edxapp_user }} \ +sudo -E -H -u {{ edxapp_user }} --preserve-env=PATH \ env "OPENEDX_ATLAS_PULL=true" \ make pull_translations From 8489ba38e681f9ea31563f690d20249edbf6aabd Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 21 Feb 2024 16:31:05 -0500 Subject: [PATCH 616/664] fix: Remove translations step to unblock build. --- .../roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 index 838c30b2b9d..fe3dc8fc89d 100644 --- a/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 +++ b/playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets.j2 @@ -11,7 +11,3 @@ sudo -E -H -u {{ edxapp_user }} \ env "PATH=$PATH" \ {{ edxapp_venv_bin }}/paver update_assets --debug-collect --settings $EDX_PLATFORM_SETTINGS {% endif %} - -sudo -E -H -u {{ edxapp_user }} --preserve-env=PATH \ - env "OPENEDX_ATLAS_PULL=true" \ - make pull_translations From 55c4bb646827ebc3e3abdda75e4a769e64c180b3 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 22 Feb 2024 15:17:45 -0500 Subject: [PATCH 617/664] feat: Pull edxapp translations via Atlas (#7116) This is to support OEP-58: https://docs.openedx.org/projects/openedx-proposals/en/latest/architectural-decisions/oep-0058-arch-translations-management.html Depends on https://github.com/openedx/edx-platform/pull/34285 --- playbooks/roles/edxapp/tasks/deploy.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 9c30c62103f..105043e14a6 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -246,6 +246,25 @@ - install - install:app-requirements +- name: "Pull translations using Atlas" + shell: | + set -eu -o pipefail + # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe + mkdir -p bin + curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas + chmod +x ./bin/atlas + # Use production-like environment and minimal config to avoid needing dev dependencies or full config. + PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ + LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ + OPENEDX_ATLAS_PULL=true make pull_translations + rm ./bin/atlas + args: + executable: /usr/bin/bash + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + tags: + - install + # Set the npm registry # This needs to be done as root since npm is weird about # chown - https://github.com/npm/npm/issues/3565 From 44cbe50127c4c40e42a9d3ad41b1ee74765f76d5 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 22 Feb 2024 15:51:23 -0500 Subject: [PATCH 618/664] fix: Activate Python venv for pull_translations (#7117) Was getting `python: Command not found` after adding this block in https://github.com/openedx/configuration/pull/7116 --- playbooks/roles/edxapp/tasks/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 105043e14a6..b201d331715 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -253,6 +253,7 @@ mkdir -p bin curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas chmod +x ./bin/atlas + source {{ edxapp_venv_dir }}/bin/activate # Use production-like environment and minimal config to avoid needing dev dependencies or full config. PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ From f2f145e211e4449b0ec19791bb72face548ff08b Mon Sep 17 00:00:00 2001 From: Farhan Umer Date: Mon, 19 Feb 2024 19:43:29 +0500 Subject: [PATCH 619/664] chore: fixed course id in sandbox internal repo --- playbooks/edx_continuous_integration.yml | 4 +- playbooks/mongo_5_0.yml | 29 ++ playbooks/roles/demo/defaults/main.yml | 5 +- playbooks/roles/demo/tasks/deploy.yml | 2 +- playbooks/roles/mongo_5_0/defaults/main.yml | 111 +++++ .../disable-transparent-hugepages.service | 11 + playbooks/roles/mongo_5_0/meta/main.yml | 5 + playbooks/roles/mongo_5_0/tasks/main.yml | 404 ++++++++++++++++++ .../templates/log-mongo-serverStatus.sh.j2 | 3 + .../mongo_5_0/templates/mongo_logrotate.j2 | 46 ++ .../roles/mongo_5_0/templates/mongod.conf.j2 | 46 ++ playbooks/roles/testcourses/defaults/main.yml | 2 +- .../roles/testcourses/tasks/import_course.yml | 2 +- util/jenkins/ansible-provision.sh | 12 +- 14 files changed, 669 insertions(+), 13 deletions(-) create mode 100644 playbooks/mongo_5_0.yml create mode 100644 playbooks/roles/mongo_5_0/defaults/main.yml create mode 100644 playbooks/roles/mongo_5_0/files/etc/systemd/system/disable-transparent-hugepages.service create mode 100644 playbooks/roles/mongo_5_0/meta/main.yml create mode 100644 playbooks/roles/mongo_5_0/tasks/main.yml create mode 100644 playbooks/roles/mongo_5_0/templates/log-mongo-serverStatus.sh.j2 create mode 100644 playbooks/roles/mongo_5_0/templates/mongo_logrotate.j2 create mode 100644 playbooks/roles/mongo_5_0/templates/mongod.conf.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index 8f7737dfa4e..c39ecc00b91 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,10 +25,10 @@ - role: edxlocal tags: edxlocal - memcache - - role: mongo_4_2 - when: MONGO_4_2_ENABLED - role: mongo_4_4 when: MONGO_4_4_ENABLED + - role: mongo_5_0 + when: MONGO_5_0_ENABLED - role: redis - { role: "edxapp", celery_worker: True, when: edxapp_containerized is defined and not edxapp_containerized } - { role: "edxapp", when: edxapp_containerized is defined and not edxapp_containerized } diff --git a/playbooks/mongo_5_0.yml b/playbooks/mongo_5_0.yml new file mode 100644 index 00000000000..24a7a20b06b --- /dev/null +++ b/playbooks/mongo_5_0.yml @@ -0,0 +1,29 @@ +# Manages a mongo cluster. +# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG +# as used by mongo_replica_set in the mongo_4_4 role. +# +# If you are initializing a cluster, your command might look like: +# ansible-playbook mongo_4_4.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml +# If you just want to deploy an updated replica set config, you can run +# ansible-playbook mongo_4_4.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set +# +# ADDING A NEW CLUSTER MEMBER +# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory +# ansible-playbook mongo_4_4.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml +- name: Deploy MongoDB + hosts: all + become: True + gather_facts: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - mongo_5_0 + - munin_node + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG diff --git a/playbooks/roles/demo/defaults/main.yml b/playbooks/roles/demo/defaults/main.yml index 8fa360a39e0..7dd36b86e72 100644 --- a/playbooks/roles/demo/defaults/main.yml +++ b/playbooks/roles/demo/defaults/main.yml @@ -14,9 +14,10 @@ DEMO_CREATE_STAFF_USER: true demo_app_dir: "{{ COMMON_APP_DIR }}/demo" -demo_code_dir: "{{ demo_app_dir }}/edx-demo-course" +demo_code_dir: "{{ demo_app_dir }}/edx-demo-course/" +demo_course_dir: "{{ demo_code_dir }}/demo-course/course" demo_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-demo-course.git" -demo_course_id: 'course-v1:edX+DemoX+Demo_Course' +demo_course_id: 'course-v1:OpenedX+DemoX+DemoCourse' DEMO_VERSION: "master" demo_hashed_password: 'pbkdf2_sha256$20000$TjE34FJjc3vv$0B7GUmH8RwrOc/BvMoxjb5j8EgnWTt3sxorDANeF7Qw=' # edx demo_test_users: diff --git a/playbooks/roles/demo/tasks/deploy.yml b/playbooks/roles/demo/tasks/deploy.yml index 15f751481ef..67d7d20d576 100644 --- a/playbooks/roles/demo/tasks/deploy.yml +++ b/playbooks/roles/demo/tasks/deploy.yml @@ -10,7 +10,7 @@ register: demo_checkout - name: import demo course - shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_edxapp_course_data_dir }} {{ demo_code_dir }}" + shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_edxapp_course_data_dir }} {{ demo_course_dir }}" args: chdir: "{{ demo_edxapp_code_dir }}" become_user: "{{ common_web_user }}" diff --git a/playbooks/roles/mongo_5_0/defaults/main.yml b/playbooks/roles/mongo_5_0/defaults/main.yml new file mode 100644 index 00000000000..531115b54f0 --- /dev/null +++ b/playbooks/roles/mongo_5_0/defaults/main.yml @@ -0,0 +1,111 @@ +mongo_logappend: true + +#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle +mongo_logrotate: reopen + +MONGO_VERSION_MAJOR_MINOR: "5.0" +MONGO_VERSION_PATCH: "24" +PYMONGO_VERSION: "3.12.3" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +mongo_port: "27017" +mongo_extra_conf: '' +mongo_key_file: '/etc/mongodb_key' + +mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo" +mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo" +mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal" +mongo_user: mongodb + +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" + +mongodb_debian_pkgs: + - "mongodb-org={{ MONGO_VERSION }}" + - "mongodb-org-server={{ MONGO_VERSION }}" + - "mongodb-org-shell={{ MONGO_VERSION }}" + - "mongodb-org-mongos={{ MONGO_VERSION }}" + - "mongodb-org-tools={{ MONGO_VERSION }}" + + + +mongo_configure_replica_set: true + +# Vars Meant to be overridden +MONGO_ADMIN_USER: 'admin' +MONGO_ADMIN_PASSWORD: 'password' +MONGO_USERS: + - user: cs_comments_service + password: password + database: cs_comments_service + roles: readWrite + - user: edxapp + password: password + database: edxapp + roles: readWrite + +# This default setting is approriate for a single machine installation +# This will need to be overridden for setups where mongo is on its own server +# and/or you are configuring mongo replication. If the override value is +# 0.0.0.0 mongo will listen on all IPs. The value may also be set to a +# specific IP. +MONGO_BIND_IP: 127.0.0.1 + +MONGO_REPL_SET: "rs0" +MONGO_AUTH: true + +MONGO_CLUSTER_KEY: "CHANGEME" + +# Cluster member configuration +# Fed directly into mongodb_replica_set module +MONGO_RS_CONFIG: + _id: '{{ MONGO_REPL_SET }}' + members: + - host: '127.0.0.1' + +# Storage engine options in 3.2: "mmapv1" or "wiredTiger" +# 3.2 and 3.4 default to wiredTiger +MONGO_STORAGE_ENGINE: "wiredTiger" + +# List of dictionaries as described in the mount_ebs role's default +# for the volumes. +# Useful if you want to store your mongo data and/or journal on separate +# disks from the root volume. By default, they will end up mongo_data_dir +# on the root disk. +MONGO_VOLUMES: [] + +# WiredTiger takes a number of optional configuration settings +# which can be defined as a yaml structure in your secure configuration. +MONGO_STORAGE_ENGINE_OPTIONS: !!null + +mongo_logpath: "{{ mongo_log_dir }}/mongodb.log" +mongo_dbpath: "{{ mongo_data_dir }}/mongodb" + +# In environments that do not require durability (devstack / Jenkins) +# you can disable the journal to reduce disk usage +mongo_enable_journal: true + +MONGO_LOG_SERVERSTATUS: true + +# Vars for configuring a mongo backup node. If enabled, this node will be provisioned with a script that uses mongodump +# to backup the database to an ebs volume at a period set by mongo_backup_cron. +# Set MONGO_BACKUP_ENABLED to true to enable. If enabled, all the other MONGO_BACKUP_ vars must be set according to your +# setup. +MONGO_BACKUP_ENABLED: false +MONGO_BACKUP_NODE: "" # note: most likely the ip address of the instance on which to perform the backups +MONGO_BACKUP_EBS_VOLUME_DEVICE: "" +MONGO_BACKUP_EBS_VOLUME_ID: "" +MONGO_BACKUP_AUTH_DATABASE: "" +MONGO_BACKUP_PRUNE_OLDER_THAN_DATE: "" # passed to `date -d`; should be a relative date like "-30days" +MONGO_BACKUP_SNITCH_URL: "" # Optional URL that will be used to ping a monitoring service (such as Dead Man's Snitch) upon successful completion of a backup. +MONGO_BACKUP_VOLUME_MOUNT_PATH: "/mnt/mongo-backup" +MONGO_BACKUP_SNAPSHOT_DESC: "mongo-backup" +mongo_backup_script_path: "/usr/local/sbin/backup-mongo.sh" +mongo_backup_cron: + minute: '12' + hour: '*/12' + day: '*' + month: '*' + weekday: '*' + +# Internal variable set to true dynamically if backups enabled and playbook running on MONGO_BACKUP_NODE. Do not +# manually override. +is_backup_node: false diff --git a/playbooks/roles/mongo_5_0/files/etc/systemd/system/disable-transparent-hugepages.service b/playbooks/roles/mongo_5_0/files/etc/systemd/system/disable-transparent-hugepages.service new file mode 100644 index 00000000000..282c9e122c3 --- /dev/null +++ b/playbooks/roles/mongo_5_0/files/etc/systemd/system/disable-transparent-hugepages.service @@ -0,0 +1,11 @@ +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +Before=mongod.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' + +[Install] +RequiredBy=mongod.service diff --git a/playbooks/roles/mongo_5_0/meta/main.yml b/playbooks/roles/mongo_5_0/meta/main.yml new file mode 100644 index 00000000000..d7223454526 --- /dev/null +++ b/playbooks/roles/mongo_5_0/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - common + - role: mount_ebs + volumes: "{{ MONGO_VOLUMES }}" diff --git a/playbooks/roles/mongo_5_0/tasks/main.yml b/playbooks/roles/mongo_5_0/tasks/main.yml new file mode 100644 index 00000000000..10dd2484cdf --- /dev/null +++ b/playbooks/roles/mongo_5_0/tasks/main.yml @@ -0,0 +1,404 @@ +--- +- name: Add disable transparent huge pages systemd service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + copy: + src: etc/systemd/system/disable-transparent-hugepages.service + dest: "/etc/systemd/system/disable-transparent-hugepages.service" + owner: root + group: root + mode: 0644 + tags: + - "hugepages" + - "install" + - "install:configuration" + +- name: Enable/start disable transparent huge pages service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + service: + name: disable-transparent-hugepages + enabled: yes + state: started + tags: + - "hugepages" + - "manage" + - "manage:start" + +- name: install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ PYMONGO_VERSION }}" + extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" + state: present + retries: 3 + register: add_mongo_signing_key + tags: + - "install" + - "install:app-requirements" + until: add_mongo_signing_key is succeeded + +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install mongo server and recommends + apt: + pkg: "{{ item }}" + state: present + install_recommends: yes + force: yes + update_cache: yes + register: install_mongo_package + with_items: "{{ mongodb_debian_pkgs }}" + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: create mongo dirs + file: + path: "{{ item }}" + state: directory + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + with_items: + - "{{ mongo_data_dir }}" + - "{{ mongo_dbpath }}" + - "{{ mongo_log_dir }}" + - "{{ mongo_journal_dir }}" + tags: + - "install" + - "install:app-configuration" + +# This will error when run on a new replica set, so we ignore_errors +# and connect anonymously next. +- name: determine if there is a replica set already + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + run_once: true + register: authed_replica_set_already_configured + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +- name: Try checking the replica set with no user/pass in case this is a new box + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + run_once: true + register: unauthed_replica_set_already_configured + when: authed_replica_set_already_configured.failed is defined + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +# We use these in the templates but also to control a whole bunch of logic +- name: set facts that default to not initializing a replica set + set_fact: + initialize_replica_set: false + skip_replica_set: false + tags: + - "install" + - "install:app-configuration" + - "update_mongod_conf" + - "manage" + - "manage:db-replication" + +# If either auth or unauthed access comes back with a replica set, we +# do not want to initialize one. Since initialization requires a bunch +# of extra templating and restarting, it's not something we want to do on +# existing boxes. +- name: track if you have a replica set + set_fact: + initialize_replica_set: true + skip_replica_set: true + when: authed_replica_set_already_configured.status is not defined + and unauthed_replica_set_already_configured.status is not defined + tags: + - "manage" + - "manage:db-replication" + +- name: warn about unconfigured replica sets + debug: msg="You do not appear to have a Replica Set configured, deploying one for you" + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: copy mongodb key file + copy: + content: "{{ MONGO_CLUSTER_KEY }}" + dest: "{{ mongo_key_file }}" + mode: 0600 + owner: mongodb + group: mongodb + register: update_mongod_key + tags: + - "manage" + - "manage:db-replication" + - "mongodb_key" + +# If skip_replica_set is true, this template will not contain a replica set stanza +# because of the fact above. +- name: copy configuration template + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + register: update_mongod_conf + tags: + - "install" + - "install:app-configuration" + - "manage" + - "manage:db-replication" + - "update_mongod_conf" + +# This sets the is_backup_node var by checking whether +# mongo backups are enabled AND we're currently running against the designated mongo backup node. +# This allows backup-related tasks below to determine whether or not they should run on the current mongo node. +- name: determine if backup tasks should run + set_fact: + is_backup_node: true + when: MONGO_BACKUP_ENABLED and '{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}' == '{{ MONGO_BACKUP_NODE }}' + tags: + - "backup:mongo" + +- name: install logrotate configuration + template: + src: mongo_logrotate.j2 + dest: /etc/logrotate.d/hourly/mongo + tags: + - "backup:mongo" + - "install" + - "install:app-configuration" + - "logrotate" + +- name: install prereqs for backup script + apt: + pkg: "{{ item }}" + state: present + update_cache: yes + with_items: + - jq + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install backup script + template: + src: backup-mongo.sh.j2 + dest: "{{ mongo_backup_script_path }}" + mode: 0700 + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: add mongo backup script to cron + cron: + name: mongo backup job + minute: "{{ mongo_backup_cron.minute | default('12') }}" + hour: "{{ mongo_backup_cron.hour | default('*/12') }}" + day: "{{ mongo_backup_cron.day | default('*') }}" + month: "{{ mongo_backup_cron.month | default('*') }}" + weekday: "{{ mongo_backup_cron.weekday | default('*') }}" + job: "{{ mongo_backup_script_path }} >> {{ mongo_log_dir }}/mongo-backup.log 2>&1" + become: yes + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: format mongo backup volume + filesystem: + dev: "{{ MONGO_BACKUP_EBS_VOLUME_DEVICE }}" + fstype: ext4 + force: true + ignore_errors: true + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# We only try passwordless superuser creation when +# we're initializing the replica set and need to use +# the localhost exemption to create a user who will be +# able to initialize the replica set. +# We can only create the users on one machine, the one +# where we will initialize the replica set. If we +# create users on multiple hosts, then they will fail +# to come into the replica set. +- name: create super user + mongodb_user: + name: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + database: admin + roles: root + when: initialize_replica_set + run_once: true + tags: + - "manage" + - "manage:db-replication" + +# Now that the localhost exemption has been used to create the superuser, we need +# to add replica set to our configuration. This will never happen if we detected +# a replica set in the 'determine if there is a replica set already' task. +- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set + set_fact: + skip_replica_set: false + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: re-copy configuration template with replica set enabled + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: restart mongo service + service: + name: mongod + state: restarted + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: configure replica set + mongodb_replica_set: + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + rs_config: "{{ MONGO_RS_CONFIG }}" + run_once: true + register: replset_status + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + - "manage:db-replication-configuration" + +# During initial replica set configuration, it can take a few seconds to vote +# a primary and for all members to reflect that status. During that window, +# use creation or other writes can fail. The best wait/check seems to be repeatedly +# checking the replica set status until we see a PRIMARY in the results. +- name: Wait for the replica set to update and (if needed) elect a primary + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + register: status + until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list + when: mongo_configure_replica_set + retries: 5 + delay: 2 + run_once: true + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + +- name: create mongodb users in a replica set + mongodb_user: + database: "{{ item.database }}" + login_database: 'admin' + login_user: "{{ MONGO_ADMIN_USER }}" + login_password: "{{ MONGO_ADMIN_PASSWORD }}" + name: "{{ item.user }}" + password: "{{ item.password }}" + roles: "{{ item.roles }}" + state: present + replica_set: "{{ MONGO_REPL_SET }}" + with_items: "{{ MONGO_USERS }}" + run_once: true + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-users" + - "manage:db-replication" + +- name: ensure mongo starts at boot time + service: + name: mongod + enabled: yes + tags: + - "manage" + - "manage:start" + +- name: add serverStatus logging script + template: + src: "log-mongo-serverStatus.sh.j2" + dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh" + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + mode: 0700 + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" + +- name: add serverStatus logging script to cron + cron: + name: mongostat logging job + minute: "*/3" + job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1 + become: yes + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" diff --git a/playbooks/roles/mongo_5_0/templates/log-mongo-serverStatus.sh.j2 b/playbooks/roles/mongo_5_0/templates/log-mongo-serverStatus.sh.j2 new file mode 100644 index 00000000000..04649d55ad1 --- /dev/null +++ b/playbooks/roles/mongo_5_0/templates/log-mongo-serverStatus.sh.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON +/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' diff --git a/playbooks/roles/mongo_5_0/templates/mongo_logrotate.j2 b/playbooks/roles/mongo_5_0/templates/mongo_logrotate.j2 new file mode 100644 index 00000000000..f2fb4483566 --- /dev/null +++ b/playbooks/roles/mongo_5_0/templates/mongo_logrotate.j2 @@ -0,0 +1,46 @@ +{{ mongo_log_dir }}/serverStatus.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} + +{% if is_backup_node %} +{{ mongo_log_dir }}/mongo-backup.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} +{% endif %} + +{{ mongo_log_dir }}/mongodb.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M + postrotate + /usr/bin/killall -USR1 mongod + endscript +} diff --git a/playbooks/roles/mongo_5_0/templates/mongod.conf.j2 b/playbooks/roles/mongo_5_0/templates/mongod.conf.j2 new file mode 100644 index 00000000000..b7d4b4a1efe --- /dev/null +++ b/playbooks/roles/mongo_5_0/templates/mongod.conf.j2 @@ -0,0 +1,46 @@ +# {{ ansible_managed }} +# mongodb.conf + + +storage: + # Where to store the data. + dbPath: {{ mongo_dbpath }} + # Storage Engine + engine: {{ MONGO_STORAGE_ENGINE }} + # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling + journal: +{% if mongo_enable_journal %} + enabled: true +{% else %} + enabled: false +{% endif %} +{% if MONGO_STORAGE_ENGINE_OPTIONS %} + {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} +{% endif %} + +systemLog: + #where to log + destination: file + path: "{{ mongo_logpath }}" +{% if mongo_logappend %} + logAppend: true +{% else %} + logAppend: false +{% endif %} + logRotate: {{ mongo_logrotate }} + +{% if not skip_replica_set %} +replication: + replSetName: {{ MONGO_REPL_SET }} + +security: + authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }} + keyFile: {{ mongo_key_file }} + +{% endif %} +net: + bindIp: {{ MONGO_BIND_IP }} + port: {{ mongo_port }} + + +{{ mongo_extra_conf }} diff --git a/playbooks/roles/testcourses/defaults/main.yml b/playbooks/roles/testcourses/defaults/main.yml index 6ef7ed67a12..0dad5a43c72 100644 --- a/playbooks/roles/testcourses/defaults/main.yml +++ b/playbooks/roles/testcourses/defaults/main.yml @@ -9,5 +9,5 @@ # ## # Defaults for role testcourses -# +# for sandbox, TESTCOURSES_EXPORTS is coming from sandbox-internal repo. TESTCOURSES_EXPORTS: [] diff --git a/playbooks/roles/testcourses/tasks/import_course.yml b/playbooks/roles/testcourses/tasks/import_course.yml index bb9be1f6e9d..3d36b8b4552 100644 --- a/playbooks/roles/testcourses/tasks/import_course.yml +++ b/playbooks/roles/testcourses/tasks/import_course.yml @@ -7,7 +7,7 @@ register: testcourse_checkout - name: import testcourse - shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_app_dir }} {{ course.course_id }}" + shell: ". {{ demo_edxapp_env }} && {{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_app_dir }}/{{ course.course_id }}" args: chdir: "{{ demo_edxapp_code_dir }}" when: testcourse_checkout.changed diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 6c7167d43b6..0d7da14c010 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -520,18 +520,18 @@ EOF_AUTH fi -if [[ $mongo_version == "4.2" ]]; then - cat << MONGO_VERSION >> $extra_vars_file -MONGO_4_2_ENABLED: True -MONGO_4_4_ENABLED: False -MONGO_VERSION -fi if [[ $mongo_version == "4.4" ]]; then cat << MONGO_VERSION >> $extra_vars_file MONGO_4_2_ENABLED: False MONGO_4_4_ENABLED: True MONGO_VERSION fi +if [[ $mongo_version == "5.0" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_4_4_ENABLED: False +MONGO_5_0_ENABLED: True +MONGO_VERSION +fi if [[ -n $nginx_users ]]; then cat << EOF_AUTH >> $extra_vars_file From 5874565cd51554920aea92d134adfc0d13f94039 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 23 Feb 2024 10:38:55 -0500 Subject: [PATCH 620/664] fix: Remove broken Atlas task (#7118) pull_translations now appears to be failing with a more mysterious error: ``` rmdir: failed to remove 'common/static/common/css': No such file or directory ``` This should be suppressed by the `|| true`, unless this *isn't* coming from node_prereqs_installation in pavelib... reverting for now to unblock pipeline. --- playbooks/roles/edxapp/tasks/deploy.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index b201d331715..9c30c62103f 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -246,26 +246,6 @@ - install - install:app-requirements -- name: "Pull translations using Atlas" - shell: | - set -eu -o pipefail - # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe - mkdir -p bin - curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas - chmod +x ./bin/atlas - source {{ edxapp_venv_dir }}/bin/activate - # Use production-like environment and minimal config to avoid needing dev dependencies or full config. - PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ - LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ - OPENEDX_ATLAS_PULL=true make pull_translations - rm ./bin/atlas - args: - executable: /usr/bin/bash - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - tags: - - install - # Set the npm registry # This needs to be done as root since npm is weird about # chown - https://github.com/npm/npm/issues/3565 From 86eec471c9d75ad74b16b8a7aa491a438f58d68e Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 23 Feb 2024 14:13:49 -0500 Subject: [PATCH 621/664] feat: Switch to Atlas translations in edxapp (attempt 3) (#7119) The previous attempt (#7117) was failing with an error in the call to `paver i18n_compilejs` after the Atlas-specific part of pull_translations, in the part where it runs `npm ci`. I don't have access to the npm log, but I bet it's just that our npm setup wasn't ready yet at this point in the playbook. I think we just need to move this call to after the npm setup. Part of OEP-58 and https://github.com/edx/edx-arch-experiments/issues/548 --- playbooks/roles/edxapp/tasks/deploy.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 9c30c62103f..353a3a2bcae 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,6 +357,26 @@ failed_when: "'17017' not in sandbox_test3.stdout" when: EDXAPP_SANDBOX_ENFORCE +- name: "Pull translations using Atlas (after Python dependencies and Node installed)" + shell: | + set -eu -o pipefail + # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe + mkdir -p bin + curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas + chmod +x ./bin/atlas + source {{ edxapp_venv_dir }}/bin/activate + # Use production-like environment and minimal config to avoid needing dev dependencies or full config. + PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ + LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ + OPENEDX_ATLAS_PULL=true make pull_translations + rm ./bin/atlas + args: + executable: /usr/bin/bash + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + tags: + - install + - name: compiling all py files in the edx-platform repo shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}" become_user: "{{ edxapp_user }}" From 39bd32e44e1d423ff614dd6a805fe1fd0c9fa06d Mon Sep 17 00:00:00 2001 From: Chris Pappas Date: Fri, 23 Feb 2024 14:33:24 -0500 Subject: [PATCH 622/664] feat: add default celery queue for ecommerce (worker) (#7120) --- .../ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index cc2aec9ad2b..a26831754c4 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -17,4 +17,4 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env # We exec so that celery is the child of supervisor and can be managed properly -exec {{ executable }} -A ecommerce_worker.celery_app:app worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing +exec {{ executable }} -A ecommerce_worker.celery_app:app worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing,ecommerce.default From 02bfd47545e39b1125e7292d4cb99501bc1dc348 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 23 Feb 2024 15:12:44 -0500 Subject: [PATCH 623/664] Revert "feat: Switch to Atlas translations in edxapp (attempt 3) (#7119)" (#7121) This reverts commit 86eec471c9d75ad74b16b8a7aa491a438f58d68e. --- playbooks/roles/edxapp/tasks/deploy.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 353a3a2bcae..9c30c62103f 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,26 +357,6 @@ failed_when: "'17017' not in sandbox_test3.stdout" when: EDXAPP_SANDBOX_ENFORCE -- name: "Pull translations using Atlas (after Python dependencies and Node installed)" - shell: | - set -eu -o pipefail - # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe - mkdir -p bin - curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas - chmod +x ./bin/atlas - source {{ edxapp_venv_dir }}/bin/activate - # Use production-like environment and minimal config to avoid needing dev dependencies or full config. - PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ - LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ - OPENEDX_ATLAS_PULL=true make pull_translations - rm ./bin/atlas - args: - executable: /usr/bin/bash - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - tags: - - install - - name: compiling all py files in the edx-platform repo shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}" become_user: "{{ edxapp_user }}" From 5376229b325f00f90a222915ed28a46a71d14e81 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 23 Feb 2024 16:30:29 -0500 Subject: [PATCH 624/664] temp: Add Atlas pull_translations back in, but with debug logging (#7122) --- playbooks/roles/edxapp/tasks/deploy.yml | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 9c30c62103f..94d539c8f5e 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,6 +357,33 @@ failed_when: "'17017' not in sandbox_test3.stdout" when: EDXAPP_SANDBOX_ENFORCE +- name: "Pull translations using Atlas (after Python dependencies and Node installed)" + shell: | + set -eu -o pipefail + # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe + mkdir -p bin + curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas + chmod +x ./bin/atlas + source {{ edxapp_venv_dir }}/bin/activate + # Use production-like environment and minimal config to avoid needing dev dependencies or full config. + PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ + LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ + OPENEDX_ATLAS_PULL=true make pull_translations || { + # Debugging for https://github.com/edx/edx-arch-experiments/issues/548 + echo "pull_translations failed. Contents of test_root/log/npm-install.log:" + echo "====================================================================" + cat test_root/log/npm-install.log + echo "====================================================================" + exit 1 + } + rm ./bin/atlas + args: + executable: /usr/bin/bash + chdir: "{{ edxapp_code_dir }}" + become_user: "{{ edxapp_user }}" + tags: + - install + - name: compiling all py files in the edx-platform repo shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}" become_user: "{{ edxapp_user }}" From 70eb14260dd808e50d0575cea773038e4f01cee8 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 26 Feb 2024 12:24:06 -0500 Subject: [PATCH 625/664] =?UTF-8?q?Revert=20"temp:=20Add=20Atlas=20pull=5F?= =?UTF-8?q?translations=20back=20in,=20but=20with=20debug=20logging=20(#7?= =?UTF-8?q?=E2=80=A6"=20(#7125)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 5376229b325f00f90a222915ed28a46a71d14e81. --- playbooks/roles/edxapp/tasks/deploy.yml | 27 ------------------------- 1 file changed, 27 deletions(-) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 94d539c8f5e..9c30c62103f 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -357,33 +357,6 @@ failed_when: "'17017' not in sandbox_test3.stdout" when: EDXAPP_SANDBOX_ENFORCE -- name: "Pull translations using Atlas (after Python dependencies and Node installed)" - shell: | - set -eu -o pipefail - # Pull down the Atlas binary into a bin/ dir and add it to the PATH for the Make recipe - mkdir -p bin - curl -sS -L https://github.com/openedx/openedx-atlas/releases/latest/download/atlas -o ./bin/atlas - chmod +x ./bin/atlas - source {{ edxapp_venv_dir }}/bin/activate - # Use production-like environment and minimal config to avoid needing dev dependencies or full config. - PATH="./bin/:$PATH" DJANGO_SETTINGS_MODULE=lms.envs.production \ - LMS_CFG=lms/envs/minimal.yml STUDIO_CFG=lms/envs/minimal.yml \ - OPENEDX_ATLAS_PULL=true make pull_translations || { - # Debugging for https://github.com/edx/edx-arch-experiments/issues/548 - echo "pull_translations failed. Contents of test_root/log/npm-install.log:" - echo "====================================================================" - cat test_root/log/npm-install.log - echo "====================================================================" - exit 1 - } - rm ./bin/atlas - args: - executable: /usr/bin/bash - chdir: "{{ edxapp_code_dir }}" - become_user: "{{ edxapp_user }}" - tags: - - install - - name: compiling all py files in the edx-platform repo shell: "{{ edxapp_venv_bin }}/python -m compileall -q -x '.git/.*|node_modules/.*' {{ edxapp_code_dir }}" become_user: "{{ edxapp_user }}" From 9863fdddbba2428ebc5db4cc4b6c5c71b0362737 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 27 Feb 2024 09:25:28 +0500 Subject: [PATCH 626/664] chore: secure keys --- playbooks/roles/datadog/defaults/main.yml | 2 +- playbooks/roles/datadog/tasks/main.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/datadog/defaults/main.yml b/playbooks/roles/datadog/defaults/main.yml index 79c86170a29..43050b45a20 100644 --- a/playbooks/roles/datadog/defaults/main.yml +++ b/playbooks/roles/datadog/defaults/main.yml @@ -21,4 +21,4 @@ datadog_debian_pkgs: - curl - gnupg -DATADOG_MONGODB_MONITORING: true +DATADOG_MONGODB_MONITORING: false diff --git a/playbooks/roles/datadog/tasks/main.yml b/playbooks/roles/datadog/tasks/main.yml index df18605728f..fe38f65d982 100644 --- a/playbooks/roles/datadog/tasks/main.yml +++ b/playbooks/roles/datadog/tasks/main.yml @@ -55,6 +55,7 @@ dest: "/etc/datadog-agent/datadog.yaml" regexp: "^api_key:.*" line: "api_key: {{ DATADOG_API_KEY }}" + no_log: true notify: - restart the datadog service tags: @@ -77,6 +78,7 @@ owner: "{{ datadog_user }}" group: "{{ datadog_user }}" mode: 0644 + no_log: true notify: - restart the datadog service tags: From 20dacf55a71686fb815fea177489cd726294906f Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 29 Feb 2024 10:30:35 -0500 Subject: [PATCH 627/664] fix: Fix edx-platform repo URLs after repo move (#7127) See https://github.com/edx/edx-arch-experiments/issues/558 --- CHANGELOG.md | 6 +++--- playbooks/roles/edxapp/defaults/main.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f49f49c162..a00d5832e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1398,8 +1398,8 @@ Add any new changes to the top (right below this line). - 2015-05-27 - Role: edxapp - - Removed deprecated variables EDXAPP_PLATFORM_TWITTER_URL, EDXAPP_PLATFORM_MEETUP_URL, EDXAPP_PLATFORM_LINKEDIN_URL, and EDXAPP_PLATFORM_GOOGLE_PLUS_URL in favor of EDXAPP_SOCIAL_MEDIA_FOOTER_URLS. These variables haven't been used in edx-platform since March 17, 2015 (when https://github.com/edx/edx-platform/pull/7383 was merged). This change is backwards incompatible with versions of edx-platform from before March 17, 2015. - - Added EDXAPP_MOBILE_STORE_URLS and EDXAPP_FOOTER_ORGANIZATION_IMAGE variables, used in https://github.com/edx/edx-platform/pull/8175 (v3 version of the edx.org footer). + - Removed deprecated variables EDXAPP_PLATFORM_TWITTER_URL, EDXAPP_PLATFORM_MEETUP_URL, EDXAPP_PLATFORM_LINKEDIN_URL, and EDXAPP_PLATFORM_GOOGLE_PLUS_URL in favor of EDXAPP_SOCIAL_MEDIA_FOOTER_URLS. These variables haven't been used in edx-platform since March 17, 2015 (when https://github.com/openedx/edx-platform/pull/7383 was merged). This change is backwards incompatible with versions of edx-platform from before March 17, 2015. + - Added EDXAPP_MOBILE_STORE_URLS and EDXAPP_FOOTER_ORGANIZATION_IMAGE variables, used in https://github.com/openedx/edx-platform/pull/8175 (v3 version of the edx.org footer). - We now remove the default syslog.d conf file (50-default.conf) this will @@ -1526,7 +1526,7 @@ Add any new changes to the top (right below this line). - 2014-05-28 - Role: Edxapp - The repo.txt requirements file is no longer being processed in anyway. This file was removed from edxplatform - via pull #3487(https://github.com/edx/edx-platform/pull/3487) + via pull #3487(https://github.com/openedx/edx-platform/pull/3487) - 2014-05-19 diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 26a4b2aa59c..353bcbbc0b1 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1754,7 +1754,7 @@ edxapp_theme_source_repo: 'https://{{ COMMON_GIT_MIRROR }}/Stanford-Online/edx-t EDXAPP_THEME_VERSION: 'master' # make this the public URL instead of writable -edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/edx/edx-platform.git" +edx_platform_repo: "https://{{ COMMON_GIT_MIRROR }}/openedx/edx-platform.git" # `EDX_PLATFORM_VERSION` can be anything that git recognizes as a commit # reference, including a tag, a branch name, or a commit hash EDX_PLATFORM_VERSION: 'release' From 3ce4019ce105b86db1f974932f3f4367db8b0f2b Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 1 Mar 2024 16:04:18 -0500 Subject: [PATCH 628/664] feat: Pull edxapp translations via Atlas (#7128) This is to support OEP-58: https://docs.openedx.org/projects/openedx-proposals/en/latest/architectural-decisions/oep-0058-arch-translations-management.html Changes since the https://github.com/openedx/configuration/pull/7119 attempt: - Move to just after main requirements are installed (just for faster feedback during testing, really) - Use already-installed `openedx-atlas` - Use variant-agnostic `EDX_PLATFORM_SETTINGS` rather than `DJANGO_SETTINGS_MODULE` - Use Ansible `environment` field for env vars - Drop `set -eu -o pipefail` since the script is now simple Most importantly, https://github.com/openedx/edx-platform/pull/34306 has since merged, so we don't have to worry about the NPM conflicts we were getting previously. --- playbooks/roles/edxapp/tasks/deploy.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 9c30c62103f..39427001bca 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -216,6 +216,29 @@ - install - install:app-requirements +# Pulling Atlas translations into the repo needs to happen after +# Python dependencies have been installed. Note: This task leaves the +# git working directory in a "dirty" state. +- name: "Pull translations using Atlas" + shell: | + source "{{ edxapp_venv_dir }}/bin/activate" + make pull_translations + args: + executable: /usr/bin/bash + chdir: "{{ edxapp_code_dir }}" + environment: + # Use production Django settings because otherwise debug_toolbar will be + # referenced and cause an error (we don't have developer Python deps installed.) + EDX_PLATFORM_SETTINGS: production + # Use minimal configs because the real configs aren't installed until + # later in the playbook. + LMS_CFG: lms/envs/minimal.yml + STUDIO_CFG: lms/envs/minimal.yml + OPENEDX_ATLAS_PULL: true + become_user: "{{ edxapp_user }}" + tags: + - install + # If using CAS and you have a function for mapping attributes, install # the module here. The next few tasks set up the python code sandbox - name: install CAS attribute module From e7d069665f68b815eb6701645664a9a23acc83b3 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 6 Mar 2024 11:55:55 +0500 Subject: [PATCH 629/664] chore: build sandbox with mongo 5.0 --- playbooks/edx_continuous_integration.yml | 3 --- util/jenkins/ansible-provision.sh | 13 ------------- 2 files changed, 16 deletions(-) diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index c39ecc00b91..ff75ba6bbcf 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -25,10 +25,7 @@ - role: edxlocal tags: edxlocal - memcache - - role: mongo_4_4 - when: MONGO_4_4_ENABLED - role: mongo_5_0 - when: MONGO_5_0_ENABLED - role: redis - { role: "edxapp", celery_worker: True, when: edxapp_containerized is defined and not edxapp_containerized } - { role: "edxapp", when: edxapp_containerized is defined and not edxapp_containerized } diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 0d7da14c010..292c84795a2 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -520,19 +520,6 @@ EOF_AUTH fi -if [[ $mongo_version == "4.4" ]]; then - cat << MONGO_VERSION >> $extra_vars_file -MONGO_4_2_ENABLED: False -MONGO_4_4_ENABLED: True -MONGO_VERSION -fi -if [[ $mongo_version == "5.0" ]]; then - cat << MONGO_VERSION >> $extra_vars_file -MONGO_4_4_ENABLED: False -MONGO_5_0_ENABLED: True -MONGO_VERSION -fi - if [[ -n $nginx_users ]]; then cat << EOF_AUTH >> $extra_vars_file NGINX_USERS: $nginx_users From 5746206c7577e6497b91bf4d961d602b96f2d078 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 15 Mar 2024 16:38:00 +0500 Subject: [PATCH 630/664] refactor: install datadog using Ansible Galaxy --- playbooks/roles/datadog/defaults/main.yml | 25 +----- playbooks/roles/datadog/handlers/main.yml | 5 -- playbooks/roles/datadog/tasks/main.yml | 84 ++----------------- .../roles/datadog/templates/conf.yaml.j2 | 9 -- 4 files changed, 10 insertions(+), 113 deletions(-) delete mode 100644 playbooks/roles/datadog/handlers/main.yml delete mode 100644 playbooks/roles/datadog/templates/conf.yaml.j2 diff --git a/playbooks/roles/datadog/defaults/main.yml b/playbooks/roles/datadog/defaults/main.yml index 43050b45a20..a557191cfc9 100644 --- a/playbooks/roles/datadog/defaults/main.yml +++ b/playbooks/roles/datadog/defaults/main.yml @@ -1,24 +1,5 @@ --- -DATADOG_API_KEY: "SPECIFY_KEY_HERE" -DATADOG_SITE: datadoghq.com - -DATADOG_AGENT_VERSION: '1:7.50.3-1' - -datadog_user: "dd-agent" - -# Variables for Datadog MongoDB Monitoring -datadog_mongo_host: 127.0.0.1 -datadog_mongo_port: 27017 -datadog_mongo_username: datadog -datadog_mongo_password: password -datadog_mongo_db: admin -datadog_authsource_db: admin - -datadog_apt_key: "0xAD9589B7" -datadog_debian_pkgs: - - apt-transport-https - - curl - - gnupg - -DATADOG_MONGODB_MONITORING: false +datadog_api_key: "{{ datadog_api_key }}" +datadog_config: "{{ datadog_config }}" +datadog_checks: "{{ datadog_checks }}" diff --git a/playbooks/roles/datadog/handlers/main.yml b/playbooks/roles/datadog/handlers/main.yml deleted file mode 100644 index 0ab42448b67..00000000000 --- a/playbooks/roles/datadog/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart the datadog service - service: - name: datadog-agent - state: restarted diff --git a/playbooks/roles/datadog/tasks/main.yml b/playbooks/roles/datadog/tasks/main.yml index fe38f65d982..d55f5c04277 100644 --- a/playbooks/roles/datadog/tasks/main.yml +++ b/playbooks/roles/datadog/tasks/main.yml @@ -13,82 +13,12 @@ # - common # - datadog # -- name: Install debian needed pkgs - apt: - name: "{{ datadog_debian_pkgs }}" - state: present - tags: - - datadog -- name: Add apt key for datadog - apt_key: - id: "33EE313BAD9589B7" - url: "{{ COMMON_UBUNTU_APT_KEYSERVER }}{{ datadog_apt_key }}" - state: present - tags: - - datadog +- name: Install Datadog role from Ansible Galaxy + ansible.builtin.command: ansible-galaxy install datadog.datadog + delegate_to: localhost + become: false -- name: Install apt repository for datadog - apt_repository: - repo: 'deb http://apt.datadoghq.com/ stable 7' - state: present - update_cache: yes - tags: - - datadog - -- name: Install datadog agent - apt: - name: "datadog-agent={{ DATADOG_AGENT_VERSION }}" - tags: - - datadog - -- name: Bootstrap config - shell: cp datadog.yaml.example datadog.yaml - args: - chdir: /etc/datadog-agent/ - creates: /etc/datadog-agent/datadog.yaml - tags: - - datadog - -- name: Update api-key - lineinfile: - dest: "/etc/datadog-agent/datadog.yaml" - regexp: "^api_key:.*" - line: "api_key: {{ DATADOG_API_KEY }}" - no_log: true - notify: - - restart the datadog service - tags: - - datadog - -- name: Update site - lineinfile: - dest: "/etc/datadog-agent/datadog.yaml" - regexp: "^# site:.*" - line: "site: {{ DATADOG_SITE }}" - notify: - - restart the datadog service - tags: - - datadog - -- name: Write MongoDB monitoring config - template: - src: "conf.yaml.j2" - dest: "/etc/datadog-agent/conf.d/mongo.d/conf.yaml" - owner: "{{ datadog_user }}" - group: "{{ datadog_user }}" - mode: 0644 - no_log: true - notify: - - restart the datadog service - tags: - - datadog - when: DATADOG_MONGODB_MONITORING | default(false) | bool - -- name: Ensure started and enabled - service: - name: datadog-agent - state: started - enabled: yes - tags: - - datadog +- name: Install datadog Agent + include_role: + name: datadog.datadog diff --git a/playbooks/roles/datadog/templates/conf.yaml.j2 b/playbooks/roles/datadog/templates/conf.yaml.j2 deleted file mode 100644 index c19a1eecc9b..00000000000 --- a/playbooks/roles/datadog/templates/conf.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -init_config: -instances: - - hosts: - - {{ datadog_mongo_host }}:{{ datadog_mongo_port }} - username: {{ datadog_mongo_username }} - password: {{ datadog_mongo_password }} - database: {{ datadog_mongo_db }} - options: - authSource: {{ datadog_authsource_db }} From 3a436ff3fb0fa93b40dfb8f43a9754596e3ffbca Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 20 Mar 2024 16:08:54 -0400 Subject: [PATCH 631/664] feat: Allow instrumenting edxapp with Datadog APM (#7136) Introduces `COMMON_ENABLE_DATADOG_APP` for APM instrumention with Datadog, as distinct from `COMMON_ENABLE_DATADOG` that just installs the agent (for infrastructure monitoring). See https://github.com/edx/edx-arch-experiments/issues/574 --- CHANGELOG.md | 3 +++ playbooks/roles/common_vars/defaults/main.yml | 3 +++ playbooks/roles/edxapp/tasks/deploy.yml | 17 +++++++++++++++++ .../edxapp/templates/edx/app/edxapp/cms.sh.j2 | 11 +++++++---- .../edxapp/templates/edx/app/edxapp/lms.sh.j2 | 10 ++++++---- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a00d5832e70..9f524eb0fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2024-03-20 + - Add `COMMON_ENABLE_DATADOG_APP` for APM instrumention, supported in LMS and CMS so far. Disabled by default. + - 2024-01-25 - Role: mfe - Added `MFE_ENVIRONMENT_DEFAULT_EXTRA` to allow operators to add extra environment variables to all MFEs when diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 1c00476dfc0..b81f99637ea 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -98,7 +98,10 @@ COMMON_MYSQL_MIGRATE_PASS: 'password' COMMON_MONGO_READ_ONLY_USER: 'read_only' COMMON_MONGO_READ_ONLY_PASS: !!null +# Enable installation of the Datadog agent (infrastructure monitoring) COMMON_ENABLE_DATADOG: False +# Enable APM monitoring with Datadog (metrics, traces, and logs) +COMMON_ENABLE_DATADOG_APP: False COMMON_ENABLE_NGINXTRA: False COMMON_ENABLE_SPLUNKFORWARDER: False COMMON_ENABLE_NEWRELIC: False diff --git a/playbooks/roles/edxapp/tasks/deploy.yml b/playbooks/roles/edxapp/tasks/deploy.yml index 39427001bca..d27bdab0782 100644 --- a/playbooks/roles/edxapp/tasks/deploy.yml +++ b/playbooks/roles/edxapp/tasks/deploy.yml @@ -216,6 +216,23 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w {{ item.extra_args|default('') }}" + virtualenv: "{{ edxapp_venv_dir }}" + state: present + become_user: "{{ edxapp_user }}" + register: edxapp_install_datadog_reqs + until: edxapp_install_datadog_reqs is succeeded + retries: 5 + delay: 15 + tags: + - install + - install:app-requirements + # Pulling Atlas translations into the repo needs to happen after # Python dependencies have been installed. Note: This task leaves the # git working directory in a "dirty" state. diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index 7b4c8cbd1dd..a72f43f8043 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -4,8 +4,10 @@ {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} +{% set executable = edxapp_venv_bin + '/gunicorn' %} + {% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/gunicorn' %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %} export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ EDXAPP_CMS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}" export NEW_RELIC_APP_NAME="{{ EDXAPP_NEWRELIC_CMS_APPNAME }}" @@ -16,10 +18,11 @@ if command -v ec2metadata >/dev/null 2>&1; then export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" fi export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" +{% endif -%} -{% else %} -{% set executable = edxapp_venv_bin + '/gunicorn' %} -{% endif %} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} +{% endif -%} export PORT="{{ edxapp_cms_gunicorn_port }}" export ADDRESS="{{ edxapp_cms_gunicorn_host }}" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index 8cf6e703f3f..3c1e2eff880 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -4,8 +4,10 @@ {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %} +{% set executable = edxapp_venv_bin + '/gunicorn' %} + {% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/gunicorn' %} +{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %} export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ EDXAPP_LMS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}" export NEW_RELIC_APP_NAME="{{ EDXAPP_NEWRELIC_LMS_APPNAME }}" @@ -16,10 +18,10 @@ if command -v ec2metadata >/dev/null 2>&1; then export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID" fi export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" +{% endif -%} -{% else %} -{% set executable = edxapp_venv_bin + '/gunicorn' %} - +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} {% endif -%} export PORT="{{ edxapp_lms_gunicorn_port }}" From c559067d257c406b3f2938653bfd0c99168b1efd Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 21 Mar 2024 11:33:10 -0400 Subject: [PATCH 632/664] docs: Add a deprecation warning. https://github.com/openedx/public-engineering/issues/51 has all the details. Also drop a bunch of trailing whitespace that was in this file. --- README.rst | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 011fa1a6e82..e09c88e7574 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,19 @@ Configuration Management ######################## -This repository is a collection of tools and scripts that edx.org uses to deploy +DEPRECATION WARNING +******************* + +This repository is Deprecated and is planned to be archived after the Redwood +Cut (Late April 2024). See https://github.com/openedx/public-engineering/issues/51 +for more details and up-to-date information. + +Description +*********** + +This repository is a collection of tools and scripts that edx.org uses to deploy openedx. The purpose of this repository is to share portions of our toolchain -with the community. This repository is *not* the best way to get started running +with the community. This repository is *not* the best way to get started running openedx. For that, please look at `Open EdX Installation options`_, which contains links to the recommended paths for new installations. @@ -28,7 +38,7 @@ What is in this Repo? be used to configure individual services in the openedx platform. See `Open EdX Installation options`_ before trying to use any of the scripts in this directory. -* `docker `__: This directory contains dockerfiles that can be used to +* `docker `__: This directory contains dockerfiles that can be used to test that playbooks execute cleanly. See `Makefiles `__ for Documentation on how to run these containers. * `requirements `__ : inputs for `pip-compile `__ @@ -44,13 +54,13 @@ Roadmap ******* This repository is in ``sustained`` status. The goal is to deprecate this codebase -and move the deployment code into the repos with the application code. +and move the deployment code into the repos with the application code. -With the adoption of containerized application platforms like `Kubernetes -`__, the tools in this repository are complex +With the adoption of containerized application platforms like `Kubernetes +`__, the tools in this repository are complex and inappropriate for building small single purpose containers. -At edx.org, we are focusing on deployment of applications using `Terraform +At edx.org, we are focusing on deployment of applications using `Terraform `__ and `Kubernetes `__. We hope to provide open source tooling for this soon. @@ -59,7 +69,7 @@ Contributing ************ * Bugfixes: If you would like to contribute a bugfix to this codebase, please open - a pull request. A bot will automatically walk your contribution through the + a pull request. A bot will automatically walk your contribution through the `Open Source Contribution process `__. From b4f003503adca3e6d039cdc61b2ff783f67fd436 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 22 Mar 2024 10:03:54 -0400 Subject: [PATCH 633/664] fix: Skip Datadog instrumentation of pymongo (#7138) --- playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 | 2 ++ playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index a72f43f8043..1410dde6ce2 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -22,6 +22,8 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} +# Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 +export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} export PORT="{{ edxapp_cms_gunicorn_port }}" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index 3c1e2eff880..26ce0039906 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -22,6 +22,8 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} +# Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 +export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} export PORT="{{ edxapp_lms_gunicorn_port }}" From 86df9ed1ff8bee8dc7cfb3d2f31de61e7587a91c Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 22 Mar 2024 15:05:36 -0400 Subject: [PATCH 634/664] feat: Add Datadog service tags for edxapp (#7139) --- playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 | 1 + playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index 1410dde6ce2..f55256758e3 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -22,6 +22,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} +export DD_TAGS="service:edxapp-cms" # Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index 26ce0039906..2574b9e9488 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -22,6 +22,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} +export DD_TAGS="service:edxapp-lms" # Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} From 57c5e553e9a239264af327d3a71ef239a37be086 Mon Sep 17 00:00:00 2001 From: Syed Awais Ali Date: Tue, 26 Mar 2024 12:26:16 +0500 Subject: [PATCH 635/664] chore: see slow qyer logs tables --- .../export_slow_query_logs.py | 89 ++++++++++--------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/util/jenkins/export_slow_logs/export_slow_query_logs.py b/util/jenkins/export_slow_logs/export_slow_query_logs.py index 9cb8813dd9b..484935d24e8 100644 --- a/util/jenkins/export_slow_logs/export_slow_query_logs.py +++ b/util/jenkins/export_slow_logs/export_slow_query_logs.py @@ -93,49 +93,52 @@ def rds_controller(rds_list, username, password): connection = pymysql.connect(host=rds_host_endpoint, port=rds_port, user=username, password=password) cursor = connection.cursor() - print(f"Checking slow log on RDS HOST {rds_host_endpoint}") - cursor.execute(""" - SELECT * - FROM mysql.slow_log - WHERE start_time > DATE_ADD(NOW(), INTERVAL -1 HOUR); - """) - rds_result = cursor.fetchall() - cursor.close() - connection.close() - if len(rds_result) > 0: - cw_logs = [] - sequencetoken = None - client = CWBotoWrapper() - loggroupname= "/slowlogs/" + rds_host_endpoint - try: - client.create_log_group(logGroupName=loggroupname) - print(('Created CloudWatch log group named "%s"', loggroupname)) - except ClientError: - print(('CloudWatch log group named "%s" already exists', loggroupname)) - LOG_STREAM = time.strftime('%Y-%m-%d') + "/[$LATEST]" + uuid.uuid4().hex - client.create_log_stream(logGroupName=loggroupname, logStreamName=LOG_STREAM) - for tables in rds_result: - temp = {} - temp["timestamp"] = int(tables[0].strftime("%s")) * 1000 - temp["message"] = "User@Host: " + str(tables[1]) + \ - "Query_time: " + str(tables[2]) + " Lock_time: " + str(tables[3]) + \ - " Rows_sent: " + str(tables[4]) + " Rows_examined: " + str(tables[5]) +\ - "Slow Query: " + str(tables[10]) - cw_logs.append(temp) - if sequencetoken == None: - response = client.put_log_events( - logGroupName=loggroupname, - logStreamName=LOG_STREAM, - logEvents=cw_logs - ) - else: - response = client.put_log_events( - logGroupName=loggroupname, - logStreamName=LOG_STREAM, - logEvents=cw_logs, - sequenceToken=sequencetoken - ) - sequencetoken = response["nextSequenceToken"] + try: + print(f"Checking slow log on RDS HOST {rds_host_endpoint}") + cursor.execute(""" + SELECT * + FROM mysql.slow_log + WHERE start_time > DATE_ADD(NOW(), INTERVAL -1 HOUR); + """) + rds_result = cursor.fetchall() + cursor.close() + connection.close() + if len(rds_result) > 0: + cw_logs = [] + sequencetoken = None + client = CWBotoWrapper() + loggroupname= "/slowlogs/" + rds_host_endpoint + try: + client.create_log_group(logGroupName=loggroupname) + print(('Created CloudWatch log group named "%s"', loggroupname)) + except ClientError: + print(('CloudWatch log group named "%s" already exists', loggroupname)) + LOG_STREAM = time.strftime('%Y-%m-%d') + "/[$LATEST]" + uuid.uuid4().hex + client.create_log_stream(logGroupName=loggroupname, logStreamName=LOG_STREAM) + for tables in rds_result: + temp = {} + temp["timestamp"] = int(tables[0].strftime("%s")) * 1000 + temp["message"] = "User@Host: " + str(tables[1]) + \ + "Query_time: " + str(tables[2]) + " Lock_time: " + str(tables[3]) + \ + " Rows_sent: " + str(tables[4]) + " Rows_examined: " + str(tables[5]) +\ + "Slow Query: " + str(tables[10]) + cw_logs.append(temp) + if sequencetoken == None: + response = client.put_log_events( + logGroupName=loggroupname, + logStreamName=LOG_STREAM, + logEvents=cw_logs + ) + else: + response = client.put_log_events( + logGroupName=loggroupname, + logStreamName=LOG_STREAM, + logEvents=cw_logs, + sequenceToken=sequencetoken + ) + sequencetoken = response["nextSequenceToken"] + except Exception as e: + print(e) @click.command() From 128ba1c9a9912fd291dec07d56fef91c71c1b651 Mon Sep 17 00:00:00 2001 From: Syed Imran Hassan <45480841+syedimranhassan@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:50:27 +0500 Subject: [PATCH 636/664] chore: Update check rds config script to ignore whitelist RDS --- util/check_rds_configs/check_rds_configs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/check_rds_configs/check_rds_configs.py b/util/check_rds_configs/check_rds_configs.py index 1f360d96468..86e84eb2fd5 100644 --- a/util/check_rds_configs/check_rds_configs.py +++ b/util/check_rds_configs/check_rds_configs.py @@ -112,9 +112,10 @@ def cli(db_engine, ignore): db_cluster_identifier = cluster['DBClusterIdentifier'] tags = rds.list_tags_for_resource(ResourceName=arn)['TagList'] print("Checking cluster tags on DB cluster {}".format(db_cluster_identifier)) - exit_status, clusters_without_tags = check_tags(clusters_without_tags, db_cluster_identifier, tags) - if cluster['CopyTagsToSnapshot'] == False: - cluster_with_disabled_snapshot_tags.append(cluster['DBClusterIdentifier']) + if db_cluster_identifier not in ignore_rds and "test" not in db_cluster_identifier: + exit_status, clusters_without_tags = check_tags(clusters_without_tags, db_cluster_identifier, tags) + if cluster['CopyTagsToSnapshot'] == False: + cluster_with_disabled_snapshot_tags.append(cluster['DBClusterIdentifier']) for instance in cluster['DBClusterMembers']: db_identifier = instance['DBInstanceIdentifier'] From 4c4842d57f7cb5c8f30ed3d2b47cd53d9b64b478 Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:34:30 +0500 Subject: [PATCH 637/664] fix: Pass Learning MFE branch for deployment (#7148) --- playbooks/learning.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/learning.yml b/playbooks/learning.yml index f939159cef9..ca771b75019 100644 --- a/playbooks/learning.yml +++ b/playbooks/learning.yml @@ -10,6 +10,7 @@ roles: - role: mfe MFE_NAME: learning + MFE_VERSION: '{{ LEARNING_MFE_VERSION }}' - role: splunkforwarder when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure From 5a38ded76c6d8bf9a488be313ccf590aa62537e0 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 5 Apr 2024 10:32:23 +0500 Subject: [PATCH 638/664] chore: enable dd agent role for prospectus and conductor --- playbooks/conductor.yml | 2 ++ playbooks/prospectus.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/playbooks/conductor.yml b/playbooks/conductor.yml index 11522c137fa..36bdc85f8b2 100644 --- a/playbooks/conductor.yml +++ b/playbooks/conductor.yml @@ -23,3 +23,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog + when: COMMON_ENABLE_DATADOG diff --git a/playbooks/prospectus.yml b/playbooks/prospectus.yml index 4c2983e052e..98cbcb2c1d2 100644 --- a/playbooks/prospectus.yml +++ b/playbooks/prospectus.yml @@ -32,3 +32,5 @@ when: COMMON_ENABLE_SPLUNKFORWARDER|bool and PROSPECTUS_ENABLE_POST_BUILD|bool - role: newrelic_infrastructure when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE|bool and PROSPECTUS_ENABLE_POST_BUILD|bool + - role: datadog + when: COMMON_ENABLE_DATADOG|bool and PROSPECTUS_ENABLE_POST_BUILD|bool From e3b31227c0461c3d39f91d803971eb643cc8d071 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Fri, 5 Apr 2024 16:40:01 +0500 Subject: [PATCH 639/664] chore: set env var DD_TAGS for forum --- playbooks/roles/forum/defaults/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index f377c9fcde9..61a03289775 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -49,6 +49,9 @@ FORUM_NEW_RELIC_ENABLE: '{{ COMMON_ENABLE_NEWRELIC_APP }}' FORUM_NEW_RELIC_LICENSE_KEY: '{{ NEWRELIC_LICENSE_KEY | default("") }}' FORUM_NEW_RELIC_APP_NAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-forum" +# Datadog CONFIG +FORUM_DD_TAGS: "service:forum" + FORUM_WORKER_PROCESSES: "4" FORUM_LISTEN_HOST: "0.0.0.0" FORUM_LISTEN_PORT: "4567" @@ -87,7 +90,7 @@ forum_base_env: &forum_base_env DATA_DIR: "{{ forum_data_dir }}" LISTEN_HOST: "{{ FORUM_LISTEN_HOST }}" LISTEN_PORT: "{{ FORUM_LISTEN_PORT }}" - + DD_TAGS: "{{ FORUM_DD_TAGS }}" forum_env: <<: *forum_base_env From d9e76dab2bbc438eca819fbef1878dbdb87802d0 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 3 Apr 2024 17:04:27 +0000 Subject: [PATCH 640/664] feat: Run all Django services with ddtrace (Datadog APM) when enabled This expands the changes in edxapp to other Django services. See https://github.com/edx/edx-arch-experiments/issues/573 --- playbooks/roles/edx_django_service/tasks/main.yml | 13 +++++++++++++ .../templates/edx/app/app/app.sh.j2 | 11 +++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/edx_django_service/tasks/main.yml b/playbooks/roles/edx_django_service/tasks/main.yml index b71fe5f979e..14004b11dcc 100644 --- a/playbooks/roles/edx_django_service/tasks/main.yml +++ b/playbooks/roles/edx_django_service/tasks/main.yml @@ -212,6 +212,19 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w {{ item.extra_args|default('') }}" + virtualenv: "{{ edx_django_service_venv_dir }}" + state: present + become_user: "{{ edx_django_service_user }}" + tags: + - install + - install:app-requirements + - name: Check for existing make_migrate container command: "docker ps -aq --filter name='{{ edx_django_service_name }}.make_migrate'" register: edx_django_service_make_migrate_container diff --git a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 index f77a68f132f..3cfaacfddef 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 @@ -4,13 +4,11 @@ {% set edx_django_service_venv_bin = edx_django_service_venv_dir + "/bin" %} -{% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = edx_django_service_venv_bin + '/newrelic-admin run-program ' + edx_django_service_venv_bin + '/gunicorn' %} -{% else %} {% set executable = edx_django_service_venv_bin + '/gunicorn' %} -{% endif %} {% if COMMON_ENABLE_NEWRELIC_APP %} +{% set executable = edx_django_service_venv_bin + '/newrelic-admin run-program ' + executable %} + export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ edx_django_service_enable_newrelic_distributed_tracing }}" export NEW_RELIC_APP_NAME="{{ edx_django_service_newrelic_appname }}" HOSTNAME=$(hostname) @@ -22,6 +20,11 @@ fi export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% endif -%} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = edx_django_service_venv_bin + '/ddtrace-run ' + executable %} +export DD_TAGS="service:{{ edx_django_service_name }}" +{% endif -%} + export EDX_REST_API_CLIENT_NAME="{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_django_service_name }}" source {{ edx_django_service_home }}/{{ edx_django_service_name }}_env From 734110064675d03a8c1af8fa525555a0f7e31917 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 3 Apr 2024 17:30:29 +0000 Subject: [PATCH 641/664] feat: Enable Datadog APM in sandboxes when Datadog is enabled overall --- util/jenkins/ansible-provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 292c84795a2..63e462652c2 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -547,6 +547,7 @@ COMMON_USER_INFO: USER_CMD_PROMPT: '[$name_tag] ' COMMON_ENABLE_NEWRELIC_APP: $enable_newrelic COMMON_ENABLE_DATADOG: $enable_datadog +COMMON_ENABLE_DATADOG_APP: $enable_datadog COMMON_OAUTH_BASE_URL: "https://${deploy_host}" FORUM_NEW_RELIC_ENABLE: $enable_newrelic ENABLE_PERFORMANCE_COURSE: $performance_course From fcb8595cd95e49289282fcda61ddcec86f274704 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Mon, 8 Apr 2024 15:32:30 +0500 Subject: [PATCH 642/664] chore: set max size limit for jounral --- playbooks/roles/vhost/tasks/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/playbooks/roles/vhost/tasks/main.yml b/playbooks/roles/vhost/tasks/main.yml index 1374ca892a2..a5ca407e69f 100644 --- a/playbooks/roles/vhost/tasks/main.yml +++ b/playbooks/roles/vhost/tasks/main.yml @@ -70,6 +70,22 @@ when: COMMON_OBJECT_STORE_LOG_SYNC and not (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty') +- name: Set maximum disk space usage for systemd journal + lineinfile: + path: /etc/systemd/journald.conf + regexp: '^#?SystemMaxUse=' + line: 'SystemMaxUse=1G' + state: present + register: journald_config_line + when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' + +- name: Restart systemd-journald (ubuntu/debian) + service: + name: systemd-journald + state: restarted + when: > + journald_config_line.changed and ansible_distribution in common_debian_variants + - name: Update /etc/dhcp/dhclient.conf template: src: etc/dhcp/dhclient.conf.j2 From 0e6a9f78d515e7869ef7e80cd79860153701fe67 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Tue, 9 Apr 2024 12:07:03 -0400 Subject: [PATCH 643/664] feat: Update node version to 18. edx-platform has been updated to use node 18. Update deploy to use that version as well. --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 353bcbbc0b1..3cc1a16a9b6 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1122,8 +1122,8 @@ edxapp_nodeenv_bin: "{{ edxapp_nodeenv_dir }}/bin" edxapp_npm_dir: "{{ edxapp_app_dir }}/.npm" edxapp_npm_bin: "{{ edxapp_npm_dir }}/bin" edxapp_settings: '{{ EDXAPP_SETTINGS }}' -EDXAPP_NODE_VERSION: "16" -EDXAPP_NPM_VERSION: "8.5.0" +EDXAPP_NODE_VERSION: "18" +EDXAPP_NPM_VERSION: "10.5.1" # This is where node installs modules, not node itself edxapp_node_bin: "{{ edxapp_code_dir }}/node_modules/.bin" edxapp_user: edxapp From a3eed493b4e072d42df53143066e2a5ab56df46d Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Wed, 10 Apr 2024 09:13:58 -0400 Subject: [PATCH 644/664] Revert "feat: Update node version to 18." --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 3cc1a16a9b6..353bcbbc0b1 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1122,8 +1122,8 @@ edxapp_nodeenv_bin: "{{ edxapp_nodeenv_dir }}/bin" edxapp_npm_dir: "{{ edxapp_app_dir }}/.npm" edxapp_npm_bin: "{{ edxapp_npm_dir }}/bin" edxapp_settings: '{{ EDXAPP_SETTINGS }}' -EDXAPP_NODE_VERSION: "18" -EDXAPP_NPM_VERSION: "10.5.1" +EDXAPP_NODE_VERSION: "16" +EDXAPP_NPM_VERSION: "8.5.0" # This is where node installs modules, not node itself edxapp_node_bin: "{{ edxapp_code_dir }}/node_modules/.bin" edxapp_user: edxapp From 2b6662eb6b01cba6c0b33ddc14b3e08df5a3d442 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 12 Apr 2024 14:33:56 -0400 Subject: [PATCH 645/664] feat: Apply DD_TRACE_LOG_STREAM_HANDLER hack to Ansible services (#7155) Backports the trace-debug log fix from edx/helm-charts. Only observed in ecommerce and analytics-api, but I've added the fix to edxapp as well just in case it shows up there later (and for general consistency). See https://github.com/edx/edx-arch-experiments/issues/591 --- .../edx_django_service/templates/edx/app/app/app.sh.j2 | 10 ++++++++++ .../roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 | 4 ++++ .../roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 index 3cfaacfddef..6f7ab4e11b0 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 @@ -23,6 +23,16 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edx_django_service_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ edx_django_service_name }}" +# Workaround for +# https://github.com/edx/edx-arch-experiments/issues/591 (heavy +# streams of trace-debug logs from ddtrace.) +# +# ddtrace is behaving as if DD_TRACE_DEBUG=true, even though that +# should be false by default, and we're not setting it anywhere that +# we can find. Overriding it to false doesn't work, and none of the +# other trace-related configs that are documented seem to help, but +# from testing DD_TRACE_LOG_STREAM_HANDLER=false seems to help. +export DD_TRACE_LOG_STREAM_HANDLER=false {% endif -%} export EDX_REST_API_CLIENT_NAME="{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_django_service_name }}" diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index f55256758e3..d6909a2b6a8 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -23,6 +23,10 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:edxapp-cms" +# Copied from edx_django_service playbook for consistency; Datadog +# trace debug logging issue doesn't actually affect edxapp for some +# reason. +export DD_TRACE_LOG_STREAM_HANDLER=false # Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index 2574b9e9488..46cbe02cee4 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -23,6 +23,10 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:edxapp-lms" +# Copied from edx_django_service playbook for consistency; Datadog +# trace debug logging issue doesn't actually affect edxapp for some +# reason. +export DD_TRACE_LOG_STREAM_HANDLER=false # Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580 export DD_TRACE_PYMONGO_ENABLED=false {% endif -%} From 0717f719bbb5622416133c18ddf548a81af8862a Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 16 Apr 2024 16:02:44 +0500 Subject: [PATCH 646/664] feat: Run ddtarce for insights and ecomworker --- playbooks/roles/ecomworker/tasks/main.yml | 13 +++++++++++++ .../templates/edx/app/ecomworker/ecomworker.sh.j2 | 12 ++++++++---- playbooks/roles/insights/tasks/main.yml | 13 +++++++++++++ .../templates/edx/app/insights/insights.sh.j2 | 12 ++++++++---- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/playbooks/roles/ecomworker/tasks/main.yml b/playbooks/roles/ecomworker/tasks/main.yml index c99eb68e6ec..e68de294d29 100644 --- a/playbooks/roles/ecomworker/tasks/main.yml +++ b/playbooks/roles/ecomworker/tasks/main.yml @@ -21,6 +21,19 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w" + virtualenv: '{{ ecommerce_worker_home }}/venvs/{{ ecommerce_worker_service_name }}' + state: present + become_user: "{{ ecommerce_worker_user }}" + tags: + - install + - install:app-requirements + - name: write out the supervisor wrapper template: src: edx/app/ecomworker/ecomworker.sh.j2 diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index a26831754c4..c6a33657be2 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -3,18 +3,22 @@ # {{ ansible_managed }} {% set ecommerce_worker_venv_bin = ecommerce_worker_home + '/venvs/' + ecommerce_worker_service_name + '/bin' %} -{% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = ecommerce_worker_venv_bin + '/newrelic-admin run-program ' + ecommerce_worker_venv_bin + '/celery' %} -{% else %} + {% set executable = ecommerce_worker_venv_bin + '/celery' %} -{% endif %} {% if COMMON_ENABLE_NEWRELIC_APP %} +{% set executable = ecommerce_worker_venv_bin + '/newrelic-admin run-program ' + ecommerce_worker_venv_bin + '/celery' %} + export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED='{{ ECOMMERCE_WORKER_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}' export NEW_RELIC_APP_NAME='{{ ECOMMERCE_WORKER_NEWRELIC_APPNAME }}' export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' {% endif -%} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = ecommerce_worker_venv_bin + '/ddtrace-run ' + executable %} +export DD_TAGS="service:{{ ecommerce_worker_service_name }}" +{% endif -%} + source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env # We exec so that celery is the child of supervisor and can be managed properly exec {{ executable }} -A ecommerce_worker.celery_app:app worker --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info --hostname=ecomworker.%%h --queue=ecommerce.fulfillment,ecommerce.email_marketing,ecommerce.default diff --git a/playbooks/roles/insights/tasks/main.yml b/playbooks/roles/insights/tasks/main.yml index e88891095c1..833becea998 100644 --- a/playbooks/roles/insights/tasks/main.yml +++ b/playbooks/roles/insights/tasks/main.yml @@ -45,6 +45,19 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w" + virtualenv: "{{ insights_venv_dir }}" + state: present + become_user: "{{ insights_user }}" + tags: + - install + - install:app-requirements + - name: create nodeenv shell: "{{ insights_venv_dir }}/bin/nodeenv {{ insights_nodeenv_dir }} --node={{ INSIGHTS_NODE_VERSION }} --prebuilt --force" become_user: "{{ insights_user }}" diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 index 38393cbb1d5..81ea7e62616 100644 --- a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 +++ b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 @@ -3,18 +3,22 @@ # {{ ansible_managed }} {% set insights_venv_bin = insights_home + '/venvs/' + insights_service_name + '/bin' %} -{% if COMMON_ENABLE_NEWRELIC_APP %} -{% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %} -{% else %} + {% set executable = insights_venv_bin + '/gunicorn' %} -{% endif %} {% if COMMON_ENABLE_NEWRELIC_APP %} +{% set executable = insights_venv_bin + '/newrelic-admin run-program ' + insights_venv_bin + '/gunicorn' %} + export NEW_RELIC_DISTRIBUTED_TRACING_ENABLED="{{ INSIGHTS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }}" export NEW_RELIC_APP_NAME="{{ INSIGHTS_NEWRELIC_APPNAME }}" export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% endif -%} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = insights_venv_bin + '/ddtrace-run ' + executable %} +export DD_TAGS="service:{{ insights_service_name }}" +{% endif -%} + source {{ insights_app_dir }}/insights_env # We exec so that gunicorn is the child of supervisor and can be managed properly From de05b926c304eeaa6b5daab2c05db6b54dd9b66b Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Tue, 16 Apr 2024 15:24:28 -0400 Subject: [PATCH 647/664] feat: Apply DD_TRACE_LOG_STREAM_HANDLER hack to insights/ecomworker (#7158) Duplicates fixes from https://github.com/openedx/configuration/pull/7155 These logs were seen in insights, and added to ecomworker for consistency. --- .../ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 | 4 ++++ .../roles/insights/templates/edx/app/insights/insights.sh.j2 | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index c6a33657be2..b6badc15480 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -17,6 +17,10 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = ecommerce_worker_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ ecommerce_worker_service_name }}" +# Copied from edx_django_service playbook for consistency; Datadog +# trace debug logging issue doesn't actually affect edxapp for some +# reason. +export DD_TRACE_LOG_STREAM_HANDLER=false {% endif -%} source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 index 81ea7e62616..abbd32d2cad 100644 --- a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 +++ b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 @@ -17,6 +17,10 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = insights_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ insights_service_name }}" +# Copied from edx_django_service playbook for consistency; Datadog +# trace debug logging issue doesn't actually affect edxapp for some +# reason. +export DD_TRACE_LOG_STREAM_HANDLER=false {% endif -%} source {{ insights_app_dir }}/insights_env From dea2ba1070e755532f736008d2f280ba42c55be5 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 17 Apr 2024 11:09:10 +0500 Subject: [PATCH 648/664] chore: Update systemd journal configuration --- playbooks/roles/vhost/tasks/main.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/vhost/tasks/main.yml b/playbooks/roles/vhost/tasks/main.yml index a5ca407e69f..cb2eaa42d99 100644 --- a/playbooks/roles/vhost/tasks/main.yml +++ b/playbooks/roles/vhost/tasks/main.yml @@ -70,12 +70,17 @@ when: COMMON_OBJECT_STORE_LOG_SYNC and not (ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty') -- name: Set maximum disk space usage for systemd journal +- name: Set maximum disk space usage, free space, retention, and file age for systemd journal lineinfile: path: /etc/systemd/journald.conf - regexp: '^#?SystemMaxUse=' - line: 'SystemMaxUse=1G' + regexp: '^#?{{ item.regexp }}' + line: '{{ item.line }}' state: present + with_items: + - { regexp: 'SystemMaxUse=', line: 'SystemMaxUse=500M' } + - { regexp: 'SystemKeepFree=', line: 'SystemKeepFree=1G' } + - { regexp: 'MaxRetentionSec=', line: 'MaxRetentionSec=1month' } + - { regexp: 'MaxFileSec=', line: 'MaxFileSec=1week' } register: journald_config_line when: ansible_distribution_release == 'bionic' or ansible_distribution_release == 'focal' From 50a769b53ea38cbb9f95567db946b2aad1a87b44 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 17 Apr 2024 14:19:38 +0500 Subject: [PATCH 649/664] feat: Run ddtrace for xqueue and xqwatcher --- playbooks/roles/xqueue/defaults/main.yml | 4 ++++ playbooks/roles/xqueue/tasks/main.yml | 13 +++++++++++++ playbooks/roles/xqueue/templates/xqueue.conf.j2 | 6 +++++- .../roles/xqueue/templates/xqueue_consumer.conf.j2 | 6 +++++- playbooks/roles/xqwatcher/defaults/main.yml | 1 + playbooks/roles/xqwatcher/tasks/deploy_watcher.yml | 13 +++++++++++++ .../edx/app/supervisor/conf.d/xqwatcher.conf.j2 | 5 ++++- 7 files changed, 45 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/xqueue/defaults/main.yml b/playbooks/roles/xqueue/defaults/main.yml index 507665e566c..2fa2f610398 100644 --- a/playbooks/roles/xqueue/defaults/main.yml +++ b/playbooks/roles/xqueue/defaults/main.yml @@ -58,6 +58,10 @@ XQUEUE_MYSQL_CONN_MAX_AGE: 0 # This is Django's default https://docs.djangoproje XQUEUE_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-xqueue" XQUEUE_CONSUMER_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-xqueue_consumer" + +XQUEUE_CONSUMER_DATADOG_APPNAME: "xqueue_consumer" +XQUEUE_DATADOG_APPNAME: "{{ xqueue_service_name }}" + # Set the number of gunicorn front end workers explicitely for xqueue XQUEUE_WORKERS: !!null diff --git a/playbooks/roles/xqueue/tasks/main.yml b/playbooks/roles/xqueue/tasks/main.yml index 5804e01522d..8fc4676a1eb 100644 --- a/playbooks/roles/xqueue/tasks/main.yml +++ b/playbooks/roles/xqueue/tasks/main.yml @@ -74,6 +74,19 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w" + virtualenv: "{{ xqueue_venv_dir }}" + state: present + become_user: "{{ xqueue_user }}" + tags: + - install + - install:app-requirements + ######## END PYTHON3 ######## - name: build virtualenv with python2.7 diff --git a/playbooks/roles/xqueue/templates/xqueue.conf.j2 b/playbooks/roles/xqueue/templates/xqueue.conf.j2 index b058f73bc3c..64767a35016 100644 --- a/playbooks/roles/xqueue/templates/xqueue.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue.conf.j2 @@ -6,12 +6,16 @@ {% set executable = xqueue_venv_bin + '/gunicorn' %} {% endif %} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = xqueue_venv_bin + '/ddtrace-run ' + executable %} +{% endif -%} + command={{ executable }} -c {{ xqueue_app_dir }}/xqueue_gunicorn.py {{ XQUEUE_GUNICORN_WORKERS_EXTRA }} xqueue.wsgi user={{ common_web_user }} directory={{ xqueue_code_dir }} -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_DATADOG_APPNAME }},{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 index fe6a0c58496..d7aeea37f9d 100644 --- a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 @@ -6,12 +6,16 @@ {% set executable = xqueue_venv_bin + '/django-admin run_consumer' %} {% endif %} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = xqueue_venv_bin + '/ddtrace-run ' + executable %} +{% endif -%} + command={{ executable }} --pythonpath={{ xqueue_code_dir }} --settings=xqueue.{{ XQUEUE_SETTINGS }} user={{ common_web_user }} directory={{ xqueue_code_dir }} -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_CONSUMER_DATADOG_APPNAME }},{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqwatcher/defaults/main.yml b/playbooks/roles/xqwatcher/defaults/main.yml index d79b0551ce6..2bfe8760ee8 100644 --- a/playbooks/roles/xqwatcher/defaults/main.yml +++ b/playbooks/roles/xqwatcher/defaults/main.yml @@ -75,6 +75,7 @@ XQWATCHER_REPOS: XQWATCHER_NEWRELIC_APPNAME: "{{ COMMON_DEPLOYMENT }}-{{ COMMON_ENVIRONMENT }}-xqwatcher" XQWATCHER_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}" +XQWATCHER_DATADOG_APPNAME: "{{ xqwatcher_service_name }}" # # # vars are namespace with the module name. diff --git a/playbooks/roles/xqwatcher/tasks/deploy_watcher.yml b/playbooks/roles/xqwatcher/tasks/deploy_watcher.yml index 8cbd29c52f3..e9a13e29113 100644 --- a/playbooks/roles/xqwatcher/tasks/deploy_watcher.yml +++ b/playbooks/roles/xqwatcher/tasks/deploy_watcher.yml @@ -14,6 +14,19 @@ - install - install:app-requirements +- name: "Install Datadog APM requirements" + when: COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP + pip: + name: + - ddtrace + extra_args: "--exists-action w" + virtualenv: "{{ xqwatcher_app_dir }}/venvs/{{ xqwatcher_service_name }}" + state: present + become_user: "{{ xqwatcher_user }}" + tags: + - install + - install:app-requirements + - name: Write out course config files template: src: "edx/app/xqwatcher/conf.d/course.json.j2" diff --git a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 index 94b94fdb05d..7fbe734dfa4 100644 --- a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 +++ b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 @@ -8,6 +8,9 @@ {% else %} {% set executable = xqwatcher_venv_dir + '/bin/python' %} {% endif %} +{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} +{% set executable = xqwatcher_venv_dir + '/ddtrace-run ' + executable %} +{% endif -%} [program:{{ xqwatcher_service_name }}] command={{ executable }} -m {{ xqwatcher_module }} -d {{ xqwatcher_conf_dir }} @@ -16,6 +19,6 @@ user={{ common_web_user }} directory={{ xqwatcher_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%} +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQWATCHER_DATADOG_APPNAME }},{% endif -%} killasgroup=true stopasgroup=true From 4c8f68a543553ee99f4cf3f0992f8043db8f0978 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 17 Apr 2024 15:38:05 +0500 Subject: [PATCH 650/664] fix: ddtrace path for xqwatcher --- .../templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 index 7fbe734dfa4..9c51b9225fa 100644 --- a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 +++ b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 @@ -9,7 +9,7 @@ {% set executable = xqwatcher_venv_dir + '/bin/python' %} {% endif %} {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} -{% set executable = xqwatcher_venv_dir + '/ddtrace-run ' + executable %} +{% set executable = xqwatcher_venv_dir + '/bin/ddtrace-run ' + executable %} {% endif -%} [program:{{ xqwatcher_service_name }}] From ae4f5f9e0e0594282a6a11d8a42902ddcee96bc0 Mon Sep 17 00:00:00 2001 From: Robert Raposa Date: Wed, 17 Apr 2024 15:38:40 -0400 Subject: [PATCH 651/664] feat: Apply DD_TRACE_LOG_STREAM_HANDLER hack to xqueue (#7162) Duplicates fixes from https://github.com/openedx/configuration/pull/7155 These logs were seen in xqueue. --- playbooks/roles/xqueue/templates/xqueue.conf.j2 | 4 +++- playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 | 4 +++- .../templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/xqueue/templates/xqueue.conf.j2 b/playbooks/roles/xqueue/templates/xqueue.conf.j2 index 64767a35016..1089fa09666 100644 --- a/playbooks/roles/xqueue/templates/xqueue.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue.conf.j2 @@ -15,7 +15,9 @@ command={{ executable }} -c {{ xqueue_app_dir }}/xqueue_gunicorn.py {{ XQUEUE_GU user={{ common_web_user }} directory={{ xqueue_code_dir }} -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_DATADOG_APPNAME }},{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +# Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required +# to disable Datadog trace debug logging. +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 index d7aeea37f9d..862b8463ed2 100644 --- a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 @@ -15,7 +15,9 @@ command={{ executable }} --pythonpath={{ xqueue_code_dir }} --settings=xqueue.{{ user={{ common_web_user }} directory={{ xqueue_code_dir }} -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_CONSUMER_DATADOG_APPNAME }},{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +# Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required +# to disable Datadog trace debug logging. +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_CONSUMER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 index 9c51b9225fa..0ed738ad74a 100644 --- a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 +++ b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 @@ -13,12 +13,14 @@ {% endif -%} [program:{{ xqwatcher_service_name }}] -command={{ executable }} -m {{ xqwatcher_module }} -d {{ xqwatcher_conf_dir }} +command={{ executable }} -m {{ xqwatcher_module }} -d {{ xqwatcher_conf_dir }} process_name=%(program_name)s user={{ common_web_user }} directory={{ xqwatcher_code_dir }} stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQWATCHER_DATADOG_APPNAME }},{% endif -%} +# Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required +# to disable Datadog trace debug logging. +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQWATCHER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%} killasgroup=true stopasgroup=true From 34b0ddc45cadcb224f41ed239d88c5f950358519 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 18 Apr 2024 11:01:25 +0500 Subject: [PATCH 652/664] chore: disable Datadog trace debug logging for forum --- playbooks/roles/forum/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 61a03289775..8013ebfcce0 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -91,6 +91,7 @@ forum_base_env: &forum_base_env LISTEN_HOST: "{{ FORUM_LISTEN_HOST }}" LISTEN_PORT: "{{ FORUM_LISTEN_PORT }}" DD_TAGS: "{{ FORUM_DD_TAGS }}" + DD_TRACE_LOG_STREAM_HANDLER: false forum_env: <<: *forum_base_env From 8c356a2143fbd574aa197b5f20ebc8ad131e3f4e Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 18 Apr 2024 12:42:25 +0500 Subject: [PATCH 653/664] fix: turn of dd trace logs forum --- playbooks/roles/forum/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index 8013ebfcce0..c312c827fb0 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -91,7 +91,7 @@ forum_base_env: &forum_base_env LISTEN_HOST: "{{ FORUM_LISTEN_HOST }}" LISTEN_PORT: "{{ FORUM_LISTEN_PORT }}" DD_TAGS: "{{ FORUM_DD_TAGS }}" - DD_TRACE_LOG_STREAM_HANDLER: false + DD_TRACE_LOG_STREAM_HANDLER: "false" forum_env: <<: *forum_base_env From cae7feaacc5da4b484b4ebf368cb21876326c959 Mon Sep 17 00:00:00 2001 From: srwang Date: Thu, 18 Apr 2024 17:50:58 -0400 Subject: [PATCH 654/664] feat: stop caching gatsby page-data --- .../templates/edx/app/nginx/sites-available/prospectus.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 5f64e56dab1..6044835f263 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -206,6 +206,11 @@ server { add_header Cache-Control "no-store, max-age=0"; } + location /page-data/ { + port_in_redirect off; + add_header Cache-Control "no-store, max-age=0"; + } + # PROSPECTUS_TEMPORARY_REDIRECTS will be a list of dictionaries which have: # - from_path: The path of the course-about page that you want redirect # - to_path: URL to redirect to From 3790e1774ef60d35190af100f78b090fb594d480 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 23 Apr 2024 11:53:00 +0500 Subject: [PATCH 655/664] feat: add mongo roles --- playbooks/edx_continuous_integration.yml | 5 + playbooks/mongo_6_0.yml | 29 ++ playbooks/mongo_7_0.yml | 29 ++ playbooks/roles/mongo_6_0/defaults/main.yml | 111 +++++ .../disable-transparent-hugepages.service | 11 + playbooks/roles/mongo_6_0/meta/main.yml | 5 + playbooks/roles/mongo_6_0/tasks/main.yml | 404 ++++++++++++++++++ .../templates/log-mongo-serverStatus.sh.j2 | 3 + .../mongo_6_0/templates/mongo_logrotate.j2 | 46 ++ .../roles/mongo_6_0/templates/mongod.conf.j2 | 46 ++ playbooks/roles/mongo_7_0/defaults/main.yml | 111 +++++ .../disable-transparent-hugepages.service | 11 + playbooks/roles/mongo_7_0/meta/main.yml | 5 + playbooks/roles/mongo_7_0/tasks/main.yml | 404 ++++++++++++++++++ .../templates/log-mongo-serverStatus.sh.j2 | 3 + .../mongo_7_0/templates/mongo_logrotate.j2 | 46 ++ .../roles/mongo_7_0/templates/mongod.conf.j2 | 40 ++ util/jenkins/ansible-provision.sh | 22 + 18 files changed, 1331 insertions(+) create mode 100644 playbooks/mongo_6_0.yml create mode 100644 playbooks/mongo_7_0.yml create mode 100644 playbooks/roles/mongo_6_0/defaults/main.yml create mode 100644 playbooks/roles/mongo_6_0/files/etc/systemd/system/disable-transparent-hugepages.service create mode 100644 playbooks/roles/mongo_6_0/meta/main.yml create mode 100644 playbooks/roles/mongo_6_0/tasks/main.yml create mode 100644 playbooks/roles/mongo_6_0/templates/log-mongo-serverStatus.sh.j2 create mode 100644 playbooks/roles/mongo_6_0/templates/mongo_logrotate.j2 create mode 100644 playbooks/roles/mongo_6_0/templates/mongod.conf.j2 create mode 100644 playbooks/roles/mongo_7_0/defaults/main.yml create mode 100644 playbooks/roles/mongo_7_0/files/etc/systemd/system/disable-transparent-hugepages.service create mode 100644 playbooks/roles/mongo_7_0/meta/main.yml create mode 100644 playbooks/roles/mongo_7_0/tasks/main.yml create mode 100644 playbooks/roles/mongo_7_0/templates/log-mongo-serverStatus.sh.j2 create mode 100644 playbooks/roles/mongo_7_0/templates/mongo_logrotate.j2 create mode 100644 playbooks/roles/mongo_7_0/templates/mongod.conf.j2 diff --git a/playbooks/edx_continuous_integration.yml b/playbooks/edx_continuous_integration.yml index ff75ba6bbcf..fdc34ee3841 100644 --- a/playbooks/edx_continuous_integration.yml +++ b/playbooks/edx_continuous_integration.yml @@ -26,6 +26,11 @@ tags: edxlocal - memcache - role: mongo_5_0 + when: MONGO_5_0_ENABLED + - role: mongo_6_0 + when: MONGO_6_0_ENABLED + - role: mongo_7_0 + when: MONGO_7_0_ENABLED - role: redis - { role: "edxapp", celery_worker: True, when: edxapp_containerized is defined and not edxapp_containerized } - { role: "edxapp", when: edxapp_containerized is defined and not edxapp_containerized } diff --git a/playbooks/mongo_6_0.yml b/playbooks/mongo_6_0.yml new file mode 100644 index 00000000000..6799b7ec7bc --- /dev/null +++ b/playbooks/mongo_6_0.yml @@ -0,0 +1,29 @@ +# Manages a mongo cluster. +# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG +# as used by mongo_replica_set in the mongo_6_0 role. +# +# If you are initializing a cluster, your command might look like: +# ansible-playbook mongo_6_0.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml +# If you just want to deploy an updated replica set config, you can run +# ansible-playbook mongo_6_0.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set +# +# ADDING A NEW CLUSTER MEMBER +# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory +# ansible-playbook mongo_6_0.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml +- name: Deploy MongoDB + hosts: all + become: True + gather_facts: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - mongo_6_0 + - munin_node + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG diff --git a/playbooks/mongo_7_0.yml b/playbooks/mongo_7_0.yml new file mode 100644 index 00000000000..1fd4c6aa8e9 --- /dev/null +++ b/playbooks/mongo_7_0.yml @@ -0,0 +1,29 @@ +# Manages a mongo cluster. +# To set up a new mongo cluster, make sure you've configured MONGO_RS_CONFIG +# as used by mongo_replica_set in the mongo_7_0 role. +# +# If you are initializing a cluster, your command might look like: +# ansible-playbook mongo_7_0.yml -i 203.0.113.11,203.0.113.12,203.0.113.13 -e@/path/to/edx.yml -e@/path/to/ed.yml +# If you just want to deploy an updated replica set config, you can run +# ansible-playbook mongo_7_0.yml -i any-cluster-ip -e@/path/to/edx.yml -e@/path/to/ed.yml --tags configure_replica_set +# +# ADDING A NEW CLUSTER MEMBER +# If you are adding a member to a cluster, you must be sure that the new machine is not first in your inventory +# ansible-playbook mongo_7_0.yml -i 203.0.113.11,203.0.113.12,new-machine-ip -e@/path/to/edx.yml -e@/path/to/ed.yml +- name: Deploy MongoDB + hosts: all + become: True + gather_facts: True + roles: + - role: aws + when: COMMON_ENABLE_AWS_ROLE + - mongo_7_0 + - munin_node + - role: datadog + when: COMMON_ENABLE_DATADOG + - role: splunkforwarder + when: COMMON_ENABLE_SPLUNKFORWARDER + - role: newrelic_infrastructure + when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE + - role: datadog-uninstall + when: not COMMON_ENABLE_DATADOG diff --git a/playbooks/roles/mongo_6_0/defaults/main.yml b/playbooks/roles/mongo_6_0/defaults/main.yml new file mode 100644 index 00000000000..a842e1ca6ed --- /dev/null +++ b/playbooks/roles/mongo_6_0/defaults/main.yml @@ -0,0 +1,111 @@ +mongo_logappend: true + +#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle +mongo_logrotate: reopen + +MONGO_VERSION_MAJOR_MINOR: "6.0" +MONGO_VERSION_PATCH: "15" +PYMONGO_VERSION: "4.4.1" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +mongo_port: "27017" +mongo_extra_conf: '' +mongo_key_file: '/etc/mongodb_key' + +mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo" +mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo" +mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal" +mongo_user: mongodb + +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" + +mongodb_debian_pkgs: + - "mongodb-org={{ MONGO_VERSION }}" + - "mongodb-org-server={{ MONGO_VERSION }}" + - "mongodb-org-shell={{ MONGO_VERSION }}" + - "mongodb-org-mongos={{ MONGO_VERSION }}" + - "mongodb-org-tools={{ MONGO_VERSION }}" + + + +mongo_configure_replica_set: true + +# Vars Meant to be overridden +MONGO_ADMIN_USER: 'admin' +MONGO_ADMIN_PASSWORD: 'password' +MONGO_USERS: + - user: cs_comments_service + password: password + database: cs_comments_service + roles: readWrite + - user: edxapp + password: password + database: edxapp + roles: readWrite + +# This default setting is approriate for a single machine installation +# This will need to be overridden for setups where mongo is on its own server +# and/or you are configuring mongo replication. If the override value is +# 0.0.0.0 mongo will listen on all IPs. The value may also be set to a +# specific IP. +MONGO_BIND_IP: 127.0.0.1 + +MONGO_REPL_SET: "rs0" +MONGO_AUTH: true + +MONGO_CLUSTER_KEY: "CHANGEME" + +# Cluster member configuration +# Fed directly into mongodb_replica_set module +MONGO_RS_CONFIG: + _id: '{{ MONGO_REPL_SET }}' + members: + - host: '127.0.0.1' + +# Storage engine options in 3.2: "mmapv1" or "wiredTiger" +# 3.2 and 3.4 default to wiredTiger +MONGO_STORAGE_ENGINE: "wiredTiger" + +# List of dictionaries as described in the mount_ebs role's default +# for the volumes. +# Useful if you want to store your mongo data and/or journal on separate +# disks from the root volume. By default, they will end up mongo_data_dir +# on the root disk. +MONGO_VOLUMES: [] + +# WiredTiger takes a number of optional configuration settings +# which can be defined as a yaml structure in your secure configuration. +MONGO_STORAGE_ENGINE_OPTIONS: !!null + +mongo_logpath: "{{ mongo_log_dir }}/mongodb.log" +mongo_dbpath: "{{ mongo_data_dir }}/mongodb" + +# In environments that do not require durability (devstack / Jenkins) +# you can disable the journal to reduce disk usage +mongo_enable_journal: true + +MONGO_LOG_SERVERSTATUS: true + +# Vars for configuring a mongo backup node. If enabled, this node will be provisioned with a script that uses mongodump +# to backup the database to an ebs volume at a period set by mongo_backup_cron. +# Set MONGO_BACKUP_ENABLED to true to enable. If enabled, all the other MONGO_BACKUP_ vars must be set according to your +# setup. +MONGO_BACKUP_ENABLED: false +MONGO_BACKUP_NODE: "" # note: most likely the ip address of the instance on which to perform the backups +MONGO_BACKUP_EBS_VOLUME_DEVICE: "" +MONGO_BACKUP_EBS_VOLUME_ID: "" +MONGO_BACKUP_AUTH_DATABASE: "" +MONGO_BACKUP_PRUNE_OLDER_THAN_DATE: "" # passed to `date -d`; should be a relative date like "-30days" +MONGO_BACKUP_SNITCH_URL: "" # Optional URL that will be used to ping a monitoring service (such as Dead Man's Snitch) upon successful completion of a backup. +MONGO_BACKUP_VOLUME_MOUNT_PATH: "/mnt/mongo-backup" +MONGO_BACKUP_SNAPSHOT_DESC: "mongo-backup" +mongo_backup_script_path: "/usr/local/sbin/backup-mongo.sh" +mongo_backup_cron: + minute: '12' + hour: '*/12' + day: '*' + month: '*' + weekday: '*' + +# Internal variable set to true dynamically if backups enabled and playbook running on MONGO_BACKUP_NODE. Do not +# manually override. +is_backup_node: false diff --git a/playbooks/roles/mongo_6_0/files/etc/systemd/system/disable-transparent-hugepages.service b/playbooks/roles/mongo_6_0/files/etc/systemd/system/disable-transparent-hugepages.service new file mode 100644 index 00000000000..282c9e122c3 --- /dev/null +++ b/playbooks/roles/mongo_6_0/files/etc/systemd/system/disable-transparent-hugepages.service @@ -0,0 +1,11 @@ +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +Before=mongod.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' + +[Install] +RequiredBy=mongod.service diff --git a/playbooks/roles/mongo_6_0/meta/main.yml b/playbooks/roles/mongo_6_0/meta/main.yml new file mode 100644 index 00000000000..d7223454526 --- /dev/null +++ b/playbooks/roles/mongo_6_0/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - common + - role: mount_ebs + volumes: "{{ MONGO_VOLUMES }}" diff --git a/playbooks/roles/mongo_6_0/tasks/main.yml b/playbooks/roles/mongo_6_0/tasks/main.yml new file mode 100644 index 00000000000..10dd2484cdf --- /dev/null +++ b/playbooks/roles/mongo_6_0/tasks/main.yml @@ -0,0 +1,404 @@ +--- +- name: Add disable transparent huge pages systemd service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + copy: + src: etc/systemd/system/disable-transparent-hugepages.service + dest: "/etc/systemd/system/disable-transparent-hugepages.service" + owner: root + group: root + mode: 0644 + tags: + - "hugepages" + - "install" + - "install:configuration" + +- name: Enable/start disable transparent huge pages service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + service: + name: disable-transparent-hugepages + enabled: yes + state: started + tags: + - "hugepages" + - "manage" + - "manage:start" + +- name: install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ PYMONGO_VERSION }}" + extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" + state: present + retries: 3 + register: add_mongo_signing_key + tags: + - "install" + - "install:app-requirements" + until: add_mongo_signing_key is succeeded + +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install mongo server and recommends + apt: + pkg: "{{ item }}" + state: present + install_recommends: yes + force: yes + update_cache: yes + register: install_mongo_package + with_items: "{{ mongodb_debian_pkgs }}" + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: create mongo dirs + file: + path: "{{ item }}" + state: directory + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + with_items: + - "{{ mongo_data_dir }}" + - "{{ mongo_dbpath }}" + - "{{ mongo_log_dir }}" + - "{{ mongo_journal_dir }}" + tags: + - "install" + - "install:app-configuration" + +# This will error when run on a new replica set, so we ignore_errors +# and connect anonymously next. +- name: determine if there is a replica set already + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + run_once: true + register: authed_replica_set_already_configured + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +- name: Try checking the replica set with no user/pass in case this is a new box + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + run_once: true + register: unauthed_replica_set_already_configured + when: authed_replica_set_already_configured.failed is defined + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +# We use these in the templates but also to control a whole bunch of logic +- name: set facts that default to not initializing a replica set + set_fact: + initialize_replica_set: false + skip_replica_set: false + tags: + - "install" + - "install:app-configuration" + - "update_mongod_conf" + - "manage" + - "manage:db-replication" + +# If either auth or unauthed access comes back with a replica set, we +# do not want to initialize one. Since initialization requires a bunch +# of extra templating and restarting, it's not something we want to do on +# existing boxes. +- name: track if you have a replica set + set_fact: + initialize_replica_set: true + skip_replica_set: true + when: authed_replica_set_already_configured.status is not defined + and unauthed_replica_set_already_configured.status is not defined + tags: + - "manage" + - "manage:db-replication" + +- name: warn about unconfigured replica sets + debug: msg="You do not appear to have a Replica Set configured, deploying one for you" + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: copy mongodb key file + copy: + content: "{{ MONGO_CLUSTER_KEY }}" + dest: "{{ mongo_key_file }}" + mode: 0600 + owner: mongodb + group: mongodb + register: update_mongod_key + tags: + - "manage" + - "manage:db-replication" + - "mongodb_key" + +# If skip_replica_set is true, this template will not contain a replica set stanza +# because of the fact above. +- name: copy configuration template + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + register: update_mongod_conf + tags: + - "install" + - "install:app-configuration" + - "manage" + - "manage:db-replication" + - "update_mongod_conf" + +# This sets the is_backup_node var by checking whether +# mongo backups are enabled AND we're currently running against the designated mongo backup node. +# This allows backup-related tasks below to determine whether or not they should run on the current mongo node. +- name: determine if backup tasks should run + set_fact: + is_backup_node: true + when: MONGO_BACKUP_ENABLED and '{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}' == '{{ MONGO_BACKUP_NODE }}' + tags: + - "backup:mongo" + +- name: install logrotate configuration + template: + src: mongo_logrotate.j2 + dest: /etc/logrotate.d/hourly/mongo + tags: + - "backup:mongo" + - "install" + - "install:app-configuration" + - "logrotate" + +- name: install prereqs for backup script + apt: + pkg: "{{ item }}" + state: present + update_cache: yes + with_items: + - jq + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install backup script + template: + src: backup-mongo.sh.j2 + dest: "{{ mongo_backup_script_path }}" + mode: 0700 + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: add mongo backup script to cron + cron: + name: mongo backup job + minute: "{{ mongo_backup_cron.minute | default('12') }}" + hour: "{{ mongo_backup_cron.hour | default('*/12') }}" + day: "{{ mongo_backup_cron.day | default('*') }}" + month: "{{ mongo_backup_cron.month | default('*') }}" + weekday: "{{ mongo_backup_cron.weekday | default('*') }}" + job: "{{ mongo_backup_script_path }} >> {{ mongo_log_dir }}/mongo-backup.log 2>&1" + become: yes + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: format mongo backup volume + filesystem: + dev: "{{ MONGO_BACKUP_EBS_VOLUME_DEVICE }}" + fstype: ext4 + force: true + ignore_errors: true + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# We only try passwordless superuser creation when +# we're initializing the replica set and need to use +# the localhost exemption to create a user who will be +# able to initialize the replica set. +# We can only create the users on one machine, the one +# where we will initialize the replica set. If we +# create users on multiple hosts, then they will fail +# to come into the replica set. +- name: create super user + mongodb_user: + name: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + database: admin + roles: root + when: initialize_replica_set + run_once: true + tags: + - "manage" + - "manage:db-replication" + +# Now that the localhost exemption has been used to create the superuser, we need +# to add replica set to our configuration. This will never happen if we detected +# a replica set in the 'determine if there is a replica set already' task. +- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set + set_fact: + skip_replica_set: false + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: re-copy configuration template with replica set enabled + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: restart mongo service + service: + name: mongod + state: restarted + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: configure replica set + mongodb_replica_set: + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + rs_config: "{{ MONGO_RS_CONFIG }}" + run_once: true + register: replset_status + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + - "manage:db-replication-configuration" + +# During initial replica set configuration, it can take a few seconds to vote +# a primary and for all members to reflect that status. During that window, +# use creation or other writes can fail. The best wait/check seems to be repeatedly +# checking the replica set status until we see a PRIMARY in the results. +- name: Wait for the replica set to update and (if needed) elect a primary + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + register: status + until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list + when: mongo_configure_replica_set + retries: 5 + delay: 2 + run_once: true + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + +- name: create mongodb users in a replica set + mongodb_user: + database: "{{ item.database }}" + login_database: 'admin' + login_user: "{{ MONGO_ADMIN_USER }}" + login_password: "{{ MONGO_ADMIN_PASSWORD }}" + name: "{{ item.user }}" + password: "{{ item.password }}" + roles: "{{ item.roles }}" + state: present + replica_set: "{{ MONGO_REPL_SET }}" + with_items: "{{ MONGO_USERS }}" + run_once: true + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-users" + - "manage:db-replication" + +- name: ensure mongo starts at boot time + service: + name: mongod + enabled: yes + tags: + - "manage" + - "manage:start" + +- name: add serverStatus logging script + template: + src: "log-mongo-serverStatus.sh.j2" + dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh" + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + mode: 0700 + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" + +- name: add serverStatus logging script to cron + cron: + name: mongostat logging job + minute: "*/3" + job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1 + become: yes + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" diff --git a/playbooks/roles/mongo_6_0/templates/log-mongo-serverStatus.sh.j2 b/playbooks/roles/mongo_6_0/templates/log-mongo-serverStatus.sh.j2 new file mode 100644 index 00000000000..04649d55ad1 --- /dev/null +++ b/playbooks/roles/mongo_6_0/templates/log-mongo-serverStatus.sh.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON +/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' diff --git a/playbooks/roles/mongo_6_0/templates/mongo_logrotate.j2 b/playbooks/roles/mongo_6_0/templates/mongo_logrotate.j2 new file mode 100644 index 00000000000..f2fb4483566 --- /dev/null +++ b/playbooks/roles/mongo_6_0/templates/mongo_logrotate.j2 @@ -0,0 +1,46 @@ +{{ mongo_log_dir }}/serverStatus.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} + +{% if is_backup_node %} +{{ mongo_log_dir }}/mongo-backup.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} +{% endif %} + +{{ mongo_log_dir }}/mongodb.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M + postrotate + /usr/bin/killall -USR1 mongod + endscript +} diff --git a/playbooks/roles/mongo_6_0/templates/mongod.conf.j2 b/playbooks/roles/mongo_6_0/templates/mongod.conf.j2 new file mode 100644 index 00000000000..b7d4b4a1efe --- /dev/null +++ b/playbooks/roles/mongo_6_0/templates/mongod.conf.j2 @@ -0,0 +1,46 @@ +# {{ ansible_managed }} +# mongodb.conf + + +storage: + # Where to store the data. + dbPath: {{ mongo_dbpath }} + # Storage Engine + engine: {{ MONGO_STORAGE_ENGINE }} + # Enable journaling, http://www.mongodb.org/display/DOCS/Journaling + journal: +{% if mongo_enable_journal %} + enabled: true +{% else %} + enabled: false +{% endif %} +{% if MONGO_STORAGE_ENGINE_OPTIONS %} + {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} +{% endif %} + +systemLog: + #where to log + destination: file + path: "{{ mongo_logpath }}" +{% if mongo_logappend %} + logAppend: true +{% else %} + logAppend: false +{% endif %} + logRotate: {{ mongo_logrotate }} + +{% if not skip_replica_set %} +replication: + replSetName: {{ MONGO_REPL_SET }} + +security: + authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }} + keyFile: {{ mongo_key_file }} + +{% endif %} +net: + bindIp: {{ MONGO_BIND_IP }} + port: {{ mongo_port }} + + +{{ mongo_extra_conf }} diff --git a/playbooks/roles/mongo_7_0/defaults/main.yml b/playbooks/roles/mongo_7_0/defaults/main.yml new file mode 100644 index 00000000000..393c0e43603 --- /dev/null +++ b/playbooks/roles/mongo_7_0/defaults/main.yml @@ -0,0 +1,111 @@ +mongo_logappend: true + +#This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle +mongo_logrotate: reopen + +MONGO_VERSION_MAJOR_MINOR: "7.0" +MONGO_VERSION_PATCH: "8" +PYMONGO_VERSION: "4.4.1" +MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" +mongo_port: "27017" +mongo_extra_conf: '' +mongo_key_file: '/etc/mongodb_key' + +mongo_data_dir: "{{ COMMON_DATA_DIR }}/mongo" +mongo_log_dir: "{{ COMMON_LOG_DIR }}/mongo" +mongo_journal_dir: "{{ COMMON_DATA_DIR }}/mongo/mongodb/journal" +mongo_user: mongodb + +MONGODB_REPO: "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/{{ MONGO_VERSION_MAJOR_MINOR }} multiverse" + +mongodb_debian_pkgs: + - "mongodb-org={{ MONGO_VERSION }}" + - "mongodb-org-server={{ MONGO_VERSION }}" + - "mongodb-org-shell={{ MONGO_VERSION }}" + - "mongodb-org-mongos={{ MONGO_VERSION }}" + - "mongodb-org-tools={{ MONGO_VERSION }}" + + + +mongo_configure_replica_set: true + +# Vars Meant to be overridden +MONGO_ADMIN_USER: 'admin' +MONGO_ADMIN_PASSWORD: 'password' +MONGO_USERS: + - user: cs_comments_service + password: password + database: cs_comments_service + roles: readWrite + - user: edxapp + password: password + database: edxapp + roles: readWrite + +# This default setting is approriate for a single machine installation +# This will need to be overridden for setups where mongo is on its own server +# and/or you are configuring mongo replication. If the override value is +# 0.0.0.0 mongo will listen on all IPs. The value may also be set to a +# specific IP. +MONGO_BIND_IP: 127.0.0.1 + +MONGO_REPL_SET: "rs0" +MONGO_AUTH: true + +MONGO_CLUSTER_KEY: "CHANGEME" + +# Cluster member configuration +# Fed directly into mongodb_replica_set module +MONGO_RS_CONFIG: + _id: '{{ MONGO_REPL_SET }}' + members: + - host: '127.0.0.1' + +# Storage engine options in 3.2: "mmapv1" or "wiredTiger" +# 3.2 and 3.4 default to wiredTiger +MONGO_STORAGE_ENGINE: "wiredTiger" + +# List of dictionaries as described in the mount_ebs role's default +# for the volumes. +# Useful if you want to store your mongo data and/or journal on separate +# disks from the root volume. By default, they will end up mongo_data_dir +# on the root disk. +MONGO_VOLUMES: [] + +# WiredTiger takes a number of optional configuration settings +# which can be defined as a yaml structure in your secure configuration. +MONGO_STORAGE_ENGINE_OPTIONS: !!null + +mongo_logpath: "{{ mongo_log_dir }}/mongodb.log" +mongo_dbpath: "{{ mongo_data_dir }}/mongodb" + +# In environments that do not require durability (devstack / Jenkins) +# you can disable the journal to reduce disk usage +mongo_enable_journal: true + +MONGO_LOG_SERVERSTATUS: true + +# Vars for configuring a mongo backup node. If enabled, this node will be provisioned with a script that uses mongodump +# to backup the database to an ebs volume at a period set by mongo_backup_cron. +# Set MONGO_BACKUP_ENABLED to true to enable. If enabled, all the other MONGO_BACKUP_ vars must be set according to your +# setup. +MONGO_BACKUP_ENABLED: false +MONGO_BACKUP_NODE: "" # note: most likely the ip address of the instance on which to perform the backups +MONGO_BACKUP_EBS_VOLUME_DEVICE: "" +MONGO_BACKUP_EBS_VOLUME_ID: "" +MONGO_BACKUP_AUTH_DATABASE: "" +MONGO_BACKUP_PRUNE_OLDER_THAN_DATE: "" # passed to `date -d`; should be a relative date like "-30days" +MONGO_BACKUP_SNITCH_URL: "" # Optional URL that will be used to ping a monitoring service (such as Dead Man's Snitch) upon successful completion of a backup. +MONGO_BACKUP_VOLUME_MOUNT_PATH: "/mnt/mongo-backup" +MONGO_BACKUP_SNAPSHOT_DESC: "mongo-backup" +mongo_backup_script_path: "/usr/local/sbin/backup-mongo.sh" +mongo_backup_cron: + minute: '12' + hour: '*/12' + day: '*' + month: '*' + weekday: '*' + +# Internal variable set to true dynamically if backups enabled and playbook running on MONGO_BACKUP_NODE. Do not +# manually override. +is_backup_node: false diff --git a/playbooks/roles/mongo_7_0/files/etc/systemd/system/disable-transparent-hugepages.service b/playbooks/roles/mongo_7_0/files/etc/systemd/system/disable-transparent-hugepages.service new file mode 100644 index 00000000000..282c9e122c3 --- /dev/null +++ b/playbooks/roles/mongo_7_0/files/etc/systemd/system/disable-transparent-hugepages.service @@ -0,0 +1,11 @@ +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +Before=mongod.service + +[Service] +Type=oneshot +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' + +[Install] +RequiredBy=mongod.service diff --git a/playbooks/roles/mongo_7_0/meta/main.yml b/playbooks/roles/mongo_7_0/meta/main.yml new file mode 100644 index 00000000000..d7223454526 --- /dev/null +++ b/playbooks/roles/mongo_7_0/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - common + - role: mount_ebs + volumes: "{{ MONGO_VOLUMES }}" diff --git a/playbooks/roles/mongo_7_0/tasks/main.yml b/playbooks/roles/mongo_7_0/tasks/main.yml new file mode 100644 index 00000000000..10dd2484cdf --- /dev/null +++ b/playbooks/roles/mongo_7_0/tasks/main.yml @@ -0,0 +1,404 @@ +--- +- name: Add disable transparent huge pages systemd service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + copy: + src: etc/systemd/system/disable-transparent-hugepages.service + dest: "/etc/systemd/system/disable-transparent-hugepages.service" + owner: root + group: root + mode: 0644 + tags: + - "hugepages" + - "install" + - "install:configuration" + +- name: Enable/start disable transparent huge pages service (http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/) + service: + name: disable-transparent-hugepages + enabled: yes + state: started + tags: + - "hugepages" + - "manage" + - "manage:start" + +- name: install python pymongo for mongo_user ansible module + pip: + name: pymongo + state: present + version: "{{ PYMONGO_VERSION }}" + extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}" + tags: + - "install" + - "install:app-requirements" + +- name: add the mongodb signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-{{ MONGO_VERSION_MAJOR_MINOR }}.asc" + state: present + retries: 3 + register: add_mongo_signing_key + tags: + - "install" + - "install:app-requirements" + until: add_mongo_signing_key is succeeded + +- name: add the mongodb repo to the sources list + apt_repository: + repo: "{{ MONGODB_REPO }}" + state: present + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install mongo server and recommends + apt: + pkg: "{{ item }}" + state: present + install_recommends: yes + force: yes + update_cache: yes + register: install_mongo_package + with_items: "{{ mongodb_debian_pkgs }}" + tags: + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: create mongo dirs + file: + path: "{{ item }}" + state: directory + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + with_items: + - "{{ mongo_data_dir }}" + - "{{ mongo_dbpath }}" + - "{{ mongo_log_dir }}" + - "{{ mongo_journal_dir }}" + tags: + - "install" + - "install:app-configuration" + +# This will error when run on a new replica set, so we ignore_errors +# and connect anonymously next. +- name: determine if there is a replica set already + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + run_once: true + register: authed_replica_set_already_configured + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +- name: Try checking the replica set with no user/pass in case this is a new box + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + run_once: true + register: unauthed_replica_set_already_configured + when: authed_replica_set_already_configured.failed is defined + ignore_errors: true + tags: + - "manage" + - "manage:db-replication" + +# We use these in the templates but also to control a whole bunch of logic +- name: set facts that default to not initializing a replica set + set_fact: + initialize_replica_set: false + skip_replica_set: false + tags: + - "install" + - "install:app-configuration" + - "update_mongod_conf" + - "manage" + - "manage:db-replication" + +# If either auth or unauthed access comes back with a replica set, we +# do not want to initialize one. Since initialization requires a bunch +# of extra templating and restarting, it's not something we want to do on +# existing boxes. +- name: track if you have a replica set + set_fact: + initialize_replica_set: true + skip_replica_set: true + when: authed_replica_set_already_configured.status is not defined + and unauthed_replica_set_already_configured.status is not defined + tags: + - "manage" + - "manage:db-replication" + +- name: warn about unconfigured replica sets + debug: msg="You do not appear to have a Replica Set configured, deploying one for you" + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: copy mongodb key file + copy: + content: "{{ MONGO_CLUSTER_KEY }}" + dest: "{{ mongo_key_file }}" + mode: 0600 + owner: mongodb + group: mongodb + register: update_mongod_key + tags: + - "manage" + - "manage:db-replication" + - "mongodb_key" + +# If skip_replica_set is true, this template will not contain a replica set stanza +# because of the fact above. +- name: copy configuration template + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + register: update_mongod_conf + tags: + - "install" + - "install:app-configuration" + - "manage" + - "manage:db-replication" + - "update_mongod_conf" + +# This sets the is_backup_node var by checking whether +# mongo backups are enabled AND we're currently running against the designated mongo backup node. +# This allows backup-related tasks below to determine whether or not they should run on the current mongo node. +- name: determine if backup tasks should run + set_fact: + is_backup_node: true + when: MONGO_BACKUP_ENABLED and '{{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}' == '{{ MONGO_BACKUP_NODE }}' + tags: + - "backup:mongo" + +- name: install logrotate configuration + template: + src: mongo_logrotate.j2 + dest: /etc/logrotate.d/hourly/mongo + tags: + - "backup:mongo" + - "install" + - "install:app-configuration" + - "logrotate" + +- name: install prereqs for backup script + apt: + pkg: "{{ item }}" + state: present + update_cache: yes + with_items: + - jq + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + - "install:app-requirements" + - "mongo_packages" + +- name: install backup script + template: + src: backup-mongo.sh.j2 + dest: "{{ mongo_backup_script_path }}" + mode: 0700 + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: add mongo backup script to cron + cron: + name: mongo backup job + minute: "{{ mongo_backup_cron.minute | default('12') }}" + hour: "{{ mongo_backup_cron.hour | default('*/12') }}" + day: "{{ mongo_backup_cron.day | default('*') }}" + month: "{{ mongo_backup_cron.month | default('*') }}" + weekday: "{{ mongo_backup_cron.weekday | default('*') }}" + job: "{{ mongo_backup_script_path }} >> {{ mongo_log_dir }}/mongo-backup.log 2>&1" + become: yes + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: format mongo backup volume + filesystem: + dev: "{{ MONGO_BACKUP_EBS_VOLUME_DEVICE }}" + fstype: ext4 + force: true + ignore_errors: true + when: + - is_backup_node + tags: + - "backup:mongo" + - "install" + +- name: restart mongo service if we changed our configuration or upgraded mongo + service: + name: mongod + state: restarted + when: update_mongod_conf.changed or update_mongod_key.changed or install_mongo_package.changed + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + tags: + - "manage" + - "manage:start" + - "manage:db-replication" + +# We only try passwordless superuser creation when +# we're initializing the replica set and need to use +# the localhost exemption to create a user who will be +# able to initialize the replica set. +# We can only create the users on one machine, the one +# where we will initialize the replica set. If we +# create users on multiple hosts, then they will fail +# to come into the replica set. +- name: create super user + mongodb_user: + name: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + database: admin + roles: root + when: initialize_replica_set + run_once: true + tags: + - "manage" + - "manage:db-replication" + +# Now that the localhost exemption has been used to create the superuser, we need +# to add replica set to our configuration. This will never happen if we detected +# a replica set in the 'determine if there is a replica set already' task. +- name: Unset our skip initializing replica set fact so that mongod.conf gets a replica set + set_fact: + skip_replica_set: false + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: re-copy configuration template with replica set enabled + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: yes + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: restart mongo service + service: + name: mongod + state: restarted + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: wait for mongo server to start + wait_for: + port: 27017 + delay: 2 + when: initialize_replica_set + tags: + - "manage" + - "manage:db-replication" + +- name: configure replica set + mongodb_replica_set: + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + rs_config: "{{ MONGO_RS_CONFIG }}" + run_once: true + register: replset_status + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + - "manage:db-replication-configuration" + +# During initial replica set configuration, it can take a few seconds to vote +# a primary and for all members to reflect that status. During that window, +# use creation or other writes can fail. The best wait/check seems to be repeatedly +# checking the replica set status until we see a PRIMARY in the results. +- name: Wait for the replica set to update and (if needed) elect a primary + mongodb_rs_status: + host: "{{ ansible_lo['ipv4']['address'] }}" + username: "{{ MONGO_ADMIN_USER }}" + password: "{{ MONGO_ADMIN_PASSWORD }}" + register: status + until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list + when: mongo_configure_replica_set + retries: 5 + delay: 2 + run_once: true + tags: + - "manage" + - "manage:db" + - "manage:db-replication" + +- name: create mongodb users in a replica set + mongodb_user: + database: "{{ item.database }}" + login_database: 'admin' + login_user: "{{ MONGO_ADMIN_USER }}" + login_password: "{{ MONGO_ADMIN_PASSWORD }}" + name: "{{ item.user }}" + password: "{{ item.password }}" + roles: "{{ item.roles }}" + state: present + replica_set: "{{ MONGO_REPL_SET }}" + with_items: "{{ MONGO_USERS }}" + run_once: true + when: mongo_configure_replica_set + tags: + - "manage" + - "manage:db" + - "manage:db-users" + - "manage:db-replication" + +- name: ensure mongo starts at boot time + service: + name: mongod + enabled: yes + tags: + - "manage" + - "manage:start" + +- name: add serverStatus logging script + template: + src: "log-mongo-serverStatus.sh.j2" + dest: "{{ COMMON_BIN_DIR }}/log-mongo-serverStatus.sh" + owner: "{{ mongo_user }}" + group: "{{ mongo_user }}" + mode: 0700 + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" + +- name: add serverStatus logging script to cron + cron: + name: mongostat logging job + minute: "*/3" + job: /edx/bin/log-mongo-serverStatus.sh >> {{ mongo_log_dir }}/serverStatus.log 2>&1 + become: yes + when: MONGO_LOG_SERVERSTATUS + tags: + - "install" + - "install:app-configuration" diff --git a/playbooks/roles/mongo_7_0/templates/log-mongo-serverStatus.sh.j2 b/playbooks/roles/mongo_7_0/templates/log-mongo-serverStatus.sh.j2 new file mode 100644 index 00000000000..04649d55ad1 --- /dev/null +++ b/playbooks/roles/mongo_7_0/templates/log-mongo-serverStatus.sh.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +# Using JSON.stringify forces output of normal JSON, as opposed to Mongo's weird non-compliant extended JSON +/usr/bin/mongo -u {{ MONGO_ADMIN_USER }} --authenticationDatabase admin -p '{{ MONGO_ADMIN_PASSWORD }}' --quiet <<< 'JSON.stringify(db.serverStatus())' diff --git a/playbooks/roles/mongo_7_0/templates/mongo_logrotate.j2 b/playbooks/roles/mongo_7_0/templates/mongo_logrotate.j2 new file mode 100644 index 00000000000..f2fb4483566 --- /dev/null +++ b/playbooks/roles/mongo_7_0/templates/mongo_logrotate.j2 @@ -0,0 +1,46 @@ +{{ mongo_log_dir }}/serverStatus.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} + +{% if is_backup_node %} +{{ mongo_log_dir }}/mongo-backup.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M +} +{% endif %} + +{{ mongo_log_dir }}/mongodb.log { + create + compress + copytruncate + delaycompress + dateext + dateformat -%Y%m%d-%s + missingok + notifempty + daily + rotate 90 + size 1M + postrotate + /usr/bin/killall -USR1 mongod + endscript +} diff --git a/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 b/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 new file mode 100644 index 00000000000..211d34cebe3 --- /dev/null +++ b/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 @@ -0,0 +1,40 @@ +# {{ ansible_managed }} +# mongodb.conf + + +storage: + # Where to store the data. + dbPath: {{ mongo_dbpath }} + # Storage Engine + engine: {{ MONGO_STORAGE_ENGINE }} +{% endif %} +{% if MONGO_STORAGE_ENGINE_OPTIONS %} + {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} +{% endif %} + +systemLog: + #where to log + destination: file + path: "{{ mongo_logpath }}" +{% if mongo_logappend %} + logAppend: true +{% else %} + logAppend: false +{% endif %} + logRotate: {{ mongo_logrotate }} + +{% if not skip_replica_set %} +replication: + replSetName: {{ MONGO_REPL_SET }} + +security: + authorization: {{ MONGO_AUTH | ternary("enabled", "disabled") }} + keyFile: {{ mongo_key_file }} + +{% endif %} +net: + bindIp: {{ MONGO_BIND_IP }} + port: {{ mongo_port }} + + +{{ mongo_extra_conf }} diff --git a/util/jenkins/ansible-provision.sh b/util/jenkins/ansible-provision.sh index 63e462652c2..8a2a23634e6 100644 --- a/util/jenkins/ansible-provision.sh +++ b/util/jenkins/ansible-provision.sh @@ -520,6 +520,28 @@ EOF_AUTH fi +if [[ $mongo_version == "5.0" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_5_0_ENABLED: True +MONGO_6_0_ENABLED: False +MONGO_7_0_ENABLED: False +MONGO_VERSION +fi +if [[ $mongo_version == "6.0" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_5_0_ENABLED: False +MONGO_6_0_ENABLED: True +MONGO_7_0_ENABLED: False +MONGO_VERSION +fi +if [[ $mongo_version == "7.0" ]]; then + cat << MONGO_VERSION >> $extra_vars_file +MONGO_5_0_ENABLED: False +MONGO_6_0_ENABLED: False +MONGO_7_0_ENABLED: True +MONGO_VERSION +fi + if [[ -n $nginx_users ]]; then cat << EOF_AUTH >> $extra_vars_file NGINX_USERS: $nginx_users From bb6ff5773e49662b744495c4218f60fc09c804fb Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 23 Apr 2024 12:38:07 +0500 Subject: [PATCH 656/664] fix: template --- playbooks/roles/mongo_7_0/templates/mongod.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 b/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 index 211d34cebe3..5b209c5cb13 100644 --- a/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 +++ b/playbooks/roles/mongo_7_0/templates/mongod.conf.j2 @@ -7,7 +7,6 @@ storage: dbPath: {{ mongo_dbpath }} # Storage Engine engine: {{ MONGO_STORAGE_ENGINE }} -{% endif %} {% if MONGO_STORAGE_ENGINE_OPTIONS %} {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }} {% endif %} From 99152d356d1674fc2e4bafc8b50b7874e5600424 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Tue, 23 Apr 2024 13:11:51 +0500 Subject: [PATCH 657/664] chore: pymongo version --- playbooks/roles/mongo_6_0/defaults/main.yml | 2 +- playbooks/roles/mongo_7_0/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/mongo_6_0/defaults/main.yml b/playbooks/roles/mongo_6_0/defaults/main.yml index a842e1ca6ed..ed802237015 100644 --- a/playbooks/roles/mongo_6_0/defaults/main.yml +++ b/playbooks/roles/mongo_6_0/defaults/main.yml @@ -5,7 +5,7 @@ mongo_logrotate: reopen MONGO_VERSION_MAJOR_MINOR: "6.0" MONGO_VERSION_PATCH: "15" -PYMONGO_VERSION: "4.4.1" +PYMONGO_VERSION: "3.12.3" MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" mongo_port: "27017" mongo_extra_conf: '' diff --git a/playbooks/roles/mongo_7_0/defaults/main.yml b/playbooks/roles/mongo_7_0/defaults/main.yml index 393c0e43603..a0d504a831f 100644 --- a/playbooks/roles/mongo_7_0/defaults/main.yml +++ b/playbooks/roles/mongo_7_0/defaults/main.yml @@ -5,7 +5,7 @@ mongo_logrotate: reopen MONGO_VERSION_MAJOR_MINOR: "7.0" MONGO_VERSION_PATCH: "8" -PYMONGO_VERSION: "4.4.1" +PYMONGO_VERSION: "3.12.3" MONGO_VERSION: "{{ MONGO_VERSION_MAJOR_MINOR }}.{{ MONGO_VERSION_PATCH }}" mongo_port: "27017" mongo_extra_conf: '' From 2f6cd231e103f7616102e05dcbdf4988d033a144 Mon Sep 17 00:00:00 2001 From: srwang Date: Tue, 23 Apr 2024 10:05:52 -0400 Subject: [PATCH 658/664] feat: change location block to target s3 --- .../edx/app/nginx/sites-available/prospectus.j2 | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 6044835f263..baaaef80212 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -120,6 +120,15 @@ server { {% endif %} } + # want to avoid caching app-data.json file as an out-of-date hash leads to the application refreshing + location /page-data/ { + port_in_redirect off; + add_header Cache-Control "no-store, max-age=0"; + proxy_pass http://edx-stage-prospectus-static.s3-website-us-east-1.amazonaws.com/71058f2-5803$request_uri; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; + } + # favicon is requested a lot. cache it at the edge. location /favicon.ico { @@ -206,11 +215,6 @@ server { add_header Cache-Control "no-store, max-age=0"; } - location /page-data/ { - port_in_redirect off; - add_header Cache-Control "no-store, max-age=0"; - } - # PROSPECTUS_TEMPORARY_REDIRECTS will be a list of dictionaries which have: # - from_path: The path of the course-about page that you want redirect # - to_path: URL to redirect to From 3e161b34dc2955ce3b16228826a3eaf2e57a984d Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Tue, 23 Apr 2024 10:15:02 -0400 Subject: [PATCH 659/664] Revert "Revert "feat: Update node version to 18."" --- playbooks/roles/edxapp/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/edxapp/defaults/main.yml b/playbooks/roles/edxapp/defaults/main.yml index 353bcbbc0b1..3cc1a16a9b6 100644 --- a/playbooks/roles/edxapp/defaults/main.yml +++ b/playbooks/roles/edxapp/defaults/main.yml @@ -1122,8 +1122,8 @@ edxapp_nodeenv_bin: "{{ edxapp_nodeenv_dir }}/bin" edxapp_npm_dir: "{{ edxapp_app_dir }}/.npm" edxapp_npm_bin: "{{ edxapp_npm_dir }}/bin" edxapp_settings: '{{ EDXAPP_SETTINGS }}' -EDXAPP_NODE_VERSION: "16" -EDXAPP_NPM_VERSION: "8.5.0" +EDXAPP_NODE_VERSION: "18" +EDXAPP_NPM_VERSION: "10.5.1" # This is where node installs modules, not node itself edxapp_node_bin: "{{ edxapp_code_dir }}/node_modules/.bin" edxapp_user: edxapp From 62261555024bb13d7ed54ba87afdb5d5df90dd52 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Wed, 24 Apr 2024 12:06:34 +0500 Subject: [PATCH 660/664] fix: update org to openedx for authn MFE --- playbooks/roles/authn/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/roles/authn/tasks/main.yml b/playbooks/roles/authn/tasks/main.yml index 8bfaab1af34..75f3b9d7f3b 100644 --- a/playbooks/roles/authn/tasks/main.yml +++ b/playbooks/roles/authn/tasks/main.yml @@ -7,3 +7,4 @@ MFE_SITE_NAME: 'Your Platform Name Here' MFE_NPM_OVERRIDES: '{{ AUTHN_MFE_NPM_OVERRIDES | default(MFE_DEPLOY_NPM_OVERRIDES) }}' MFE_ENVIRONMENT_EXTRA: '{{ authn_env_extra | default(MFE_DEPLOY_ENVIRONMENT_EXTRA) }}' + MFE_GIT_PATH: 'openedx' From f0caf849c94b51d3ce5a6b90f3571b29817f3ee7 Mon Sep 17 00:00:00 2001 From: srwang <12386424+srwang@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:05:17 -0400 Subject: [PATCH 661/664] Revert "feat: stop caching prospectus page-data" --- .../edx/app/nginx/sites-available/prospectus.j2 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index baaaef80212..5f64e56dab1 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -120,15 +120,6 @@ server { {% endif %} } - # want to avoid caching app-data.json file as an out-of-date hash leads to the application refreshing - location /page-data/ { - port_in_redirect off; - add_header Cache-Control "no-store, max-age=0"; - proxy_pass http://edx-stage-prospectus-static.s3-website-us-east-1.amazonaws.com/71058f2-5803$request_uri; - # Hide client headers from S3 to prevent request headers too big error - proxy_pass_request_headers off; - } - # favicon is requested a lot. cache it at the edge. location /favicon.ico { From 5ffca09c8bc7eece09480f17404eac2f552bee92 Mon Sep 17 00:00:00 2001 From: srwang Date: Thu, 25 Apr 2024 13:50:47 -0400 Subject: [PATCH 662/664] feat: do not cache app-data --- .../edx/app/nginx/sites-available/prospectus.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index 5f64e56dab1..d415836d02a 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -120,6 +120,15 @@ server { {% endif %} } + # want to avoid caching app-data.json file as an out-of-date hash leads to the application refreshing + location /page-data/app-data.json { + port_in_redirect off; + add_header Cache-Control "no-store, max-age=0"; + proxy_pass http://edx-stage-prospectus-static.s3-website-us-east-1.amazonaws.com/71058f2-5803$request_uri; + # Hide client headers from S3 to prevent request headers too big error + proxy_pass_request_headers off; + } + # favicon is requested a lot. cache it at the edge. location /favicon.ico { From 647df7d3d5ec4d83149262e5d7da8d80b3ffb13e Mon Sep 17 00:00:00 2001 From: srwang <12386424+srwang@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:52:54 -0400 Subject: [PATCH 663/664] Revert "feat: stop caching prospectus app-data.json" --- .../edx/app/nginx/sites-available/prospectus.j2 | 9 --------- 1 file changed, 9 deletions(-) diff --git a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 index d415836d02a..5f64e56dab1 100644 --- a/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 +++ b/playbooks/roles/nginx/templates/edx/app/nginx/sites-available/prospectus.j2 @@ -120,15 +120,6 @@ server { {% endif %} } - # want to avoid caching app-data.json file as an out-of-date hash leads to the application refreshing - location /page-data/app-data.json { - port_in_redirect off; - add_header Cache-Control "no-store, max-age=0"; - proxy_pass http://edx-stage-prospectus-static.s3-website-us-east-1.amazonaws.com/71058f2-5803$request_uri; - # Hide client headers from S3 to prevent request headers too big error - proxy_pass_request_headers off; - } - # favicon is requested a lot. cache it at the edge. location /favicon.ico { From c8aa592a7e79be203032e7895970d031700ed99c Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 6 May 2024 10:44:25 -0400 Subject: [PATCH 664/664] feat: Set Datadog resource name to just the handler in all apps (#7175) This switches us from method+urlpattern resource naming (e.g. `GET ^/`) to using handler (e.g. `lms.djangoapps.branding.views.index`). This is applied to the root span as the `resource_name` tag. As method and urlpattern are available on the root span as `http.method` and `http.route` but there doesn't seem to be any other way to get the handler info, this is our only option for ensuring that the handler will be available as a tag (without having to parse after the fact or write more custom code.) Part of https://github.com/edx/edx-arch-experiments/issues/620 --- .../ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 | 1 + .../roles/edx_django_service/templates/edx/app/app/app.sh.j2 | 1 + playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 | 1 + playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 | 1 + playbooks/roles/forum/defaults/main.yml | 1 + .../roles/insights/templates/edx/app/insights/insights.sh.j2 | 1 + playbooks/roles/xqueue/templates/xqueue.conf.j2 | 2 +- playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 | 2 +- .../templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 | 2 +- 9 files changed, 9 insertions(+), 3 deletions(-) diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index b6badc15480..82e5bf062f4 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -17,6 +17,7 @@ export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}' {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = ecommerce_worker_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ ecommerce_worker_service_name }}" +export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true # Copied from edx_django_service playbook for consistency; Datadog # trace debug logging issue doesn't actually affect edxapp for some # reason. diff --git a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 index 6f7ab4e11b0..17bac3789ee 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 @@ -23,6 +23,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edx_django_service_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ edx_django_service_name }}" +export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true # Workaround for # https://github.com/edx/edx-arch-experiments/issues/591 (heavy # streams of trace-debug logs from ddtrace.) diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 index d6909a2b6a8..e92a73db9e1 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/cms.sh.j2 @@ -23,6 +23,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:edxapp-cms" +export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true # Copied from edx_django_service playbook for consistency; Datadog # trace debug logging issue doesn't actually affect edxapp for some # reason. diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 index 46cbe02cee4..e514b406148 100644 --- a/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 +++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/lms.sh.j2 @@ -23,6 +23,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:edxapp-lms" +export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true # Copied from edx_django_service playbook for consistency; Datadog # trace debug logging issue doesn't actually affect edxapp for some # reason. diff --git a/playbooks/roles/forum/defaults/main.yml b/playbooks/roles/forum/defaults/main.yml index c312c827fb0..93228acc236 100644 --- a/playbooks/roles/forum/defaults/main.yml +++ b/playbooks/roles/forum/defaults/main.yml @@ -91,6 +91,7 @@ forum_base_env: &forum_base_env LISTEN_HOST: "{{ FORUM_LISTEN_HOST }}" LISTEN_PORT: "{{ FORUM_LISTEN_PORT }}" DD_TAGS: "{{ FORUM_DD_TAGS }}" + DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT: "true" DD_TRACE_LOG_STREAM_HANDLER: "false" forum_env: diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 index abbd32d2cad..55a44b59c4e 100644 --- a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 +++ b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 @@ -17,6 +17,7 @@ export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}" {% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %} {% set executable = insights_venv_bin + '/ddtrace-run ' + executable %} export DD_TAGS="service:{{ insights_service_name }}" +export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true # Copied from edx_django_service playbook for consistency; Datadog # trace debug logging issue doesn't actually affect edxapp for some # reason. diff --git a/playbooks/roles/xqueue/templates/xqueue.conf.j2 b/playbooks/roles/xqueue/templates/xqueue.conf.j2 index 1089fa09666..2eeb446058f 100644 --- a/playbooks/roles/xqueue/templates/xqueue.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue.conf.j2 @@ -17,7 +17,7 @@ directory={{ xqueue_code_dir }} # Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required # to disable Datadog trace debug logging. -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true,DD_TAGS=service:{{ XQUEUE_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 index 862b8463ed2..7d7f951641a 100644 --- a/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue_consumer.conf.j2 @@ -17,7 +17,7 @@ directory={{ xqueue_code_dir }} # Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required # to disable Datadog trace debug logging. -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQUEUE_CONSUMER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_STARTUP_TIMEOUT=10,NEW_RELIC_APP_NAME={{ XQUEUE_CONSUMER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true,DD_TAGS=service:{{ XQUEUE_CONSUMER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%}LANG={{ XQUEUE_LANG }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log diff --git a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 index 0ed738ad74a..47b8970d253 100644 --- a/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 +++ b/playbooks/roles/xqwatcher/templates/edx/app/supervisor/conf.d/xqwatcher.conf.j2 @@ -21,6 +21,6 @@ stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log # Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required # to disable Datadog trace debug logging. -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_TAGS=service:{{ XQWATCHER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%} +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQWATCHER_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true,DD_TAGS=service:{{ XQWATCHER_DATADOG_APPNAME }},DD_TRACE_LOG_STREAM_HANDLER=false,{% endif -%} killasgroup=true stopasgroup=true