Merge pull request #3442 from edx/jibsheet/use-configuration-source-checking

Jibsheet/use configuration source checking

util/jenkins/ansible-provision.sh

@@ -86,13 +86,14 @@ if [[ ( -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ) && (! -f $BOTO_CONF
 fi
 
 extra_vars_file="/var/tmp/extra-vars-$$.yml"
-sandbox_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
+sandbox_secure_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
+sandbox_internal_vars_file="${WORKSPACE}/configuration-internal/ansible/vars/developer-sandbox.yml"
 extra_var_arg="-e@${extra_vars_file}"
 
 if [[ $edx_internal == "true" ]]; then
     # if this is a an edx server include
     # the secret var file
-    extra_var_arg="-e@${sandbox_vars_file} -e@${extra_vars_file}"
+    extra_var_arg="-e@${sandbox_internal_vars_file} -e@${sandbox_secure_vars_file} -e@${extra_vars_file}"
 fi
 
 if [[ -z $region ]]; then
@@ -376,7 +377,7 @@ fi
 
 # deploy the edx_ansible role
 run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
-cat $sandbox_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
+cat $sandbox_secure_vars_file $sandbox_internal_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
 ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b
 ret=$?
 if [[ $ret -ne 0 ]]; then

util/jenkins/build-ami.sh

@@ -88,6 +88,14 @@ if [[ ! -z "$configurationprivaterepo" ]]; then
   fi
 fi
 
+configurationinternal_params=""
+if [[ ! -z "$configurationinternalrepo" ]]; then
+  configurationinternal_params="--configuration-internal-repo $configurationinternalrepo"
+  if [[ ! -z "$configurationinternalversion" ]]; then
+    configurationinternal_params="$configurationinternal_params --configuration-internal-version $configurationinternalversion"
+  fi
+fi
+
 hipchat_params=""
 if [[ ! -z "$hipchat_room_id" ]] && [[ ! -z "$hipchat_api_token"  ]]; then
   hipchat_params="--hipchat-room-id $hipchat_room_id --hipchat-api-token $hipchat_api_token"
@@ -127,4 +135,4 @@ cd util/vpc-tools/
 
 echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
 cat /var/tmp/$BUILD_ID-extra-vars.yml
-python -u abbey.py -p $play -t m3.large -d $deployment -e $environment $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params $notification_params $datadog_params $region_params $identity_params
+python -u abbey.py -p $play -t m3.large -d $deployment -e $environment $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $configurationinternal_params $hipchat_params $cleanup_params $notification_params $datadog_params $region_params $identity_params

util/jenkins/run-ansible.sh

@@ -56,6 +56,22 @@ else
     ansible_ssh_user="${ssh_user}"
 fi
 
+if [[ -f ${WORKSPACE}/configuration-internal/ansible/vars/${deployment_tag}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-internal/ansible/vars/${deployment_tag}.yml"
+fi
+
+if [[ -f ${WORKSPACE}/configuration-internal/ansible/vars/${environment_tag}-${deployment_tag}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-internal/ansible/vars/${environment_tag}-${deployment_tag}.yml"
+fi
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${deployment_tag}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${deployment_tag}.yml"
+fi
+
+if [[ -f ${WORKSPACE}/configuration-secure/ansible/vars/${environment_tag}-${deployment_tag}.yml ]]; then
+    extra_var_args+=" -e@${WORKSPACE}/configuration-secure/ansible/vars/${environment_tag}-${deployment_tag}.yml"
+fi
+
 export PYTHONUNBUFFERED=1
 env
-ansible-playbook -v -D -u $ansible_ssh_user $ansible_play -i ./ec2.py $ansible_task_tags --limit $ansible_limit -e@"$WORKSPACE/configuration-secure/ansible/vars/${deployment_tag}.yml" -e@"$WORKSPACE/configuration-secure/ansible/vars/${environment_tag}-${deployment_tag}.yml" $ansible_extra_vars 
+ansible-playbook -v -D -u $ansible_ssh_user $ansible_play -i ./ec2.py $ansible_task_tags --limit $ansible_limit $extra_var_args $ansible_extra_vars

util/vpc-tools/abbey.py

@@ -86,6 +86,12 @@ def parse_args():
     parser.add_argument('--configuration-secure-repo', required=False,
                         default="git@github.com:edx-ops/prod-secure",
                         help="repo to use for the secure files")
+    parser.add_argument('--configuration-internal-version', required=False,
+                        help="configuration-internal repo gitref",
+                        default="master")
+    parser.add_argument('--configuration-internal-repo', required=False,
+                        default="",
+                        help="repo to use for internal (non-secure) configuration data")
     parser.add_argument('--configuration-private-version', required=False,
                         help="configuration-private repo gitref",
                         default="master")
@@ -247,6 +253,7 @@ git_ssh="$base_dir/git_ssh.sh"
 configuration_version="{configuration_version}"
 configuration_secure_version="{configuration_secure_version}"
 configuration_private_version="{configuration_private_version}"
+configuration_internal_version="{configuration_internal_version}"
 environment="{environment}"
 deployment="{deployment}"
 play="{play}"
@@ -258,9 +265,13 @@ git_repo_secure="{configuration_secure_repo}"
 git_repo_secure_name=$(basename $git_repo_secure .git)
 git_repo_private="{configuration_private_repo}"
 git_repo_private_name=$(basename $git_repo_private .git)
+git_repo_internal="{configuration_internal_repo}"
+git_repo_internal_name=$(basename $git_repo_internal .git)
 secure_vars_file={secure_vars_file}
 environment_deployment_secure_vars="$base_dir/$git_repo_secure_name/ansible/vars/{environment}-{deployment}.yml"
 deployment_secure_vars="$base_dir/$git_repo_secure_name/ansible/vars/{deployment}.yml"
+environment_deployment_internal_vars="$base_dir/$git_repo_internal_name/ansible/vars/{environment}-{deployment}.yml"
+deployment_internal_vars="$base_dir/$git_repo_internal_name/ansible/vars/{deployment}.yml"
 instance_id=\\
 $(curl http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null)
 instance_ip=\\
@@ -380,6 +391,13 @@ if [[ ! -z $git_repo_private ]]; then
     cd $base_dir
 fi
 
+if [[ ! -z $git_repo_internal ]]; then
+    $git_cmd clone $git_repo_internal $git_repo_internal_name
+    cd $git_repo_internal_name
+    $git_cmd checkout $configuration_internal_version
+    cd $base_dir
+fi
+
 
 cd $base_dir/$git_repo_name
 sudo pip install -r pre-requirements.txt
@@ -387,6 +405,14 @@ sudo pip install -r requirements.txt
 
 cd $playbook_dir
 
+if [[ -r "$deployment_internal_vars" ]]; then
+    extra_args_opts+=" -e@$deployment_internal_vars"
+fi
+
+if [[ -r "$environment_deployment_internal_vars" ]]; then
+    extra_args_opts+=" -e@$environment_deployment_internal_vars"
+fi
+
 if [[ -r "$deployment_secure_vars" ]]; then
     extra_args_opts+=" -e@$deployment_secure_vars"
 fi
@@ -414,6 +440,8 @@ rm -rf $base_dir
                 configuration_secure_repo=args.configuration_secure_repo,
                 configuration_private_version=args.configuration_private_version,
                 configuration_private_repo=args.configuration_private_repo,
+                configuration_internal_version=args.configuration_internal_version,
+                configuration_internal_repo=args.configuration_internal_repo,
                 environment=args.environment,
                 deployment=args.deployment,
                 play=args.play,