Merge pull request #2147 from edx/master

Update Cypress RC

playbooks/roles/ecommerce/defaults/main.yml

@@ -42,6 +42,7 @@ ECOMMERCE_EDX_API_KEY: 'put-your-edx-api-auth-token-here'
 ECOMMERCE_ECOMMERCE_URL_ROOT: 'https://www.example.com'
 ECOMMERCE_LMS_URL_ROOT: 'https://www.example.com'
 ECOMMERCE_JWT_SECRET_KEY: 'generated-key-that-matches-ECOMMERCE_API_SIGNING_KEY-in-edxapp'
+ECOMMERCE_JWT_VERIFY_EXPIRATION: true
 
 # Generated by the app that you're using as your auth source
 # in most cases this will be the edx-platform
@@ -104,6 +105,8 @@ ECOMMERCE_SERVICE_CONFIG:
   LMS_DASHBOARD_URL: '{{ ECOMMERCE_LMS_URL_ROOT }}/dashboard'
   JWT_AUTH:
     JWT_SECRET_KEY: '{{ ECOMMERCE_JWT_SECRET_KEY }}'
+    JWT_ISSUER: '{{ ECOMMERCE_LMS_URL_ROOT }}/oauth2'
+    JWT_VERIFY_EXPIRATION: '{{ ECOMMERCE_JWT_VERIFY_EXPIRATION }}'
   SOCIAL_AUTH_EDX_OIDC_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY }}'
   SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
   SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY }}'

playbooks/roles/edxapp/defaults/main.yml

@@ -229,6 +229,7 @@ EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL: 'no-reply@example.com'
 EDXAPP_BULK_EMAIL_LOG_SENT_EMAILS: false
 EDXAPP_UNIVERSITY_EMAIL: 'university@example.com'
 EDXAPP_PRESS_EMAIL: 'press@example.com'
+EDXAPP_JWT_EXPIRATION: 30  # Number of seconds until expiration
 
 EDXAPP_PLATFORM_TWITTER_ACCOUNT: '@YourPlatformTwitterAccount'
 EDXAPP_PLATFORM_FACEBOOK_ACCOUNT: 'http://www.facebook.com/YourPlatformFacebookAccount'
@@ -717,6 +718,8 @@ generic_env_config:  &edxapp_generic_env
   WIKI_ENABLED: true
   SYSLOG_SERVER:  "{{ EDXAPP_SYSLOG_SERVER }}"
   LOG_DIR:  "{{ COMMON_DATA_DIR }}/logs/edx"
+  JWT_ISSUER: "https://{{ EDXAPP_LMS_BASE }}/oauth2"
+  JWT_EXPIRATION: '{{ EDXAPP_JWT_EXPIRATION }}'
 
   #must end in slash (https://docs.djangoproject.com/en/1.4/ref/settings/#media-url)
   MEDIA_URL:  "{{ EDXAPP_MEDIA_URL }}/"

playbooks/roles/launch_ec2/tasks/main.yml

@@ -74,7 +74,7 @@
     zone: "{{ dns_zone }}"
     type: CNAME
     ttl: 300
-    record: "studio.{{ dns_name }}.{{ dns_zone }}"
+    record: "studio-{{ dns_name }}.{{ dns_zone }}"
     value: "{{ item.public_dns_name }}"
   with_items: ec2.instances
 
@@ -86,7 +86,7 @@
     zone: "{{ dns_zone }}"
     type: CNAME
     ttl: 300
-    record: "preview.{{ dns_name }}.{{ dns_zone }}"
+    record: "preview-{{ dns_name }}.{{ dns_zone }}"
     value: "{{ item.public_dns_name }}"
   with_items: ec2.instances
 

playbooks/roles/minos/defaults/main.yml

@@ -30,8 +30,9 @@ minos_voter_cfg: "{{ COMMON_CFG_DIR }}/{{ minos_service_name }}/conf.d/"
 minos_git_ssh: "/tmp/git.sh"
 minos_git_identity: "{{ minos_app_dir }}/minos-git-identity"
 minos_edx_server_tools_repo: "git@github.com/edx-ops/edx-minos.git"
-minos_edx_server_tools_version: "release"
-minos_requirement: "git+ssh://{{ minos_edx_server_tools_repo }}@{{ minos_edx_server_tools_version }}#egg=edx-minos"
+minos_edx_server_tools_branch: "release"
+minos_edx_server_tools_version: "0.3"
+minos_requirement: "git+ssh://{{ minos_edx_server_tools_repo }}@{{ minos_edx_server_tools_branch }}#egg=edx-minos"
   
 #
 # OS packages

playbooks/roles/minos/tasks/main.yml

@@ -58,8 +58,7 @@
     mode=0755 owner=root group=root
   with_items:
     - "BellwetherVoter"
-    - "ProccessQuienscenceVoterCelery"
-    - "ProccessQuienscenceVoterGunicorn"
+    - "ProccessQuiescenceVoterPython"
     - "TrackingLogVoter"
     - "ZippedTrackingLogVoter"
     - "RolledTrackingLogVoter"
@@ -90,6 +89,7 @@
     virtualenv="{{ minos_app_dir }}/venvs/"
     state=present
     extra_args="--exists-action w"
+    version="{{ minos_edx_server_tools_version }}"
   environment:
     GIT_SSH: "{{ minos_git_ssh }}"
   with_items:

playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuienscenceVoterGunicorn.yml.j2

-ProccessQuiescenceVoter:
-  config:
-    process_name: 'gunicorn'
\ No newline at end of file

playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuiescenceVoterCelery.yml.j2

-ProccessQuiescenceVoter:
-  config:
-    process_name: 'celery'
\ No newline at end of file

playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuiescenceVoterGunicorn.yml.j2

-ProccessQuiescenceVoter:
-  config:
-    process_name: 'gunicorn'
\ No newline at end of file

playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuienscenceVoterCelery.yml.j2 → playbooks/roles/minos/templates/edx/etc/minos/conf.d/ProccessQuiescenceVoterPython.yml.j2

 ProccessQuiescenceVoter:
   config:
-    process_name: 'celery'
+    process_name: 'python'
\ No newline at end of file
+    username: '{{ common_web_user }}'

playbooks/roles/stop_all_edx_services/handlers/main.yml

@@ -42,8 +42,9 @@
 - name: stop mongodb
   service: name=mongodb state=stopped arguments="{{ STOP_ALL_EDX_SERVICES_EXTRA_ARGS }}"
 
+# Celery and Supervisord should not be killed because they may have long running tasks that need to finish
 - name: kill processes by user
-  shell: pkill -u {{ item }} || true
+  shell: pgrep -u {{ item }} -lf | grep -v celery | grep -v supervisord | grep -v gunicorn |  awk '{ print $1}' | xargs -I {} kill {} || true
   with_items:
   - www-data
   - devpi.supervisor

util/jenkins/ansible-provision.sh

@@ -174,9 +174,9 @@ if [[ $edx_internal == "true" ]]; then
     # user and set edx_internal to True so that
     # xserver is installed
     cat << EOF >> $extra_vars_file
-EDXAPP_PREVIEW_LMS_BASE: preview.${deploy_host}
+EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host}
 EDXAPP_LMS_BASE: ${deploy_host}
-EDXAPP_CMS_BASE: studio.${deploy_host}
+EDXAPP_CMS_BASE: studio-${deploy_host}
 EDXAPP_SITE_NAME: ${deploy_host}
 CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090"
 CERTS_VERIFY_URL: "http://${deploy_host}:18090"

util/jenkins/build-ami.sh

@@ -112,4 +112,4 @@ cd util/vpc-tools/
 
 echo "$vars" > /var/tmp/$BUILD_ID-extra-vars.yml
 cat /var/tmp/$BUILD_ID-extra-vars.yml
-python -u abbey.py -p $play -t c3.large -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params $notification_params
+python -u abbey.py -p $play -t m3.large -d $deployment -e $environment -i /edx/var/jenkins/.ssh/id_rsa $base_params $blessed_params $playbookdir_params --vars /var/tmp/$BUILD_ID-extra-vars.yml -c $BUILD_NUMBER --configuration-version $configuration --configuration-secure-version $configuration_secure -k $jenkins_admin_ec2_key --configuration-secure-repo $jenkins_admin_configuration_secure_repo $configurationprivate_params $hipchat_params $cleanup_params $notification_params

util/vpc-tools/db-clone.py

-#!/usr/bin/env python -u
-import boto
-import boto.route53
-import boto.route53.record
-import boto.ec2.elb
-import boto.rds2
-import time
-from argparse import ArgumentParser, RawTextHelpFormatter
-import datetime
-import sys
-from vpcutil import rds_subnet_group_name_for_stack_name, all_stack_names
-import os
-
-description = """
-
-   Creates a new RDS instance using restore
-   from point in time using the latest available backup.
-   The new db will be the same size as the original.
-   The name of the db will remain the same, the master db password
-   will be changed and is set on the command line.
-
-   If stack-name is provided the RDS instance will be launched
-   in the VPC that corresponds to that name.
-
-   New db name defaults to "from-<source db name>-<human date>-<ts>"
-   A new DNS entry will be created for the RDS when provided
-   on the command line
-
-"""
-
-RDS_SIZES = [
-    'db.m1.small',
-    'db.m1.large',
-    'db.m1.xlarge',
-    'db.m2.xlarge',
-    'db.m2.2xlarge',
-    'db.m2.4xlarg',
-]
-
-def parse_args(args=sys.argv[1:]):
-
-    stack_names = all_stack_names()
-    rds = boto.rds2.connect_to_region('us-east-1')
-    dbs = [db['DBInstanceIdentifier']
-           for db in rds.describe_db_instances()['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']]
-
-    parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
-    parser.add_argument('--vpc', default=None, action="store_true",
-                        help='this is for a vpc')
-    parser.add_argument('--security-group', default=None,
-                        help='security group name that should be assigned to the new RDS instance (vpc only!)')
-    parser.add_argument('--subnet', default=None,
-                        help='subnet that should be used for the RDS instance (vpc only!)')
-    parser.add_argument('-t', '--type', choices=RDS_SIZES,
-                        default='db.m1.small', help='RDS size to create instances of')
-    parser.add_argument('-d', '--db-source', choices=dbs,
-                        default=u'stage-edx', help="source db to clone")
-    parser.add_argument('-p', '--password',
-                        help="password for the new database", metavar="NEW PASSWORD")
-    parser.add_argument('-r', '--region', default='us-east-1',
-                        help="region to connect to")
-    parser.add_argument('--dns',
-                        help="dns entry for the new rds instance")
-    parser.add_argument('--clean-wwc', action="store_true",
-                        default=False,
-                        help="clean the wwc db after launching it into the vpc, removing sensitive data")
-    parser.add_argument('--clean-prod-grader', action="store_true",
-                        default=False,
-                        help="clean the prod_grader db after launching it into the vpc, removing sensitive data")
-    parser.add_argument('--dump', action="store_true",
-                        default=False,
-                        help="create a sql dump after launching it into the vpc")
-    parser.add_argument('-s', '--secret-var-files', action="append", required=True,
-                        help="use one or more secret var files to run ansible against the host to update db users")
-    parser.add_argument('-o', '--dest-option-group', default="default:mysql-5-6",
-                        help="the option group for the new rds.")
-
-    return parser.parse_args(args)
-
-
-def wait_on_db_status(db_name, region='us-east-1', wait_on='available', aws_id=None, aws_secret=None):
-    rds = boto.rds2.connect_to_region(region)
-    while True:
-        statuses = rds.describe_db_instances(db_name)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']
-        if len(statuses) > 1:
-            raise Exception("More than one instance returned for {0}".format(db_name))
-        if statuses[0]['DBInstanceStatus'] == wait_on:
-            print("Status is: {}".format(wait_on))
-            break
-        sys.stdout.write("status is {}..\n".format(statuses[0]['DBInstanceStatus']))
-        sys.stdout.flush()
-        time.sleep(10)
-    return
-
-if __name__ == '__main__':
-    args = parse_args()
-    sanitize_wwc_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-wwc.sql")
-    sanitize_prod_grader_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-prod_grader.sql")
-    play_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../playbooks/edx-east")
-
-    rds = boto.rds2.connect_to_region(args.region)
-    restore_dbid = 'from-{0}-{1}-{2}'.format(args.db_source, datetime.date.today(), int(time.time()))
-    restore_args = dict(
-        source_db_instance_identifier=args.db_source,
-        target_db_instance_identifier=restore_dbid,
-        use_latest_restorable_time=True,
-        db_instance_class=args.type,
-        option_group_name=args.dest_option_group,
-    )
-    if args.vpc:
-        restore_args['db_subnet_group_name'] = args.subnet
-    rds.restore_db_instance_to_point_in_time(**restore_args)
-    wait_on_db_status(restore_dbid)
-    print("Getting db host")
-    db_host = rds.describe_db_instances(restore_dbid)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances'][0]['Endpoint']['Address']
-
-    modify_args = dict(
-        apply_immediately=True
-    )
-    if args.password:
-        modify_args['master_user_password'] = args.password
-
-    if args.vpc:
-        modify_args['vpc_security_group_ids'] = [args.security_group]
-    else:
-        # dev-edx is the default security group for dbs that
-        # are not in the vpc, it allows connections from the various
-        # NAT boxes and from sandboxes
-        modify_args['db_security_groups'] = ['dev-edx']
-
-    # Update the db immediately
-    print("Updating db instance: {}".format(modify_args))
-    rds.modify_db_instance(restore_dbid, **modify_args)
-    print("Waiting 15 seconds before checking to see if db is available")
-    time.sleep(15)
-    wait_on_db_status(restore_dbid)
-    print("Waiting another 15 seconds")
-    time.sleep(15)
-    if args.clean_wwc:
-        # Run the mysql clean sql file
-        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} wwc < {sanitize_wwc_sql_file} """.format(
-            root_pass=args.password,
-            db_host=db_host,
-            sanitize_wwc_sql_file=sanitize_wwc_sql_file)
-        print("Running {}".format(sanitize_cmd))
-        os.system(sanitize_cmd)
-
-    if args.clean_prod_grader:
-        # Run the mysql clean sql file
-        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} prod_grader < {sanitize_prod_grader_sql_file} """.format(
-            root_pass=args.password,
-            db_host=db_host,
-            sanitize_prod_grader_sql_file=sanitize_prod_grader_sql_file)
-        print("Running {}".format(sanitize_cmd))
-        os.system(sanitize_cmd)
-
-    if args.secret_var_files:
-        extra_args = ""
-        for secret_var_file in args.secret_var_files:
-            extra_args += " -e@{}".format(secret_var_file)
-
-        db_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, create_dbs.yml """ \
-            """{extra_args} -e "edxapp_db_root_user=root xqueue_db_root_user=root" """ \
-            """ -e "db_root_pass={root_pass}" """ \
-            """ -e "EDXAPP_MYSQL_HOST={db_host}" """ \
-            """ -e "XQUEUE_MYSQL_HOST={db_host}" """.format(
-            root_pass=args.password,
-            extra_args=extra_args,
-            db_host=db_host,
-            play_path=play_path)
-        print("Running {}".format(db_cmd))
-        os.system(db_cmd)
-
-    if args.dns:
-        dns_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, create_cname.yml """ \
-            """-e "dns_zone=edx.org dns_name={dns} sandbox={db_host}" """.format(
-            play_path=play_path,
-            dns=args.dns,
-            db_host=db_host)
-        print("Running {}".format(dns_cmd))
-        os.system(dns_cmd)