Merge pull request #287 from edx/feanil/flatten_vars

Feanil/flatten vars

playbooks/edx-east/edx_dev2.yml

 ---
+- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_bastion
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/dev/dev2.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+  roles:
+    - common
 - hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_edxapp
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/dev2.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
@@ -18,6 +28,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/dev2.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
@@ -32,6 +43,7 @@
 #  sudo: True
 #  vars_files:
 #    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/dev/common.yml"
 #    - "{{ secure_dir }}/vars/users.yml"
 #  roles:
 #    - common
@@ -42,6 +54,7 @@
 #  sudo: True
 #  vars_files:
 #    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/dev/common.yml"
 #    - "{{ secure_dir }}/vars/users.yml"
 #  roles:
 #    - common
@@ -50,6 +63,7 @@
 #  sudo: True
 #  vars_files:
 #    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/dev/common.yml"
 #    - "{{ secure_dir }}/vars/users.yml"
 #  roles:
 #    - common

playbooks/edx-east/edx_feanilsandbox.yml

@@ -3,6 +3,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
@@ -18,6 +19,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/common/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
@@ -32,6 +34,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
@@ -42,6 +45,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
@@ -50,6 +54,7 @@
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/dev/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common

playbooks/roles/edxapp/defaults/main.yml

@@ -10,97 +10,150 @@
 
 # These are custom variables that can be overridden
 # on the command line to change specific values in the hash
-c_lms_base: ''
-c_preview_lms_base: ''
+EDXAPP_LMS_BASE: ''
+EDXAPP_PREVIEW_LMS_BASE: ''
+EDXAPP_CMS_BASE: ''
+EDXAPP_AWS_ACCESS_KEY_ID:  ''
+EDXAPP_AWS_SECRET_ACCESS_KEY:  ''
+EDXAPP_XQUEUE_BASIC_AUTH: [ 'edx', 'edx' ]
+EDXAPP_XQUEUE_DJANGO_AUTH:
+  username: 'lms'
+  password: 'password'
+
+EDXAPP_MONGO_HOST: ['localhost']
+EDXAPP_MONGO_PASSWORD: 'password'
+EDXAPP_MONGO_PORT: 27017
+EDXAPP_MONGO_USER: 'mongo'
+
+EDXAPP_MYSQL_DB_NAME: 'edxapp'
+EDXAPP_MYSQL_USER: 'root'
+EDXAPP_MYSQL_PASSWORD: ''
+EDXAPP_MYSQL_HOST: 'localhost'
+EDXAPP_MYSQL_PORT: '3306'
+
+EDXAPP_EMAIL_BACKEND: 'django.core.mail.backends.smtp.EmailBackend'
+
+EDXAPP_LOG_LEVEL: 'INFO'
+
+EDXAPP_MEMCACHE: [ 'localhost:11211' ]
+EDXAPP_COMMENTS_SERVICE_URL:  'http://localhost:4567'
+EDXAPP_COMMENTS_SERVICE_KEY:  'password'
+
+EDXAPP_EDXAPP_SECRET_KEY: ''
+
+EDXAPP_PEARSON_TEST_PASWORD: ''
+
+EDXAPP_OEE_URL: 'http://localhost:18091/'
+EDXAPP_OEE_USER: 'lms'
+EDXAPP_OEE_PASSWORD: 'password'
+
+EDXAPP_ANALYTICS_API_KEY:  ''
+EDXAPP_ZENDESK_USER: ''
+EDXAPP_ZENDESK_API_KEY: ''
+EDXAPP_CELERY_USER: 'celery'
+EDXAPP_CELERY_PASSWORD: ''
+
+EDXAPP_MITX_FEATURES:
+  AUTH_USE_OPENID_PROVIDER: true
+  CERTIFICATES_ENABLED: true
+  ENABLE_DISCUSSION_SERVICE: true
+  ENABLE_INSTRUCTOR_ANALYTICS: true
+  ENABLE_PEARSON_HACK_TEST: false
+  SUBDOMAIN_BRANDING: false
+  SUBDOMAIN_COURSE_LISTINGS: false
+  PREVIEW_LMS_BASE: $EDXAPP_PREVIEW_LMS_BASE
+
+EDXAPP_BOOK_URL: ''
+EDXAPP_SITE_NAME: 'example.com'
+EDXAPP_MEDIA_URL:  ''
+EDXAPP_ANALYTICS_SERVER_URL:  ''
+EDXAPP_FEEDBACK_SUBMISSION_EMAIL: ''
+EDXAPP_CELERY_BROKER_HOSTNAME: ''
+EDXAPP_LOGGING_ENV:  'sandbox'
+
+EDXAPP_SYSLOG_SERVER: ''
+EDXAPP_RABBIT_HOSTNAME: 'rabbit.{{ENV_NAME}}.vpc.edx.org'
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
 #see http://atechie.net/2009/07/merging-hashes-in-yaml-conf-files/
 edxapp_generic_auth_config:  &edxapp_generic_auth
-  'AWS_ACCESS_KEY_ID':  ''
-  'AWS_SECRET_ACCESS_KEY':  ''
-  'SECRET_KEY':  ''
+  'AWS_ACCESS_KEY_ID':  $EDXAPP_AWS_ACCESS_KEY_ID
+  'AWS_SECRET_ACCESS_KEY': $EDXAPP_AWS_SECRET_ACCESS_KEY
+  'SECRET_KEY':  $EDXAPP_EDXAPP_SECRET_KEY
   'XQUEUE_INTERFACE':
-    'basic_auth': [ 'edx',  'edx']
-    'django_auth': { 'password':  'password',
-      'username':  'lms'}
-    'url':  'http://localhost:18040'
+    'basic_auth': $EDXAPP_XQUEUE_BASIC_AUTH
+    'django_auth': $EDXAPP_XQUEUE_DJANGO_AUTH
+    'url': $EDXAPP_XQUEUE_URL
   'CONTENTSTORE':
     'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
     'OPTIONS':
-      'db':  'edxapp'
-      'host': [ 'localhost' ]
-      'password':  'password'
-      'port': 27017
-      'user':  'mongo'
+      'db': $EDXAPP_MONGO_DB_NAME
+      'host': $EDXAPP_MONGO_HOSTS
+      'password': $EDXAPP_MONGO_PASSWORD
+      'port': $EDXAPP_MONGO_PORT
+      'user': $EDXAPP_MONGO_USER
   'MODULESTORE':
     'default':
       'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore'
       'OPTIONS':  &generic_modulestore_default_options
         'collection':  'modulestore'
-        'db':  'edxapp'
+        'db':  $EDXAPP_MONGO_DB_NAME
         'default_class':  'xmodule.hidden_module.HiddenDescriptor'
         'fs_root':  '/opt/wwc/data'
-        'host': [ 'localhost' ]
-        'password':  'password'
-        'port': 27017
+        'host': $EDXAPP_MONGO_HOSTS
+        'password':  $EDXAPP_MONGO_PASSWORD
+        'port': $EDXAPP_MONGO_PORT
         'render_template':  'mitxmako.shortcuts.render_to_string'
-        'user':  'mongo'
+        'user': $EDXAPP_MONGO_USER
+    # Needed for the CMS to be able to run update_templates
     'direct':
       'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore'
       'OPTIONS':  *generic_modulestore_default_options
   'DATABASES':
     'default':
       'ENGINE': 'django.db.backends.mysql'
-      'NAME': 'edxapp'
-      'USER': 'root'
-      'PASSWORD': ''
-      # Provide the name of a host running mysql.
-      'HOST': 'localhost'
-      'PORT': '3306'
-  'PEARSON_TEST_PASSWORD':  ''
+      'NAME': $EDXAPP_MYSQL_DB_NAME
+      'USER': $EDXAPP_MYSQL_USER
+      'PASSWORD': $EDXAPP_MYSQL_PASSWORD
+      'HOST': $EDXAPP_MYSQL_HOST
+      'PORT': $EDXAPP_MYSQL_PORT
+  'PEARSON_TEST_PASSWORD':  $EDXAPP_PEARSON_TEST_PASSWORD
   'OPEN_ENDED_GRADING_INTERFACE':
-    'url':  'http://localhost:18091/'
-    'password':  'password'
+    'url': $EDXAPP_OEE_URL
+    'password': $EDXAPP_OEE_PASSWORD
     'peer_grading':  'peer_grading'
     'staff_grading':  'staff_grading'
     'grading_controller':  'grading_controller'
-    'username':  'lms'
-  'ANALYTICS_API_KEY':  ''
-  'ZENDESK_USER': ''
-  'ZENDESK_API_KEY': ''
-  'CELERY_BROKER_USER': 'celery'
-  'CELERY_BROKER_PASSWORD': ''
+    'username': $EDXAPP_OEE_USER
+  'ANALYTICS_API_KEY':  $EDXAPP_ANALYTICS_API_KEY
+  'ZENDESK_USER': $EDXAPP_ZENDESK_USER
+  'ZENDESK_API_KEY': $EDXAPP_ZENDESK_API_KEY
+  'CELERY_BROKER_USER': $EDXAPP_CELERY_USER
+  'CELERY_BROKER_PASSWORD': $EDXAPP_CELERY_PASSWORD
 
 generic_env_config:  &edxapp_generic_env
-  'LMS_BASE': "{{ c_lms_base }}"
-  'BOOK_URL':  ''
+  'LMS_BASE': $EDXAPP_LMS_BASE
+  'CMS_BASE': $EDXAPP_CMS_BASE
+  'BOOK_URL':  $EDXAPP_BOOK_URL
   'CERT_QUEUE':  'certificates'
-  'LOCAL_LOGLEVEL':  'INFO'
+  'LOCAL_LOGLEVEL': $EDXAPP_LOG_LEVEL
   # default email backed set to local SMTP
-  'EMAIL_BACKEND': 'django.core.mail.backends.smtp.EmailBackend'
-  'MITX_FEATURES':
-    'AUTH_USE_OPENID_PROVIDER': true
-    'CERTIFICATES_ENABLED': true
-    'ENABLE_DISCUSSION_SERVICE': true
-    'ENABLE_INSTRUCTOR_ANALYTICS': true
-    'ENABLE_PEARSON_HACK_TEST': false
-    'SUBDOMAIN_BRANDING': false
-    'SUBDOMAIN_COURSE_LISTINGS': false
-    'PREVIEW_LMS_BASE': "{{ c_preview_lms_base }}"
+  'EMAIL_BACKEND': $EDXAPP_EMAIL_BACKEND
+  'MITX_FEATURES': $EDXAPP_MITX_FEATURES
   'WIKI_ENABLED': true
-  'SYSLOG_SERVER':  'syslog.a.m.i4x.org'
-  'SITE_NAME':  'example.com'
+  'SYSLOG_SERVER':  $EDXAPP_SYSLOG_SERVER
+  'SITE_NAME':  $EDXAPP_SITE_NAME
   'LOG_DIR':  '/mnt/logs/edx'
-  'MEDIA_URL':  ''
-  'ANALYTICS_SERVER_URL':  ''
-  'FEEDBACK_SUBMISSION_EMAIL': ''
+  'MEDIA_URL':  $EDXAPP_MEDIA_URL
+  'ANALYTICS_SERVER_URL':  $EDXAPP_ANALYTICS_SERVER_URL
+  'FEEDBACK_SUBMISSION_EMAIL': $EDXAPP_FEEDBACK_SUBMISSION_EMAIL
   'TIME_ZONE': 'America/New_York'
   'CACHES':
     'default': &default_generic_cache
       'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'
       'KEY_FUNCTION':  'util.memcache.safe_key'
       'KEY_PREFIX':  'sandbox_default'
-      'LOCATION': [ 'localhost:11211' ]
+      'LOCATION': $EDXAPP_MEMCACHE
     'general':
       <<: *default_generic_cache
       'KEY_PREFIX':  'sandbox_general'
@@ -114,16 +167,17 @@ generic_env_config:  &edxapp_generic_env
       <<: *default_generic_cache
       'KEY_PREFIX':  'integration_celery'
   'CELERY_BROKER_TRANSPORT': 'amqp'
-  'CELERY_BROKER_HOSTNAME': ''
-  'COMMENTS_SERVICE_URL':  'http://localhost:4567'
-  'LOGGING_ENV':  'sandbox'
+  'CELERY_BROKER_HOSTNAME': $EDXAPP_RABBIT_HOSTNAME
+  'COMMENTS_SERVICE_URL': $EDXAPP_COMMENTS_SERVICE_URL
+  'LOGGING_ENV': $EDXAPP_LOGGING_ENV
   'SESSION_COOKIE_DOMAIN':  !!null
-  'COMMENTS_SERVICE_KEY':  'password'
+  'COMMENTS_SERVICE_KEY': $EDXAPP_COMMENTS_SERVICE_KEY
   'SEGMENT_IO_LMS': true
   'CODE_JAIL':
     'limits':
       'VMEM': 0
       'REALTIME': 3
+
 lms_auth_config:
   <<: *edxapp_generic_auth
   'MODULESTORE':

playbooks/roles/edxapp/tasks/deploy.yml

@@ -223,7 +223,7 @@
   - lms-xml
   - lms-preview
   - cms
-  - syncdb
+  - migrate 
 
 - name: restart edxapp
   service: name=edxapp state=restarted

playbooks/roles/rabbitmq/defaults/main.yml

@@ -16,14 +16,20 @@ rabbitmq_port: 5672
 rabbitmq_management_port: 15672
 rabbitmq_ip: "{{ ansible_default_ipv4.address }}"
 
-rabbitmq_auth_config:
-  erlang_cookie: "CHANGE ME"
+# Vars meant to be overridden.
+RABBIT_ERLANG_COOKIE: 'DEFAULT_COOKIE'
+RABBIT_USERS:
   admins:
   - name: 'admin'
     password: 'the example admin password'
   - name: 'edx'
     password: 'edx'
 
+# Structure for auth config file.
+rabbitmq_auth_config:
+  erlang_cookie: $RABBIT_ERLANG_COOKIE
+  admins: $RABBIT_USERS
+
 # If the system is running out of an Amazon Web Services
 # cloudformation stack, this group name can used to pull out
 # the name of the stack the rabbit server resides in.

playbooks/roles/xqueue/defaults/main.yml

@@ -14,30 +14,54 @@ xqueue_user: "xqueue"
 xqueue_user_home: "/opt/xqueue"
 xqueue_venv_dir: "{{ xqueue_user_home }}/virtualenvs/{{ xqueue_user }}"
 
+XQUEUE_QUEUES:
+  # push queue
+  'edX-DemoX': 'http://localhost:18050'
+  # pull queues
+  'test-pull': !!null
+  'certificates': !!null
+  'open-ended': !!null
+XQUEUE_LOGGING_ENV: sandbox
+XQUEUE_SYSLOG_SERVER: 'syslog.a.m.i4x.org'
+XQUEUE_S3_BUCKET : 'sandbox-bucket'
+XQUEUE_S3_PATH_PREFIX: 'sandbox-xqueue'
+XQUEUE_LOCAL_LOGLEVEL: 'DEBUG'
+XQUEUE_AWS_ACCESS_KEY_ID : ''
+XQUEUE_AWS_SECRET_ACCESS_KEY : ''
+XQUEUE_BASIC_AUTH: ['edx', 'edx']
+XQUEUE_DJANGO_USER: 'lms'
+XQUEUE_DJANGO_PASSWORD: 'password'
+XQUEUE_RABBITMQ_USER: 'edx'
+XQUEUE_RABBITMQ_PASS: 'edx'
+
+XQUEUE_MYSQL_DB_NAME: 'xqueue'
+XQUEUE_MYSQL_USER: 'root'
+XQUEUE_MYSQL_PASSWORD: ''
+XQUEUE_MYSQL_HOST: 'localhost'
+XQUEUE_MYSQL_PORT: '3306'
+
 xqueue_env_config:
-  'XQUEUES':
-    # push queue
-    'edX-DemoX': 'http://localhost:18050'
-      # pull queues
-    'test-pull': !!null
-    'certificates': !!null
-    'open-ended': !!null
+  'XQUEUES': $XQUEUE_QUEUES
   'XQUEUE_WORKERS_PER_QUEUE': 12
-  'LOGGING_ENV' : 'sandbox'
-  'LOG_DIR' : '/logs'
-  'SYSLOG_SERVER' : 'syslog.a.m.i4x.org'
-  'RABBIT_HOST' : 'localhost'
-  'S3_BUCKET_PREFIX' : 'sandbox-bucket'
+  'LOGGING_ENV' : $XQUEUE_LOGGING_ENV
+  'SYSLOG_SERVER': $XQUEUE_SYSLOG_SERVER
+  'LOG_DIR' : '/mnt/logs/xqueue'
+  'RABBIT_HOST' : $XQUEUE_RABBIT_HOSTNAME
+  'S3_BUCKET' : $XQUEUE_S3_BUCKET
+  'S3_PATH_PREFIX': $XQUEUE_S3_PATH_PREFIX
+  'LOCAL_LOGLEVEL': $XQUEUE_LOCAL_LOGLEVEL
 
 xqueue_auth_config:
-  'AWS_ACCESS_KEY_ID' : ''
-  'AWS_SECRET_ACCESS_KEY' : ''
-  'REQUESTS_BASIC_AUTH': ['edx', 'edx']
-  'USERS': {'lms': 'password'}
-  'RABBITMQ_USER': 'edx'
-  'RABBITMQ_PASS': 'edx'
+  'AWS_ACCESS_KEY_ID':  $XQUEUE_AWS_ACCESS_KEY_ID
+  'AWS_SECRET_ACCESS_KEY': $XQUEUE_AWS_SECRET_ACCESS_KEY
+  'REQUESTS_BASIC_AUTH': $XQUEUE_BASIC_AUTH
+  'USERS': { '{{XQUEUE_DJANGO_USER}}' : $XQUEUE_DJANGO_PASSWORD }
   'DATABASES':
-    'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'xqueue', 'USER': 'root', 'PASSWORD': '', 'HOST': 'localhost', 'PORT': '3306' }
+      'NAME': $XQUEUE_MYSQL_DB_NAME
+      'USER': $XQUEUE_MYSQL_USER
+      'PASSWORD': $XQUEUE_MYSQL_PASSWORD
+      'HOST': $XQUEUE_MYSQL_HOST
+      'PORT': $XQUEUE_MYSQL_PORT
 
 xqueue_create_db: 'yes'
 xqueue_source_repo: https://github.com/edx/xqueue.git

playbooks/roles/xserver/defaults/main.yml

 # Variables for the xserver.
 ---
 
-xserver_env_config: {}
+RABBIT_RUN_URL: ''
+RABBIT_GRADER_ROOT: ''
+RABBIT_LOGGING_ENV: 'sandbox'
+RABBIT_SYSLOG_SERVER: ''
+
+xserver_env_config:
+  RUN_URL: $RABBIT_RUN_URL
+  GRADER_ROOT: $RABBIT_GRADER_ROOT
+  LOGGING_ENV: $RABBIT_LOGGING_ENV
+  LOG_DIR: '/mnt/logs/xserver'
+  SYSLOG_SERVER: $RABBIT_SYSLOG_SERVER
+  SANDBOX_PYTHON: '/opt/edx_apparmor_sandbox/bin/python'
+
 xserver_git_identity_path: "{{ secure_dir }}/files/git-identity"
 xserver_code_dir: "{{ app_base_dir }}/xserver"
 xserver_source_repo: "git://github.com/edx/xserver.git"