Merge branch 'master' into mariadb

CHANGELOG.md

+- Role: analytics_api
+  - Added `ANALYTICS_API_AGGREGATE_PAGE_SIZE`, default value 10.  Adjust this parameter to increase the number of
+    aggregate search results returned by the Analytics API, i.e. in course_metadata: enrollment_modes, cohorts, and
+    segments.
+- Role: programs
+  - This role has been removed as this service is no longer supported. The role is still available on the [Ficus branch](https://github.com/edx/configuration/releases/tag/open-release%2Fficus.1).
 - Role: xqueue
   - Changed `XQUEUE_RABBITMQ_TLS` default from `true` to `false`.
-
 - Role: credentials
   - Added `CREDENTIALS_EXTRA_APPS` to enable the inclusion of additional Django apps in the Credentials Service.
 - Role: common
@@ -225,3 +230,6 @@
 
 - Role: ecommerce
   - Removed `SEGMENT_KEY` which is no longer used.  Segment key is now defined in DB configuration. (https://github.com/edx/ecommerce/pull/1121)
+
+- Role: edxapp
+  - Added `EDXAPP_BLOCK_STRUCTURES_SETTINGS` to configure S3-backed Course Block Structures.

docker/build/docker-tools/Dockerfile

@@ -10,7 +10,6 @@
 FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
-ENV PROGRAMS_VERSION=master
 ENV REPO_OWNER=edx
 
 ADD . /edx/app/edx_ansible/edx_ansible

docker/build/go-agent/Dockerfile

@@ -58,7 +58,7 @@ RUN \
 
 
 # Install AWS command-line interface - for AWS operations in a go-agent task.
-RUN pip install awscli
+RUN pip install 'awscli>=1.11.58'
 
 # !!!!NOTICE!!!! ---- Runner of this pipeline take heed!! You must replace go_github_key.pem with the REAL key material
 # that can checkout private github repositories used as pipeline materials. The key material here is faked and is only

docker/build/graphite/Dockerfile

+FROM edxops/xenial-common:latest
+MAINTAINER edxops
+
+USER root
+ADD . /edx/app/edx_ansible/edx_ansible
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
+RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook edx-monitoring.yml -c local \
+   -i '127.0.0.1,'

docker/build/programs/Dockerfile

-# To build this Dockerfile:
-#
-# From the root of configuration:
-#
-# docker build -f docker/build/programs/Dockerfile .
-#
-# This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
-# with the currently checked-out configuration repo.
-
-FROM edxops/xenial-common:latest
-MAINTAINER edxops
-
-ENV PROGRAMS_VERSION=master
-ENV REPO_OWNER=edx
-
-ADD . /edx/app/edx_ansible/edx_ansible
-
-WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-
-COPY docker/build/programs/ansible_overrides.yml /
-RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook programs.yml \
-    -c local -i '127.0.0.1,' \
-    -t 'install,assets' \
-    --extra-vars="@/ansible_overrides.yml" \
-    --extra-vars="PROGRAMS_VERSION=$PROGRAMS_VERSION" \
-    --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"
-
-USER root 
-CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]

docker/build/programs/ansible_overrides.yml

----
-DOCKER_TLD: "edx"
-# The prod settings assume an rsyslogd
-# In addition, on systemd systems, and newer rsyslogd
-# there may be issues with /dev/log existing
-# http://www.projectatomic.io/blog/2014/09/running-syslog-within-a-docker-container/
-PROGRAMS_DJANGO_SETTINGS_MODULE: programs.settings.devstack
-PROGRAMS_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ PROGRAMS_DEFAULT_DB_NAME }}'
-    USER: 'programs001'
-    PASSWORD: 'password'
-    HOST: 'db.{{ DOCKER_TLD  }}'
-    PORT: '3306'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60

docker/compose/edx/docker-compose.yml

@@ -93,7 +93,7 @@ forum:
   # Image built from the opencraft fork as it fixes
   # an auth bug.  Update when the change merges
   # upstream
-  image: edxops/forum:opencraft-v2
+  image: edxops/forums:opencraft-v2
   volumes:
     - ${DOCKER_EDX_ROOT}/cs_comments_service:/edx/app/forum/cs_comments_service
   ports:

docker/plays/programs.yml

-- name: Deploy Programs
-  hosts: all
-  sudo: True
-  gather_facts: True
-  roles:
-    - nginx
-    - docker
-    - role: programs
-      nginx_default_sites:
-        - programs

playbooks/continuous_delivery/create_ami.yml

@@ -9,10 +9,6 @@
 #   - app_repo                        - the url of the github repo for this app
 #   - app_version                     - git hash of the app (play, service, IDA) being deployed
 #   - play                            - the play that was run
-#   - configuration_repo              - The github url for the configuration repo
-#   - configuration_version           - The version (git hash) of configuration
-#   - configuration_secure_repo       - the github url for the configuration secure repository
-#   - configuration_secure_version    - the version (git hash) of configuration secure
 #   - cache_id                        - the cache_id version
 #
 # Other variables
@@ -21,12 +17,12 @@
 #   - ami_wait                        - (yes/no) should ansible pause while
 #   - no_reboot                       - (yes/no) should the instance not be rebooted during AMI creation
 #   - artifact_path                   - the path to where this ansible run stores the artifacts for the pipeline
-#   - edxapp_theme_source_repo        - the edx_app_theme repository
-#   - edxapp_theme_version            - the edx_app_theme repository version (git hash)
 #   - hipchat_token                   - API token to send messages to hipchat
 #   - hipchat_room                    - ID or name of the room to send the notification
 #   - hipchat_url                     - URL of the hipchat API  (defaults to v1 of the api)
 #   - extra_name_identifier           - Makes each AMI unique if desired - Default: 0
+#   - version_tags                    - A mapping of {app: [repo, version], ...}, used to generate
+#                                       a "version:app = repo version" tag on the AMI
 #
 # Example command line to run this playbook:
 #    ansible-playbook -vvvv -i "localhost," -c local \
@@ -36,9 +32,7 @@
 #       -e deployment=edx \
 #       -e edx_environment=sandbox \
 #       -e app_version=12345 \
-#       -e configuration_version=12345
-#       -e configuration_secure_version=12345
-#       -e cache_id=12345
+#       -e cache_id=12345 \
 #       create_ami.yml
 #
 
@@ -74,8 +68,6 @@
       # used a JSON object here as there is a string interpolation in the keys.
       tags: "{
                 'version:{{ play }}':'{{ app_repo }} {{ app_version }}',
-                'version:configuration':'{{ configuration_repo }} {{ configuration_version }}',
-                'version:configuration_secure':'{{ configuration_secure_repo }} {{ configuration_secure_version }}',
                 'play':'{{ play }}',
                 'cache_id':'{{ cache_id }}',
                 'environment':'{{ edx_environment }}',
@@ -89,13 +81,16 @@
       resource: "{{ ami_register.image_id }}"
       tags: "{{ instance_tags.tags }}"
 
-  - name: add optional edx-theme tag
+  - name: Add any version tags that were passed on the commandline
     ec2_tag:
       region: "{{ ec2_region }}"
       resource: "{{ ami_register.image_id }}"
-      tags:
-        version:edxapp_theme: "{{ edxapp_theme_source_repo }} {{ edxapp_theme_version }}"
-    when: edxapp_theme_version is defined and edxapp_theme_source_repo is defined
+      tags: "{
+          {% for name, (repo, version) in version_tags.items() %}
+          'version:{{ name }}': '{{ repo }} {{ version }}',
+          {% endfor %}
+      }"
+    when: version_tags is defined
 
   - name: Fetch tags on the AMI
     ec2_tag:

playbooks/continuous_delivery/rollback_migrations.yml

@@ -38,7 +38,7 @@
 #    -e ARTIFACT_PATH=`/bin/pwd`/../{artifact_path}/migrations \
 #    -e DB_MIGRATION_USER=$DB_MIGRATION_USER \
 #    -e DB_MIGRATION_PASS=$DB_MIGRATION_PASS \
-#    -e ../{artifact_path}/migration_input_file \
+#    -e @../{artifact_path}/migration_input_file.yml \
 #    -e SUB_APPLICATION_NAME={sub_application_name} \
 #    playbooks/continuous_delivery/rollback_migrations.yml
 

playbooks/edx-east/ansible.cfg

@@ -11,3 +11,4 @@ ansible_managed=This file is created and updated by ansible, edit at your peril
 
 [ssh_connection]
 ssh_args=-o ControlMaster=auto -o ControlPersist=60s -o ControlPath="~/.ansible/tmp/ansible-ssh-%h-%p-%r" -o ServerAliveInterval=30
+retries=5

playbooks/edx-east/bootstrap_python.yml

+---
+# Runs the python bootstratpping role against an ubuntu machine
+# This is not as complete as ansible_bootstrap.sh (intentionally so)
+# This lets you get pythong2.7 installed on a machine so you can followup
+# with your actual playbook or role.  The key is gather_facts: False.
+#
+# Usage:
+#   ansible-playbook ./bootstrap_python.yml -i "hostname,"
+#
+- hosts: all
+  become: True
+  gather_facts: False
+  roles:
+    - python

playbooks/edx-east/edx_continuous_integration.yml

@@ -15,7 +15,6 @@
       - xserver
       - analytics_api
       - ecommerce
-      - programs
       - credentials
       nginx_default_sites:
       - lms
@@ -38,7 +37,6 @@
     - edx_ansible
     - analytics_api
     - ecommerce
-    - programs
     - credentials
     - oauth_client_setup
     - role: datadog

playbooks/edx-east/elasticsearch.yml

@@ -27,7 +27,6 @@
   roles:
     - common
     - aws
-    - oraclejdk
     - elasticsearch
   post_tasks:
     - debug:

playbooks/edx-east/oauth_client_setup.yml

@@ -7,7 +7,6 @@
     - "roles/edxapp/defaults/main.yml"
     - "roles/insights/defaults/main.yml"
     - "roles/ecommerce/defaults/main.yml"
-    - "roles/programs/defaults/main.yml"
     - "roles/credentials/defaults/main.yml"
     - "roles/discovery/defaults/main.yml"
   roles:

playbooks/edx-east/populate_configuration_model.yml

@@ -6,7 +6,7 @@
 # code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
 # license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
 #
-# Usage: ansible-playbook -i lms-host-1, -e "file=/path/to/json/file" -e "user=username"
+# Usage: ansible-playbook -i lms-host-1, ./populate_configuration_model.yml -e "file=/path/to/json/file" -e "user=username"
 #
 # Overview:
 # This executes the "populate_model" management command to populate a ConfigurationModel with
@@ -36,6 +36,8 @@
   vars:
     python_path: /edx/bin/python.edxapp
     manage_path: /edx/bin/manage.edxapp
+  become_user: www-data
+  become: true
   tasks:
     - name: Create a temp directory
       shell: mktemp -d /tmp/ansible_xblock_config.XXXXX

playbooks/edx-east/programs.yml

-- name: Deploy edX Programs Service
-  hosts: all
-  become: True
-  gather_facts: True
-  vars:
-    ENABLE_DATADOG: False
-    ENABLE_SPLUNKFORWARDER: False
-    ENABLE_NEWRELIC: False
-    CLUSTER_NAME: 'programs'
-  roles:
-    - aws
-    - role: nginx
-      nginx_sites:
-        - programs
-      nginx_default_sites:
-        - programs
-    - programs
-    - role: datadog
-      when: COMMON_ENABLE_DATADOG
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
-    - role: newrelic
-      when: COMMON_ENABLE_NEWRELIC

playbooks/edx-east/xsy.yml

-# Configure an admin instance with jenkins and asgard.
-- name: Configure instance(s)
-  hosts: all
-  become: True
-  gather_facts: True
-  roles:
-    - aws
-    - xsy

playbooks/edx-monitoring.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - role: python
+      tags:
+        - install
+        - install:system-requirements
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  roles:
+    - graphite
+    - role: nginx
+      nginx_sites:
+      - graphite

playbooks/edx_sandbox.yml

@@ -57,7 +57,6 @@
     - role: ecomworker
       ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
       when: SANDBOX_ENABLE_ECOMMERCE
-    - programs
     - analytics_api
     - insights
     # not ready yet: - edx_notes_api

playbooks/roles/analytics_api/defaults/main.yml

@@ -66,6 +66,7 @@ ANALYTICS_API_DATE_FORMAT: '%Y-%m-%d'
 ANALYTICS_API_DATETIME_FORMAT: '%Y-%m-%dT%H%M%S'
 ANALYTICS_API_DEFAULT_PAGE_SIZE: 25
 ANALYTICS_API_MAX_PAGE_SIZE: 100
+ANALYTICS_API_AGGREGATE_PAGE_SIZE: 10
 
 ANALYTICS_API_BASE_URL: 'http://localhost:8100'
 ANALYTICS_API_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ analytics_api_service_name }}'
@@ -123,6 +124,7 @@ ANALYTICS_API_SERVICE_CONFIG:
   DATETIME_FORMAT: '{{ ANALYTICS_API_DATETIME_FORMAT }}'
   DEFAULT_PAGE_SIZE: '{{ ANALYTICS_API_DEFAULT_PAGE_SIZE }}'
   MAX_PAGE_SIZE: '{{ ANALYTICS_API_MAX_PAGE_SIZE }}'
+  AGGREGATE_PAGE_SIZE: '{{ ANALYTICS_API_AGGREGATE_PAGE_SIZE }}'
   REPORT_DOWNLOAD_BACKEND: '{{ ANALYTICS_API_REPORT_DOWNLOAD_BACKEND }}'
 
 ANALYTICS_API_REPOS:

playbooks/roles/aws/defaults/main.yml

@@ -44,7 +44,7 @@ aws_debian_pkgs:
 
 aws_pip_pkgs:
   - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
-  - awscli==1.10.28
+  - awscli==1.11.58
   - boto=="{{ common_boto_version }}"
   - s3cmd==1.6.1
 

playbooks/roles/browsers/defaults/main.yml

@@ -5,6 +5,7 @@ browser_deb_pkgs:
     - libgconf2-4
     - libnss3-1d
     - libxss1
+    - ubuntu-restricted-extras
     - xdg-utils
     - xvfb
 

playbooks/roles/config-encoders/.gitignore

+*.pyc

playbooks/roles/config-encoders/LICENSE

+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
+

playbooks/roles/config-encoders/README.rst

+Originally authored by Jiri Tyr (https://github.com/jtyr/ansible-config_encoder_filters)
+
+Config Encoder Filters
+======================
+
+This is an Ansible role used to deliver the Config Encoder Filters as
+a dependency of another Ansible role.
+
+
+Table of Contents
+-----------------
+
+- Motivation_
+- Example_
+- Usage_
+- Installation_
+- `Supported encoders`_
+    - encode_apache_
+    - encode_erlang_
+    - encode_haproxy_
+    - encode_ini_
+    - encode_json_
+    - encode_logstash_
+    - encode_nginx_
+    - encode_pam_
+    - encode_toml_
+    - encode_xml_
+    - encode_yaml_
+- Utilities_
+    - template_replace_
+- License_
+- Author_
+
+----
+
+
+.. _Motivation:
+
+Motivation
+----------
+
+Ansible Galaxy contains a lot of useful roles. Some of them exist in
+many variations which differ only by their parameterization. The
+parameterization is often used mainly in templates which generate the
+configuration file. A good example such issues are roles for Nginx of
+which you can find almost 200 in the Ansible Galaxy.
+
+Nginx is possible to configure in infinite number of ways and therefore
+is almost impossible to create an Ansible template file which would
+capture all possible variations of the configuration. Even if a suitable
+roles is found, users often want to customize even more. This is where
+people normally clone the role and add parameters they are missing. Some
+people try to get the change back into the original role by creating a
+pull request (PR) but sometimes such change is not accepted by the
+maintainer of the original role and the user ends up maintaining his/her
+own clone forever.
+
+This is why the Config Encoder filters were developed to facilitate the
+creation of Ansible roles with universal configuration. The structure of
+the configuration file is described as a YAML data structure stored in a
+variable. The variable together with he Config Encoder filter is then
+used in the template file which is used to generate the final
+configuration file. This approach allows to shift the paradigm of
+thinking about configuration files as templates to thinking about them as
+data structures. The data structure can be dynamically generated which
+allows to create truly universal configuration.
+
+
+.. _Example:
+
+Example
+-------
+
+Imagine the following INI file::
+
+    [section1]
+    option11=value11
+    option12=value12
+
+Such configuration file can be described as a YAML data structure::
+
+    myapp_config:
+      section1:
+        option11: value11
+        option12: value12
+
+The variable is then used together with the ``encode_ini`` Config Encoder
+filter in the template file ``myapp.cfg.j2`` like this::
+
+    {{ myapp_config | encode_ini }}
+
+And finally, the template file is used in a task like this::
+
+    - name: Create config file
+      template:
+        src: myapp.cfg.j2
+        dest: /etc/myapp/myapp.cfg
+
+When the task is executed, it creates exactly the same file as the
+original INI file.
+
+So we can describe the configuration as a data structure which is then
+converted into the final configuration file format with the Config
+Encoder filter.
+
+In order to change the above configuration, we would have to overwrite
+the ``myapp_config`` which is not very practical. Therefore we break the
+monolithic variable into a set of variables which will allow us to change
+any part of the configuration without the need to overwrite the whole
+data structure::
+
+    myapp_config_section1_option11: value11
+    myapp_config_section1_option12: value12
+
+    myapp_config_section1__default:
+      option11: "{{ myapp_config_section1_option11 }}"
+      option12: "{{ myapp_config_section1_option12 }}"
+
+    myapp_config_section1__custom: {}
+
+    myapp_config_default:
+      section1: "{{
+        myapp_config_section1__default.update(myapp_config_section1__custom) }}{{
+        myapp_config_section1__default }}"
+
+    myapp_config__custom: {}
+
+    myapp_config: "{{
+      myapp_config__default.update(myapp_config__custom) }}{{
+      myapp_config__default }}"
+
+Like this, if we want to change the value of the ``option11``, we only
+override the variable ``myapp_config_section1_option11``::
+
+    myapp_config_section1_option11: My new value
+
+If we want to add a new option into the ``section1``, we add it into the
+variable ``myapp_config_section1__custom`` which is then merged with the
+default list of options::
+
+    myapp_config_section1__custom:
+      section13: value13
+
+And if we want to add a new section, we add it into the variable
+``myapp_config__custom`` which is then merged with the default list of
+sections::
+
+    myapp_config__custom:
+      section2:
+        option21: value21
+
+The above is showing an example for INI configuration files only but the
+same principle is possible to use for all the supported Config Encoders
+listed bellow.
+
+
+.. _Usage:
+
+Usage
+-----
+
+Config Encoder filters can be used in any Ansible role by adding the
+``config_encoder_filters`` role into the list of dependencies in the
+``meta/main.yml`` file::
+
+    dependencies:
+      - config_encoder_filters
+
+The usage directy from a Playbook requires to add the
+``config_encoder_filters`` into the list of roles::
+
+    - name: My test Play
+      hosts: all
+      roles:
+        - config_encoder_filters
+      tasks:
+        - name: Create config file
+          template:
+            src: my.conf.j2
+            dest: /tmp/my.conf
+
+
+.. _Installation:
+
+Installation
+------------
+
+The role can be downloaded either via Ansible Galaxy command::
+
+    $ ansible-galaxy install jtyr.config_encoder_filters,master,config_encoder_filters
+
+or via Ansible Gallaxy requirements file::
+
+    $ cat ./requirements.yaml
+    ---
+
+    - src: https://github.com/jtyr/ansible-config_encoder_filters.git
+      name: config_encoder_filters
+    $ ansible-galaxy -r ./requirements.yaml
+
+or via Git::
+
+    $ git clone https://github.com/jtyr/ansible-config_encoder_filters.git config_encoder_filters
+
+
+.. _`Supported encoders`:
+
+Supported encoders
+------------------
+
+The following is the list of supported Config Encoder filters. Each
+filter requires special data structure as its input. Each filter also has
+a set of parameters which can modify the behaviour of the filter.
+
+
+.. _encode-apache:
+
+encode_apache
+^^^^^^^^^^^^^
+
+This filter helps to create configuration in the format used by Apache
+web server. The expected data structure is the following::
+
+    my_apache_vhost:
+      content:
+        - sections:
+          - name: VirtualHost
+            param: "*:80"
+            content:
+              - options:
+                - DocumentRoot: /www/example1
+                - ServerName: www.example.com
+                - ErrorLog: /var/log/httpd/www.example.com-error_log
+                - CustomLog:
+                  - /var/log/httpd/www.example.com-access_log
+                  - common
+                - "#": Other directives here ...
+
+The variable starts with ``content`` which can contain list of
+``sections`` or ``options``. ``sections`` then contain list of individual
+sections which has the ``name``, ``param`` and ``content`` parameter. The
+``content`` can again contain a list of `sections`` or ``options``.
+
+The above variable can be used in the template file like this::
+
+    {{ my_apache_vhost | encode_apache }}
+
+The output of such template would be::
+
+    <VirtualHost *:80>
+      DocumentRoot /www/example1
+      ServerName www.example.com
+      ErrorLog /var/log/httpd/www.example.com-error_log
+      CustomLog /var/log/httpd/www.example.com-access_log common
+      # "Other directives here ..."
+    </VirtualHost>
+
+The filter can have the following parameters:
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+- ``quote_all_nums=false``
+
+  Number values are not quoted by default. This parameter will force to
+  quote all numbers.
+
+- ``quote_all_strings=false``
+
+  String values are quoted only if they contain a space. This parameter
+  will force to quote all strings regardless if the they contain the
+  space or not.
+
+
+.. _encode-erlang:
+
+encode_erlang
+^^^^^^^^^^^^^
+
+This filter helps to create configuration in the Erlang format. The
+expected data structure is the following::
+
+    my_rabbitmq_config:
+      - rabbit:
+        - tcp_listeners:
+          - '"127.0.0.1"': 5672
+        - ssl_listeners:
+          - 5671
+        - ssl_options:
+          - cacertfile: /path/to/testca/cacert.pem
+          - certfile: /path/to/server/cert.pem
+          - keyfile: /path/to/server/key.pem
+          - verify: verify_peer
+          - fail_if_no_peer_cert: true
+
+The variable consists of a lists of dictionaries. The value of the key-value
+pair can be another list or simple value like a string or a number. Erlang
+tuples can be enforced by prepending the value with the special character
+specified in the ``atom_value_indicator``.
+
+The above variable can be used in the template file like this::
+
+    {{ my_rabbitmq_config | encode_erlang }}
+
+The output of such template would be::
+
+    [
+      {rabbit, [
+          {tcp_listeners, [
+              {"127.0.0.1", 5672}
+          ]},
+          {ssl_listeners, [
+            5671
+          ]},
+          {ssl_options, [
+              {cacertfile, "/path/to/testca/cacert.pem"},
+              {certfile, "/path/to/server/cert.pem"},
+              {keyfile, "/path/to/server/key.pem"},
+              {verify, "verify_peer"},
+              {fail_if_no_peer_cert, true}
+          ]}
+      ]}
+    ].
+
+The filter can have the following parameters:
+
+- ``atom_value_indicator=":"``
+
+  The value of this parameter indicates the string which must be
+  prepended to a string value to treat it as an atom value.
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+
+.. _encode-haproxy:
+
+encode_haproxy
+^^^^^^^^^^^^^^
+
+This filter helps to create configuration in the format used in Haproxy.
+The expected data structure is the following::
+
+    my_haproxy_config:
+      - global:
+        - daemon
+        - maxconn 256
+      - "# This is the default section"
+      - defaults:
+        - mode http
+        - timeout connect 5000ms
+        - timeout client 50000ms
+        - timeout server 50000ms
+      - frontend http-in:
+        - "# This is the bind address/port"
+        - bind *:80
+        - default_backend servers
+        - backend servers
+        - server server1 127.0.0.1:8000 maxconn 32
+
+The variable is a list which can contain a simple string value or a dictionary
+which indicates a section.
+
+The above variable can be used in the template file like this::
+
+    {{ my_haproxy_config | encode_haproxy }}
+
+The output of such template would be::
+
+    global
+      daemon
+      maxconn 256
+
+    # This is the default section
+    defaults
+      mode http
+      timeout connect 5000ms
+      timeout client 50000ms
+      timeout server 50000ms
+
+    frontend http-in
+      # This is the bind address/port
+      bind *:80
+      default_backend servers
+      backend servers
+      server server1 127.0.0.1:8000 maxconn 32
+
+The filter can have the following parameters:
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+
+.. _encode-ini:
+
+encode_ini
+^^^^^^^^^^
+
+This filter helps to create configuration in the INI format. The expected
+data structure is the following::
+
+    my_rsyncd_config:
+      uid: nobody
+      gid: nobody
+      use chroot: no
+      max connections: 4
+      syslog facility: local5
+      pid file: /run/rsyncd.pid
+      ftp:
+        path: /srv/ftp
+        comment: ftp area
+
+The variable consist of dictionaries which can be nested. If the value of the
+key-value pair on the first level is of a simple type (string, number, boolean),
+such pair is considered to be global and gets processed first. If the value of
+the key-value pair on the first level is another dictionary, the key is
+considered to be the name of the section and the inner dictionary as properties
+of the section.
+
+The above variable can be used in the template file like this::
+
+    {{ my_rsyncd_config | encode_ini }}
+
+The output of such template would be::
+
+    gid=nobody
+    max connections=4
+    pid file=/run/rsyncd.pid
+    syslog facility=local5
+    uid=nobody
+    use chroot=False
+
+    [ftp]
+    comment=ftp area
+    path=/srv/ftp
+
+The filter can have the following parameters:
+
+- ``comment="#"``
+
+  Sign used to comment out lines when `section_is_comment=true`.
+
+- ``delimiter="="``
+
+  Sign separating the *property* and the *value*. By default it's set to
+  ``'='`` but it can also be set for example to ``' = '``.
+
+- ``quote=""``
+
+  Sets the quoting of the value. Use ``quote="'"`` or ``quote='"'``.
+
+- ``section_is_comment=false``
+
+  If this parameter is set to ``true``, the section value will be used as
+  a comment for the following properties of the section.
+
+- ``ucase_prop=false``
+
+  Indicates whether the *property* should be made upper case.
+
+
+.. _encode-json:
+
+encode_json
+^^^^^^^^^^^
+
+This filter helps to create configuration in the JSON format. The
+expected data structure is the following::
+
+    my_sensu_client_config:
+      client:
+        name: localhost
+        address: 127.0.0.1
+        subscriptions:
+          - test
+
+Because JSON is very similar to YAML, the variable consists of
+dictionaries of which value can be either an simple type (number, string,
+boolean), list or another dictionary. All can be nested in any number of
+levels.
+
+The above variable can be used in the template file like this::
+
+    {{ my_sensu_client_config | encode_json }}
+
+The output of such template would be::
+
+    {
+      "client": {
+        "address": "127.0.0.1",
+        "name": "localhost",
+        "subscriptions": [
+          "test"
+        ]
+      }
+    }
+
+The filter can have the following parameters:
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+
+.. _encode-logstash:
+
+encode_logstash
+^^^^^^^^^^^^^^^
+
+This filter helps to create configuration in the format used by Logstash.
+The expected data structure is the following::
+
+    my_logstash_config:
+      - :input:
+          - :file:
+              path: /var/log/httpd/access_log
+              start_position: beginning
+      - :filter:
+          - ':if [path] =~ "access"':
+              - :mutate:
+                  replace:
+                    type: apache_access
+              - :grok:
+                  match:
+                    message: "%{COMBINEDAPACHELOG}"
+              - :date:
+                  match:
+                    - timestamp
+                    - dd/MMM/yyyy:HH:mm:ss Z
+          - ':else if [path] =~ "error"':
+              - :mutate:
+                  replace:
+                    type: "apache_error"
+          - :else:
+              - :mutate:
+                  replace:
+                    type: "random_logs"
+      - :output:
+          - :elasticsearch:
+              hosts:
+                - localhost:9200
+          - :stdout:
+              codec: rubydebug
+
+The variable consists of a list of sections where each section is
+prefixed by a special character specified by the ``section_prefix``
+(``:`` by default). The value of the top level sections can be either
+another section or a dictionary. The value of the dictionary can be a
+simple value, list or another dictionary.
+
+The above variable can be used in the template file like this::
+
+    {{ my_logstash_config | encode_logstash }}
+
+The output of such template would be::
+
+    input {
+      file {
+        path => "/var/log/httpd/access_log"
+        start_position => "beginning"
+      }
+    }
+    filter {
+      if [path] =~ "access" {
+        mutate {
+          replace => {
+            "type" => "apache_access"
+          }
+        }
+        grok {
+          match => {
+            "message" => "%{COMBINEDAPACHELOG}"
+          }
+        }
+        date {
+          match => [
+            "timestamp",
+            "dd/MMM/yyyy:HH:mm:ss Z"
+          ]
+        }
+      }
+      else if [path] =~ "error" {
+        mutate {
+          replace => {
+            "type" => "apache_error"
+          }
+        }
+      }
+      else {
+        mutate {
+          replace => {
+            "type" => "random_logs"
+          }
+        }
+      }
+    }
+    output {
+      elasticsearch {
+        hosts => [
+          "localhost:9200"
+        ]
+      }
+      stdout {
+        codec => "rubydebug"
+      }
+    }
+
+The filter can have the following parameters:
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+- ``section_prefix=":"``
+
+  This parameter specifies which character will be used to identify the
+  Logstash section.
+
+
+.. _encode-nginx:
+
+encode_nginx
+^^^^^^^^^^^^
+
+This filter helps to create configuration in the format used by Nginx
+wweb server. The expected data structure is the following::
+
+    my_nginx_vhost_config:
+      - server:
+        - listen 80
+        - server_name $hostname
+        - "location /":
+          - root /srv/www/myapp
+          - index index.html
+
+As Nginx configuration is order sensitive, the all configuration is
+defined as a nested list. As it would be difficult to recognize how many
+elements each configuration definition has, the list item value is no
+further separated into key/value dictionary. Every line of the
+configuration is treated either as a key indicating another nested list
+or simply as a string.
+
+The above variable can be used in the template file like this::
+
+    {{ my_nginx_vhost | encode_nginx }}
+
+The output of such template would be::
+
+    server {
+      listen 80;
+      server_name $hostname;
+
+      location / {
+        root /srv/www/myapp;
+        index index.html;
+      }
+    }
+
+The filter can have the following parameters:
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+- ``block_semicolon=false``
+
+  Allows to add a semicolon to the end of each block.
+
+
+.. _encode-pam:
+
+encode_pam
+^^^^^^^^^^
+
+This filter helps to create configuration in the format user by Linux
+Pluggable Authentication Modules (PAM). The expected data structure is
+the following::
+
+    my_system_auth_config:
+      aa:
+        type: auth
+        control: required
+        path: pam_unix.so
+        args:
+          - try_first_pass
+          - nullok
+      bb:
+        type: auth
+        control: optional
+        path: pam_permit.so
+      cc:
+        type: auth
+        control: required
+        path: pam_env.so
+      dd:
+        type: account
+        control: required
+        path: pam_unix.so
+      ee:
+        type: account
+        control: optional
+        path: pam_permit.so
+      ff:
+        type: account
+        control: required
+        path: pam_time.so
+      gg:
+        type: password
+        control: required
+        path: pam_unix.so
+        args:
+          - try_first_pass
+          - nullok
+          - sha512
+          - shadow
+      hh:
+        type: password
+        control: optional
+        path: pam_permit.so
+        args:
+      ii:
+        type: session
+        control: required
+        path: pam_limits.so
+      jj:
+        type: session
+        control: required
+        path: pam_unix.so
+      kk:
+        type: session
+        control: optional
+        path: pam_permit.so
+
+The variable is a dictionary of which the key is a labels and the value
+is the PAM rule. The label is used to order the PAM rules. Using labels
+with even number of characters allows to insert another rule in between
+of any two rules.
+
+The above variable can be used in the template file like this::
+
+    {{ my_system_auth_config | encode_pam }}
+
+The output of such template would be::
+
+    auth  required  pam_unix.so  try_first_pass nullok
+    auth  optional  pam_permit.so
+    auth  required  pam_env.so
+
+    account  required  pam_unix.so
+    account  optional  pam_permit.so
+    account  required  pam_time.so
+
+    password  required  pam_unix.so  try_first_pass nullok sha512 shadow
+    password  optional  pam_permit.so
+
+    session  required  pam_limits.so
+    session  required  pam_unix.so
+    session  optional  pam_permit.so
+
+The filter can have the following parameters:
+
+- ``print_label=false``
+
+  Print labels as a comment in the output.
+
+- ``separate_types=true``
+
+  Add a newline between the groups of types.
+
+- ``separator="  "``
+
+  Separator between the collection of tokens.
+
+
+.. _encode-toml:
+
+encode_toml
+^^^^^^^^^^^
+
+This filter helps to create configuration in the TOML format. The
+expected data structure is the following::
+
+    my_grafana_ldap_config:
+      verbose_logging: false
+      servers:
+        - host: 127.0.0.1
+          port: 389
+          use_ssl: false
+          ssl_skip_verify: false
+          bind_dn: cn=admin,dc=grafana,dc=org
+          bind_password: grafana
+          search_filter: "(cn=%s)"
+          search_base_dns:
+            - dc=grafana,dc=org
+      servers.attributes:
+        name: givenName
+        surname: sn
+        username: cn
+        member_of: memberOf
+        email: email
+      servers.group_mappings:
+        - group_dn: cn=admins,dc=grafana,dc=org
+          org_role: Admin
+        - group_dn: cn=users,dc=grafana,dc=org
+          org_role: Editor
+        - group_dn: "*"
+          org_role: Viewer
+
+The variable is a dictionary of which value can be either a simple type
+(number, string, boolean), list or another dictionary. The dictionaries
+and lists can be nested.
+
+The above variable can be used in the template file like this::
+
+    {{ my_grafana_ldap_config | encode_toml }}
+
+The output of such template would be::
+
+    verbose_logging = false
+
+      [[servers]]
+      bind_dn = "cn=admin,dc=grafana,dc=org"
+      bind_password = "grafana"
+      host = "127.0.0.1"
+      port = 389
+      search_base_dns = ["dc=grafana,dc=org"]
+      search_filter = "(cn=%s)"
+      ssl_skip_verify = false
+      use_ssl = false
+
+    [servers.attributes]
+    email = "email"
+    member_of = "memberOf"
+    name = "givenName"
+    surname = "sn"
+    username = "cn"
+
+      [[servers.group_mappings]]
+      group_dn = "cn=admins,dc=grafana,dc=org"
+      org_role = "Admin"
+
+      [[servers.group_mappings]]
+      group_dn = "cn=users,dc=grafana,dc=org"
+      org_role = "Editor"
+
+      [[servers.group_mappings]]
+      group_dn = "*"
+      org_role = "Viewer"
+
+The filter can have the following parameters:
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+- ``quote='"'``
+
+  Sets the quoting of the value. Use ``quote="'"`` or ``quote='"'``.
+
+
+.. _encode-xml:
+
+encode_xml
+^^^^^^^^^^
+
+This filter helps to create configuration in the XML format. The expected
+data structure is the following::
+
+    my_oddjob_config:
+      - oddjobconfig:
+        - service:
+          - ^name: com.redhat.oddjob
+          - object:
+            - ^name: /com/redhat/oddjob
+            - interface:
+              - ^name: com.redhat.oddjob
+              - method:
+                - ^name: listall
+                - allow:
+                  - ^min_uid: 0
+                  - ^max_uid: 0
+              - method:
+                - ^name: list
+                - allow: ''
+              - method:
+                - ^name: quit
+                - allow:
+                  - ^user: root
+              - method:
+                - ^name: reload
+                - allow:
+                  - ^user: root
+        - include:
+          - ^ignore_missing: "yes"
+          - /etc/oddjobd.conf.d/*.conf
+        - include:
+          - ^ignore_missing: "yes"
+          - /etc/oddjobd-local.conf
+
+The variable can be a list of dictionaries, lists or strings. This config
+encoder does not handle mixed content very well so the safest way how to
+include mixed content is to define it as a string and use the parameter
+``escape_xml=false``. This config encoder also produces no XML declaration.
+Any XML declaration or DOCTYPE must be a part of the template file.
+
+The above variable can be used in the template file like this::
+
+    {{ my_oddjob_config | encode_xml }}
+
+The output of such template would be::
+
+    <oddjobconfig>
+      <service name="com.redhat.oddjob">
+        <object name="/com/redhat/oddjob">
+          <interface name="com.redhat.oddjob">
+            <method name="listall">
+              <allow min_uid="0" max_uid="0" />
+            </method>
+            <method name="list">
+              <allow />
+            </method>
+            <method name="quit">
+              <allow user="root" />
+            </method>
+            <method name="reload">
+              <allow user="root" />
+            </method>
+          </interface>
+        </object>
+      </service>
+      <include ignore_missing="yes">/etc/oddjobd.conf.d/*.conf</include>
+      <include ignore_missing="yes">/etc/oddjobd-local.conf</include>
+    </oddjobconfig>
+
+The filter can have the following parameters:
+
+- ``attribute_sign="^"``
+
+  XML attribute indicator.
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+
+.. _encode-yaml:
+
+encode_yaml
+^^^^^^^^^^^
+
+This filter helps to create configuration in the YAML format. The
+expected data structure is the following::
+
+    my_mongodb_config:
+      systemLog:
+        destination: file
+        logAppend: true
+        path: /var/log/mongodb/mongod.log
+      storage:
+        dbPath: /var/lib/mongo
+        journal:
+          enabled: true
+      processManagement:
+        fork: true
+        pidFilePath: /var/run/mongodb/mongod.pid
+      net:
+        port: 27017
+        bindIp: 127.0.0.1
+
+The variable is ordinary YAML. The only purpose of this encoder filter is
+to be able to convert YAML data structure into the string in a template
+file in unified way compatible with the other config encoders.
+
+The above variable can be used in the template file like this::
+
+    {{ my_mongodb_config | encode_yaml }}
+
+The output of such template would be::
+
+    net:
+      bindIp: "127.0.0.1"
+      port: 27017
+    processManagement:
+      fork: true
+      pidFilePath: "/var/run/mongodb/mongod.pid"
+    storage:
+      dbPath: "/var/lib/mongo"
+      journal:
+        enabled: true
+    systemLog:
+      destination: "file"
+      logAppend: true
+      path: "/var/log/mongodb/mongod.log"
+
+The filter can have the following parameters:
+
+- ``convert_bools=false``
+
+  Indicates whether Boolean values presented as a string should be
+  converted to a real Boolean value. For example ``var1: 'True'`` would
+  be represented as a string but by using the ``convert_bools=true`` it
+  will be converted into Boolean like it would be defined like ``var1:
+  true``.
+
+- ``convert_nums=false``
+
+  Indicates whether number presented as a string should be converted to
+  number. For example ``var1: '123'`` would be represented as a string
+  but by using the ``convert_nums=true`` it will be converted it to a
+  number like it would be defined like ``var1: 123``. It's also possible
+  to use the YAML type casting to convert string to number (e.g. ``!!int
+  "1234"``, ``!!float "3.14"``).
+
+- ``indent="  "``
+
+  Defines the indentation unit.
+
+- ``level=0``
+
+  Indicates the initial level of the indentation. Value ``0`` starts
+  indenting from the beginning of the line. Setting the value to higher
+  than ``0`` indents the content by ``indent * level``.
+
+- ``quote='"'``
+
+  Sets the quoting of the value. Use ``quote="'"`` or ``quote='"'``.
+
+
+.. _Utilities:
+
+Utilities
+---------
+
+The followng is a list of utilities that can be used in conjunction with the
+Config Encoder filters.
+
+
+.. _template-replace:
+
+template_replace
+^^^^^^^^^^^^^^^^
+
+This filter allows to use extra templating layer which gets processed during
+the template file processing. That can be useful if it's necessary to create
+repetitive but slightly different definitions inside the template file.
+
+The extra templating layer is represented by a templating variable which
+contains specially decorated variables which get replaced by its real value at
+the time of template file processing. The template variable can be composed
+dynamically which provides extra flexibility that would otherwise have to be
+hardcoded in the template file.
+
+The filter expects the template variable containing the specially decorated
+variables as its input. The filter has one parameter which is used to replaced
+the specially decorated variables in the template variable.
+
+Let's have a look at an example of such usage::
+
+    # The variable used as the replacement in the template variable
+    my_clients:
+      - host: myclient01
+        jobdefs: Default
+        password: Passw0rd1
+        file_retention: 30 days
+      - host: myclient02
+        jobdefs: HomeOnly
+        password: Passw0rd2
+        file_retention: 90 days
+
+    # The actual template variable used in the template file
+    bacula_director_config_job_client:
+      # First template variable containing the specially decorated variables
+      - template:
+          - Job:
+            - Name = Job-{[{ item['jobdefs'] }]}-{[{ item['host'] }]}
+            - Client = {[{ item['host'] }]}-fd
+            - JobDefs = {[{ item['jobdefs'] }]}
+        # Variable used to replace the specially decorated variables
+        items: "{{ my_clients }}"
+      # Second template and its items
+      - template:
+          - Client:
+            - Name = {[{ item['host'] }]}-fd
+            - Address = {[{ item['host'] }]}
+            - FD Port = 9102
+            - Catalog = Default
+            - Password = {[{ item['password'] }]}
+            - File Retention = {[{ item['file_retention'] }]}
+            - Job Retention = 3 months
+            - AutoPrune = yes
+        items: "{{ my_clients }}"
+
+The above variable can be used together with the `template_replace` filter in
+the template file (``bacula-dir.conf.j2``) like this::
+
+    {% for record in bacula_director_config_job_client %}
+      {%- for item in record['items'] -%}
+        {{ record['template'] | template_replace(item) | encode_nginx }}{{ "\n" }}
+      {%- endfor -%}
+    {% endfor %}
+
+The template file can be called from the playbook/role like this::
+
+    - name: Configure Bacula Director
+      template:
+        src: bacula-dir.conf.j2
+        dest: /etc/bacula/bacula-dir.conf
+
+And the result of such usage is the following::
+
+    Job {
+      Name = Job-Default-myclient01;
+      Client = myclient01-fd;
+      JobDefs = Default;
+    }
+
+    Job {
+      Name = Job-HomeOnly-myclient02;
+      Client = myclient02-fd;
+      JobDefs = HomeOnly;
+    }
+
+    Client {
+      Name = myclient01-fd;
+      Address = myclient01;
+      FD Port = 9102;
+      Catalog = Default;
+      Password = Passw0rd1;
+      File Retention = 30 days;
+      Job Retention = 3 months;
+      AutoPrune = yes;
+    }
+
+    Client {
+      Name = myclient02-fd;
+      Address = myclient02;
+      FD Port = 9102;
+      Catalog = Default;
+      Password = Passw0rd2;
+      File Retention = 90 days;
+      Job Retention = 3 months;
+      AutoPrune = yes;
+    }
+
+
+.. _License:
+
+License
+-------
+
+GPLv3
+
+
+.. _Author:
+
+Author
+------
+
+Jiri Tyr

playbooks/roles/config-encoders/filter_plugins/config_encoders.py

+# (c) 2016, Jiri Tyr <jiri.tyr@gmail.com>
+#
+# This file is part of Config Encoder Filters (CEF)
+#
+# CEF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# CEF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CEF.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Config Encoder Filters
+
+More information: https://github.com/jtyr/ansible-config_encoder_filters
+"""
+
+from __future__ import (absolute_import, division, print_function)
+from ansible import errors
+from copy import copy
+import re
+
+
+def _str_is_bool(data):
+    """Verify if data is boolean."""
+
+    return re.match(r"^(true|false)$", str(data), flags=re.IGNORECASE)
+
+
+def _str_is_int(data):
+    """Verify if data is integer."""
+
+    return re.match(r"^[-+]?(0|[1-9][0-9]*)$", str(data))
+
+
+def _str_is_float(data):
+    """Verify if data is float."""
+
+    return re.match(
+        r"^[-+]?(0|[1-9][0-9]*)(\.[0-9]*)?(e[-+]?[0-9]+)?$",
+        str(data), flags=re.IGNORECASE)
+
+
+def _str_is_num(data):
+    """Verify if data is either integer or float."""
+
+    return _str_is_int(data) or _str_is_float(data)
+
+
+def _is_num(data):
+    """Verify if data is either int or float.
+
+    Could be replaced by:
+
+        from numbers import Number as number
+        isinstance(data, number)
+
+    but that requires Python v2.6+.
+    """
+
+    return isinstance(data, int) or isinstance(data, float)
+
+
+def _escape(data, quote='"', format=None):
+    """Escape special characters in a string."""
+
+    if format == 'xml':
+        return (
+            str(data).
+            replace('&', '&amp;').
+            replace('<', '&lt;').
+            replace('>', '&gt;'))
+    elif format == 'control':
+        return (
+            str(data).
+            replace('\b', '\\b').
+            replace('\f', '\\f').
+            replace('\n', '\\n').
+            replace('\r', '\\r').
+            replace('\t', '\\t'))
+    elif quote is not None and len(quote):
+        return str(data).replace('\\', '\\\\').replace(quote, "\\%s" % quote)
+    else:
+        return data
+
+
+def encode_apache(
+        data, convert_bools=False, convert_nums=False, indent="  ", level=0,
+        quote_all_nums=False, quote_all_strings=False, block_type='sections'):
+    """Convert Python data structure to Apache format."""
+
+    # Return value
+    rv = ""
+
+    if block_type == 'sections':
+        for c in data['content']:
+            # First check if this section has options
+            if 'options' in c:
+                rv += encode_apache(
+                    c['options'],
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    quote_all_nums=quote_all_nums,
+                    quote_all_strings=quote_all_strings,
+                    block_type='options')
+
+            is_empty = False
+
+            # Check if this section has some sub-sections
+            if 'sections' in c:
+                for s in c['sections']:
+                    # Check for empty sub-sections
+                    for i in s['content']:
+                        if (
+                                ('options' in i and len(i['options']) > 0) or
+                                ('sections' in i and len(i['sections']) > 0)):
+                            is_empty = True
+
+                    if is_empty:
+                        rv += "%s<%s " % (indent * level, s['name'])
+
+                        if 'operator' in s:
+                            rv += "%s " % s['operator']
+
+                        if 'param' in s:
+                            rv += encode_apache(
+                                s['param'],
+                                convert_bools=convert_bools,
+                                convert_nums=convert_nums,
+                                indent=indent,
+                                level=level+1,
+                                quote_all_nums=quote_all_nums,
+                                quote_all_strings=quote_all_strings,
+                                block_type='value')
+
+                        rv += ">\n"
+                        rv += encode_apache(
+                            s,
+                            convert_bools=convert_bools,
+                            convert_nums=convert_nums,
+                            indent=indent,
+                            level=level+1,
+                            quote_all_nums=quote_all_nums,
+                            quote_all_strings=quote_all_strings,
+                            block_type='sections')
+                        rv += "%s</%s>\n" % (indent * level, s['name'])
+
+                        # If not last item of the loop
+                        if c['sections'][-1] != s:
+                            rv += "\n"
+
+            if (
+                    data['content'][-1] != c and (
+                        'options' in c and len(c['options']) > 0 or (
+                            'sections' in c and
+                            len(c['sections']) > 0 and
+                            is_empty))):
+                rv += "\n"
+
+    elif block_type == 'options':
+        for o in data:
+            for key, val in sorted(o.iteritems()):
+                rv += "%s%s " % (indent * (level-1), key)
+                rv += encode_apache(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    quote_all_nums=quote_all_nums,
+                    quote_all_strings=quote_all_strings,
+                    block_type='value')
+                rv += "\n"
+
+    elif block_type == 'value':
+        if isinstance(data, bool) or convert_bools and _str_is_bool(data):
+            # Value is a boolean
+
+            rv += str(data).lower()
+
+        elif (
+                _is_num(data) or
+                (convert_nums and _str_is_num(data))):
+            # Value is a number
+            if quote_all_nums:
+                rv += '"%s"' % data
+            else:
+                rv += str(data)
+
+        elif isinstance(data, basestring):
+            # Value is a string
+            if (
+                    quote_all_strings or
+                    " " in data or
+                    "\t" in data or
+                    "\n" in data or
+                    "\r" in data or
+                    data == ""):
+
+                rv += '"%s"' % _escape(data)
+            else:
+                rv += data
+
+        elif isinstance(data, list):
+            # Value is a list
+            for v in data:
+                rv += encode_apache(
+                    v,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    quote_all_nums=quote_all_nums,
+                    quote_all_strings=quote_all_strings,
+                    block_type='value')
+
+                # If not last item of the loop
+                if data[-1] != v:
+                    rv += " "
+
+    return rv
+
+
+def encode_erlang(
+        data, atom_value_indicator=":", convert_bools=False,
+        convert_nums=False, indent="  ", level=0):
+    """Convert Python data structure to Erlang format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, dict):
+        # It's a dict
+
+        rv += "\n"
+
+        for key, val in sorted(data.iteritems()):
+            rv += "%s{%s," % (indent*level, key)
+
+            if not isinstance(val, dict):
+                rv += " "
+
+            rv += encode_erlang(
+                val,
+                convert_bools=convert_bools,
+                convert_nums=convert_nums,
+                indent=indent,
+                level=level+1)
+
+            rv += "}"
+    elif (
+            data == "null" or
+            _is_num(data) or
+            isinstance(data, bool) or
+            (convert_nums and _str_is_num(data)) or
+            (convert_bools and _str_is_bool(data))):
+        # It's null, number or boolean
+
+        rv += str(data).lower()
+
+    elif isinstance(data, basestring):
+        # It's a string
+
+        atom_len = len(atom_value_indicator)
+
+        if (
+                len(data) > atom_len and
+                data[0:atom_len] == atom_value_indicator):
+
+            # Atom configuration value
+            rv += data[atom_len:]
+        else:
+            rv += '"%s"' % _escape(data)
+
+    else:
+        # It's a list
+
+        rv += "["
+
+        for val in data:
+            if (
+                    isinstance(val, basestring) or
+                    _is_num(val)):
+                rv += "\n%s" % (indent*level)
+
+            rv += encode_erlang(
+                val,
+                convert_bools=convert_bools,
+                convert_nums=convert_nums,
+                indent=indent,
+                level=level+1)
+
+            if data[-1] == val:
+                # Last item of the loop
+                rv += "\n"
+            else:
+                rv += ","
+
+        if len(data) > 0:
+            rv += "%s]" % (indent * (level-1))
+        else:
+            rv += "]"
+
+        if level == 0:
+            rv += ".\n"
+
+    return rv
+
+
+def encode_haproxy(data, indent="  "):
+    """Convert Python data structure to HAProxy format."""
+
+    # Return value
+    rv = ""
+    # Indicates first loop
+    first = True
+    # Indicates whether the previous section was a comment
+    prev_comment = False
+
+    for section in data:
+        if first:
+            first = False
+        elif prev_comment:
+            prev_comment = False
+        else:
+            # Print empty line between sections
+            rv += "\n"
+
+        if isinstance(section, dict):
+            # It's a section
+            rv += "%s\n" % section.keys()[0]
+
+            # Process all parameters of the section
+            for param in section.values()[0]:
+                rv += "%s%s\n" % (indent, param)
+        else:
+            # It's a comment of a parameter
+            rv += "%s\n" % section
+            prev_comment = True
+
+    return rv
+
+
+def encode_ini(
+        data, comment="#", delimiter=" = ", quote="", section_is_comment=False,
+        ucase_prop=False):
+    """Convert Python data structure to INI format."""
+
+    # Return value
+    rv = ""
+
+    # First process all standalone properties
+    for prop, val in sorted(data.iteritems()):
+        if ucase_prop:
+            prop = prop.upper()
+
+        vals = []
+
+        if isinstance(val, list):
+            vals = val
+        elif not isinstance(val, dict):
+            vals = [val]
+
+        for item in vals:
+            if (
+                    len(quote) == 0 and
+                    isinstance(item, basestring) and
+                    len(item) == 0):
+                item = '""'
+
+            if item is not None:
+                rv += "%s%s%s%s%s\n" % (
+                    prop, delimiter, quote, _escape(item, quote), quote)
+
+    # Then process all sections
+    for section, props in sorted(data.iteritems()):
+        if isinstance(props, dict):
+            if rv != "":
+                rv += "\n"
+
+            if section_is_comment:
+                rv += "%s %s\n" % (comment, section)
+            else:
+                rv += "[%s]\n" % (section)
+
+            # Let process all section options as standalone properties
+            rv += encode_ini(
+                props,
+                delimiter=delimiter,
+                quote=quote,
+                section_is_comment=section_is_comment,
+                ucase_prop=ucase_prop)
+
+    return rv
+
+
+def encode_json(
+        data, convert_bools=False, convert_nums=False, indent="  ", level=0):
+    """Convert Python data structure to JSON format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, dict):
+        # It's a dict
+
+        rv += "{"
+
+        if len(data) > 0:
+            rv += "\n"
+
+        items = sorted(data.iteritems())
+
+        for key, val in items:
+            rv += '%s"%s": ' % (indent * (level+1), key)
+            rv += encode_json(
+                val,
+                convert_bools=convert_bools,
+                convert_nums=convert_nums,
+                indent=indent,
+                level=level+1)
+
+            # Last item of the loop
+            if items[-1] == (key, val):
+                rv += "\n"
+            else:
+                rv += ",\n"
+
+        if len(data) > 0:
+            rv += "%s}" % (indent * level)
+        else:
+            rv += "}"
+
+        if level == 0:
+            rv += "\n"
+
+    elif (
+            data == "null" or
+            _is_num(data) or
+            (convert_nums and _str_is_num(data)) or
+            (convert_bools and _str_is_bool(data))):
+        # It's a number, null or boolean
+
+        rv += str(data).lower()
+
+    elif isinstance(data, basestring):
+        # It's a string
+
+        rv += '"%s"' % _escape(_escape(data), format='control')
+
+    else:
+        # It's a list
+
+        rv += "["
+
+        if len(data) > 0:
+            rv += "\n"
+
+        for val in data:
+            rv += indent * (level+1)
+            rv += encode_json(
+                val,
+                convert_bools=convert_bools,
+                convert_nums=convert_nums,
+                indent=indent,
+                level=level+1)
+
+            # Last item of the loop
+            if data[-1] == val:
+                rv += "\n"
+            else:
+                rv += ",\n"
+
+        if len(data) > 0:
+            rv += "%s]" % (indent * level)
+        else:
+            rv += "]"
+
+    return rv
+
+
+def encode_logstash(
+        data, convert_bools=False, convert_nums=False, indent="  ", level=0,
+        prevtype="", section_prefix=":"):
+    """Convert Python data structure to Logstash format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, dict):
+        # The item is a dict
+
+        if prevtype in ('value', 'value_hash', 'array'):
+            rv += "{\n"
+
+        items = sorted(data.iteritems())
+
+        for key, val in items:
+            if key[0] == section_prefix:
+                rv += "%s%s {\n" % (indent * level, key[1:])
+                rv += encode_logstash(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    prevtype='block')
+
+                # Last item of the loop
+                if items[-1] == (key, val):
+                    if (
+                            isinstance(val, basestring) or
+                            _is_num(val) or
+                            isinstance(val, bool) or (
+                                isinstance(val, dict) and
+                                val.keys()[0][0] != section_prefix)):
+                        rv += "\n%s}\n" % (indent * level)
+                    else:
+                        rv += "%s}\n" % (indent * level)
+            else:
+                rv += indent * level
+
+                if prevtype == 'value_hash':
+                    rv += '"%s" => ' % key
+                else:
+                    rv += "%s => " % key
+
+                rv += encode_logstash(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    prevtype=(
+                        'value_hash' if isinstance(val, dict) else 'value'))
+
+            if (
+                    items[-1] != (key, val) and (
+                        isinstance(val, basestring) or
+                        _is_num(val) or
+                        isinstance(val, bool))):
+                rv += "\n"
+
+        if prevtype in ('value', 'value_hash', 'array'):
+            rv += "\n%s}" % (indent * (level-1))
+
+            if prevtype in ('value', 'value_array'):
+                rv += "\n"
+
+    elif (
+            _is_num(data) or
+            isinstance(data, bool) or
+            (convert_nums and _str_is_num(data)) or
+            (convert_bools and _str_is_bool(data))):
+        # It's number or boolean
+
+        rv += str(data).lower()
+
+    elif isinstance(data, basestring):
+        # It's a string
+
+        rv += '"%s"' % _escape(data)
+
+    else:
+        # It's a list
+
+        for val in data:
+            if isinstance(val, dict) and val.keys()[0][0] == section_prefix:
+                # Value is a block
+
+                rv += encode_logstash(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level,
+                    prevtype='block')
+            else:
+                # First item of the loop
+                if data[0] == val:
+                    rv += "[\n"
+
+                rv += indent * level
+                rv += encode_logstash(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    prevtype='array')
+
+                # Last item of the loop
+                if data[-1] == val:
+                    rv += "\n%s]" % (indent * (level-1))
+                else:
+                    rv += ",\n"
+
+    return rv
+
+
+def encode_nginx(data, indent="  ", level=0, block_semicolon=False):
+    """Convert Python data structure to Nginx format."""
+
+    # Return value
+    rv = ""
+    # Indicates the item type [section|line]
+    item_type = ""
+
+    for item in data:
+        if isinstance(item, dict):
+            # Section
+            if item_type in ('section', 'line'):
+                rv += "\n"
+
+            rv += "%s%s {\n" % (level*indent, item.keys()[0])
+            rv += encode_nginx(
+                item.values()[0],
+                level=level+1,
+                block_semicolon=block_semicolon)
+            rv += "%s}%s\n" % (level*indent, ';' if block_semicolon else '')
+
+            item_type = 'section'
+
+        elif isinstance(item, basestring):
+            # Normal line
+            if item_type == 'section':
+                rv += "\n"
+
+            item_type = 'line'
+
+            rv += "%s%s" % (level*indent, item)
+
+            # Do not finish comments with semicolon
+            if item.startswith("# "):
+                rv += "\n"
+            else:
+                rv += ";\n"
+
+        else:
+            raise errors.AnsibleFilterError(
+                "Unexpected data type: %s" % (type(item)))
+
+    return rv
+
+
+def encode_pam(
+        data, print_label=False, separate_types=True, separator="  "):
+    """Convert Python data structure to PAM format."""
+
+    # Return value
+    rv = ""
+    # Remember previous type to make newline between type blocks
+    prev_type = None
+
+    for label, rule in sorted(data.iteritems()):
+        if separate_types:
+            # Add extra newline to separate blocks of the same type
+            if prev_type is not None and prev_type != rule['type']:
+                rv += "\n"
+
+            prev_type = rule['type']
+
+        if print_label:
+            rv += "# %s\n" % label
+
+        if 'service' in rule:
+            rv += "%s%s" % (rule['service'], separator)
+
+        if 'silent' in rule and rule['silent']:
+            rv += '-'
+
+        rv += "%s%s" % (rule['type'], separator)
+
+        if isinstance(rule['control'], list):
+            rv += "[%s]%s" % (
+                " ".join(
+                    map(
+                        lambda k: "=".join(map(str, k)),
+                        map(lambda x: x.items()[0], rule['control']))),
+                separator)
+        else:
+            rv += "%s%s" % (rule['control'], separator)
+
+        rv += rule['path']
+
+        if 'args' in rule and rule['args']:
+            rv += separator
+
+            for i, arg in enumerate(rule['args']):
+                if i > 0:
+                    rv += ' '
+
+                if isinstance(arg, dict):
+                    rv += "=".join(map(str, arg.items()[0]))
+                else:
+                    rv += arg
+
+        rv += "\n"
+
+    return rv
+
+
+def encode_toml(
+        data, convert_bools=False, convert_nums=False, first=True,
+        indent="  ", level=0, prevkey="", prevtype="", quote='"'):
+    """Convert Python data structure to TOML format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, dict):
+        # It's a dict
+
+        # First process all standalone strings, numbers, booleans and lists
+        for key, val in sorted(data.iteritems()):
+            if (
+                    isinstance(val, basestring) or
+                    _is_num(val) or
+                    isinstance(val, bool) or (
+                        isinstance(val, list) and
+                        len(val) > 0 and
+                        not isinstance(val[0], dict))):
+                # The value is string, number, boolean or list
+
+                rv += "%s%s = " % (indent * level, key)
+                rv += encode_toml(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    first=first,
+                    indent=indent,
+                    level=level,
+                    prevkey=prevkey)
+
+                first = False
+
+        # Then process all data structures
+        for key, val in sorted(data.iteritems()):
+            if (
+                    isinstance(val, dict) or
+                    isinstance(val, list) and isinstance(val[0], dict)):
+
+                # Values for the next recursive call
+                tmp_prevkey = prevkey
+                tmp_level = level
+
+                if isinstance(val, dict):
+                    # The val is a dict
+                    if prevkey != "" and prevkey != key:
+                        tmp_level += 1
+
+                    if re.match(r'^[a-zA-Z0-9_-]+$', key) is None:
+                        key = '"%s"' % key
+
+                    if prevkey == "":
+                        tmp_prevkey = key
+                    else:
+                        tmp_prevkey = "%s.%s" % (prevkey, key)
+
+                    if not first:
+                        rv += "\n"
+
+                    rv += "%s[%s]\n" % (indent * tmp_level, tmp_prevkey)
+                elif isinstance(val[0], dict):
+                    # The val is a table
+                    if re.match(r'^[a-zA-Z0-9_-]+$', key) is None:
+                        key = '"%s"' % key
+
+                    if prevkey == "":
+                        tmp_prevkey = key
+                    else:
+                        tmp_prevkey = "%s.%s" % (prevkey, key)
+
+                    tmp_level += 1
+
+                rv += encode_toml(
+                    val,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    first=first,
+                    indent=indent,
+                    level=tmp_level,
+                    prevkey=tmp_prevkey)
+
+                first = False
+
+    elif (
+            _is_num(data) or
+            isinstance(data, bool) or
+            (convert_nums and _str_is_num(data)) or
+            (convert_bools and _str_is_bool(data))):
+        # It's number or boolean
+
+        rv += str(data).lower()
+
+        if prevtype != 'list':
+            rv += "\n"
+
+    elif isinstance(data, basestring):
+        # It's a string
+
+        rv += "%s%s%s" % (
+            quote, _escape(data, quote), quote)
+
+        if prevtype != 'list':
+            rv += "\n"
+
+    else:
+        # It's a list
+
+        if isinstance(data[0], dict):
+            for d in data:
+                rv += "\n%s[[%s]]\n" % (indent * level, prevkey)
+                rv += encode_toml(
+                    d,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    first=first,
+                    indent=indent,
+                    level=level)
+        else:
+            rv += "["
+
+            for d in data:
+                rv += encode_toml(
+                    d,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    first=first,
+                    indent=indent,
+                    level=level,
+                    prevtype='list')
+
+                # Last item of the loop
+                if data[-1] != d:
+                    rv += ", "
+
+            rv += "]"
+
+            if prevtype != 'list':
+                rv += "\n"
+
+    return rv
+
+
+def encode_xml(
+        data, attribute_sign="^", escape_xml=True, indent="  ", level=0):
+    """Convert Python data structure to XML format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, list):
+        # Pocess anything what's not attribute
+        for item in data:
+            if (
+                    not (
+                        isinstance(item, dict) and
+                        item.keys()[0].startswith(attribute_sign))):
+                rv += encode_xml(
+                    item,
+                    attribute_sign=attribute_sign,
+                    indent=indent,
+                    level=level,
+                    escape_xml=escape_xml)
+    elif isinstance(data, dict):
+        # It's eiher an attribute or an element
+
+        key, val = data.items()[0]
+
+        if key.startswith(attribute_sign):
+            # Process attribute
+            rv += ' %s="%s"' % (key[1:], _escape(val))
+        else:
+            # Process element
+            rv = '%s<%s' % (level*indent, key)
+
+            # Check if there are any attributes
+            if isinstance(val, list):
+                num_attrs = 0
+
+                for item in val:
+                    if (
+                            isinstance(item, dict) and
+                            item.keys()[0].startswith(attribute_sign)):
+                        num_attrs += 1
+                        rv += encode_xml(
+                            item,
+                            attribute_sign=attribute_sign,
+                            indent=indent,
+                            level=level)
+
+            if val == '' or (isinstance(val, list) and num_attrs == len(val)):
+                # Close the element as empty
+                rv += " />\n"
+            else:
+                # Close the element as normal
+                rv += ">"
+
+                # Check if the value is text
+                val_not_text = False
+
+                if isinstance(val, list):
+                    # Check if it contains only attributes and a text value
+                    for item in val:
+                        if (
+                                isinstance(item, dict) and
+                                not item.keys()[0].startswith(attribute_sign)):
+                            val_not_text = True
+                            break
+                elif isinstance(val, dict):
+                    val_not_text = True
+
+                if val_not_text:
+                    rv += "\n"
+
+                # Process inner content of the element
+                rv += encode_xml(
+                    val,
+                    attribute_sign=attribute_sign,
+                    indent=indent,
+                    level=level+1,
+                    escape_xml=escape_xml)
+
+                if val_not_text:
+                    rv += level*indent
+
+                rv += "</%s>\n" % key
+    else:
+        # It's a string
+
+        rv += "%s" % _escape(data, format=('xml' if escape_xml else None))
+
+    return rv
+
+
+def encode_yaml(
+        data, convert_bools=False, convert_nums=False, indent="  ", level=0,
+        quote='"', skip_indent=False):
+    """Convert Python data structure to YAML format."""
+
+    # Return value
+    rv = ""
+
+    if isinstance(data, dict):
+        # It's a dictionary
+
+        if len(data.keys()) == 0:
+            rv += "{}\n"
+        else:
+            for i, (key, val) in enumerate(sorted(data.iteritems())):
+                # Skip indentation only for the first pair
+                rv += "%s%s:" % ("" if i == 0 and skip_indent else level*indent, key)
+
+                if isinstance(val, dict) and len(val.keys()) == 0:
+                    rv += " {}\n"
+                else:
+                    if (
+                            isinstance(val, dict) or (
+                                isinstance(val, list) and
+                                len(val) != 0)):
+                        rv += "\n"
+                    else:
+                        rv += " "
+
+                    rv += encode_yaml(
+                        val,
+                        convert_bools=convert_bools,
+                        convert_nums=convert_nums,
+                        indent=indent,
+                        level=level+1,
+                        quote=quote)
+
+    elif isinstance(data, list):
+        # It's a list
+
+        if len(data) == 0:
+            rv += "[]\n"
+        else:
+            for item in data:
+                list_indent = "%s- " % (level*indent)
+
+                rv += "%s%s" % (list_indent, encode_yaml(
+                    item,
+                    convert_bools=convert_bools,
+                    convert_nums=convert_nums,
+                    indent=indent,
+                    level=level+1,
+                    quote=quote,
+                    skip_indent=True))
+
+    elif (
+            data == "null" or
+            isinstance(data, bool) or
+            (convert_bools and _str_is_bool(data))):
+        # It's a boolean
+
+        rv += "%s\n" % str(data).lower()
+
+    elif (
+            _is_num(data) or
+            (convert_nums and _str_is_num(data))):
+        # It's a number
+
+        rv += "%s\n" % str(data)
+
+    else:
+        # It's a string
+
+        rv += "%s%s%s\n" % (quote, _escape(data, quote), quote)
+
+    return rv
+
+
+def __eval_replace(match):
+    """Evaluate the real value of the variable specified as a string."""
+
+    ret = '__item'
+    ret += ''.join(match.groups()[1:])
+
+    # Try to evaluate the value of the special string
+    try:
+        ret = eval(ret)
+    except Exception:
+        # Return empty string if something went wrong
+        ret = ''
+
+    return str(ret)
+
+
+def template_replace(data, replacement):
+    """Replace special template decorated variable with its real value."""
+
+    # Make the replacement variable visible for the __eval_replace function
+    global __item
+    __item = replacement
+
+    # Clone the data to keep the original untouched
+    local_data = copy(data)
+
+    # Walk through the data structure and try to replace all special strings
+    if isinstance(local_data, list):
+        local_data = map(
+            lambda x: template_replace(x, replacement), local_data)
+    elif isinstance(local_data, dict):
+        for key, val in local_data.iteritems():
+            local_data[key] = template_replace(val, replacement)
+    elif isinstance(local_data, basestring):
+        # Replace the special string by it's evaluated value
+        p = re.compile(r'\{\[\{\s*(\w+)([^}\s]+|)\s*\}\]\}')
+        local_data = p.sub(__eval_replace, local_data)
+
+    return local_data
+
+
+class FilterModule(object):
+    """Ansible encoder Jinja2 filters."""
+
+    def filters(self):
+        """Expose filters to ansible."""
+
+        return {
+            'encode_apache': encode_apache,
+            'encode_erlang': encode_erlang,
+            'encode_haproxy': encode_haproxy,
+            'encode_ini': encode_ini,
+            'encode_json': encode_json,
+            'encode_logstash': encode_logstash,
+            'encode_nginx': encode_nginx,
+            'encode_pam': encode_pam,
+            'encode_toml': encode_toml,
+            'encode_xml': encode_xml,
+            'encode_yaml': encode_yaml,
+            'template_replace': template_replace,
+        }

playbooks/roles/config-encoders/meta/main.yml

+---
+dependencies: []

playbooks/roles/credentials/defaults/main.yml

@@ -63,14 +63,8 @@ CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
 
 CREDENTIALS_PLATFORM_NAME: 'Your Platform Name Here'
 
-# credentials service user in programs service and lms
 CREDENTIALS_SERVICE_USER: 'credentials_service_user'
 
-# Absolute URL used to get programs from the programs service.
-CREDENTIALS_PROGRAMS_API_URL: 'https://127.0.0.1:8004/api/v1/'
-CREDENTIALS_PROGRAMS_API_JWT_AUDIENCE: 'SET-ME-TO-THE-SAME-AS-PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY'
-CREDENTIALS_PROGRAMS_API_JWT_SECRET_KEY: 'SET-ME-TO-THE-SAME-AS-PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET'
-
 # Absolute URL used to get organization data from the organizations api in LMS
 CREDENTIALS_ORGANIZATIONS_API_URL: 'https://127.0.0.1:8000/api/organizations/v0/'
 CREDENTIALS_ORGANIZATIONS_API_AUDIENCE: '{{ EDXAPP_JWT_AUDIENCE | default("lms-key") }}'
@@ -193,14 +187,8 @@ CREDENTIALS_SERVICE_CONFIG:
   # Set credentials files storage backend
   FILE_STORAGE_BACKEND: '{{ CREDENTIALS_FILE_STORAGE_BACKEND }}'
 
-  # credentials service user in programs service and lms
   CREDENTIALS_SERVICE_USER: '{{ CREDENTIALS_SERVICE_USER }}'
 
-  # programs api configuration
-  PROGRAMS_API_URL: '{{ CREDENTIALS_PROGRAMS_API_URL }}'
-  PROGRAMS_JWT_AUDIENCE: '{{ CREDENTIALS_PROGRAMS_API_JWT_AUDIENCE }}'
-  PROGRAMS_JWT_SECRET_KEY: '{{ CREDENTIALS_PROGRAMS_API_JWT_SECRET_KEY }}'
-
   # organizations api configuration
   ORGANIZATIONS_API_URL: '{{ CREDENTIALS_ORGANIZATIONS_API_URL }}'
   ORGANIZATIONS_AUDIENCE: '{{ CREDENTIALS_ORGANIZATIONS_API_AUDIENCE }}'

playbooks/roles/discovery/defaults/main.yml

@@ -89,7 +89,6 @@ DISCOVERY_COURSES_API_URL: '{{ DISCOVERY_LMS_ROOT_URL }}/api/courses/v1/'
 DISCOVERY_ORGANIZATIONS_API_URL: '{{ DISCOVERY_LMS_ROOT_URL }}/api/organizations/v0/'
 DISCOVERY_MARKETING_API_URL: 'https://example.org/api/catalog/v2/'
 DISCOVERY_MARKETING_URL_ROOT: 'https://example.org/'
-DISCOVERY_PROGRAMS_API_URL: 'https://replace-me/api/v1/'
 
 DISCOVERY_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ discovery_service_name }}'
 DISCOVERY_MEDIA_ROOT: '{{ DISCOVERY_DATA_DIR }}/media'
@@ -150,7 +149,6 @@ DISCOVERY_SERVICE_CONFIG:
   ORGANIZATIONS_API_URL: '{{ DISCOVERY_ORGANIZATIONS_API_URL }}'
   MARKETING_API_URL: '{{ DISCOVERY_MARKETING_API_URL }}'
   MARKETING_URL_ROOT: '{{ DISCOVERY_MARKETING_URL_ROOT }}'
-  PROGRAMS_API_URL: '{{ DISCOVERY_PROGRAMS_API_URL }}'
 
   EDX_DRF_EXTENSIONS: '{{ DISCOVERY_EDX_DRF_EXTENSIONS }}'
 

playbooks/roles/ecommerce/tasks/main.yml

@@ -68,6 +68,24 @@
     - install
     - install:app-requirements
 
+# This is a hacked fix for the fact that the table `thumbnail_kvstore` exists in
+# some environments, which won't need the 3rd party newly introduced migration
+# to create this table, so we fake the migration.
+# This is required for the Ginkgo release.
+# TODO: Delete this task for the Hawthorn release.
+- name: fake thumbnails
+  shell: >
+    table_exists=`mysql -uroot -ss -e "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_schema = '{{ ECOMMERCE_DEFAULT_DB_NAME }}' AND table_name = 'thumbnail_kvstore')"`;
+    if [ "$table_exists" -eq "1" ]; then {{ ecommerce_venv_dir }}/bin/python ./manage.py migrate thumbnail 0001 --fake; fi;
+  args:
+    chdir: "{{ ecommerce_code_dir }}"
+  become_user: "{{ ecommerce_user }}"
+  environment: "{{ ecommerce_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+  tags:
+    - migrate
+    - migrate:db
+
 - name: Migrate
   shell: >
     DB_MIGRATION_USER='{{ COMMON_MYSQL_MIGRATE_USER }}'
@@ -83,19 +101,12 @@
     - migrate:db
 
 - name: Populate countries
-  shell: "DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }} DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }} {{ ecommerce_venv_dir }}/bin/python ./manage.py oscar_populate_countries"
+  shell: "DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }} DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }} {{ ecommerce_venv_dir }}/bin/python ./manage.py oscar_populate_countries --initial-only"
   args:
     chdir: "{{ ecommerce_code_dir }}"
   become_user: "{{ ecommerce_user }}"
   environment: "{{ ecommerce_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
-  # the `register` and `failed_when` directives below are here to emulate idempotency for this oscar command.
-  # if and when https://github.com/django-oscar/django-oscar/pull/1841 is merged, the directives can be removed
-  # in favor of the (proposed) --initial-only command option.
-  register: command_result
-  failed_when:
-    - "'You already have countries in your database' not in command_result.stderr"
-    - "command_result.rc != 0"
   tags:
     - migrate
     - migrate:db

playbooks/roles/edx_ansible/templates/update.j2

@@ -13,7 +13,7 @@ IFS=","
             -h        this
 
     <repo> - must be one of edx-platform, edx-workers, xqueue, cs_comments_service, credentials, xserver, configuration,
-    read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, programs, course_discovery,
+    read-only-certificate-code, edx-analytics-data-api, edx-ora2, insights, ecommerce, course_discovery,
     notifier
     <version> - can be a commit or tag
 
@@ -59,7 +59,6 @@ repos_to_cmd["edx-analytics-data-api"]="$edx_ansible_cmd analyticsapi.yml -e 'AN
 repos_to_cmd["edx-ora2"]="$edx_ansible_cmd ora2.yml -e 'ora2_version=$2'"
 repos_to_cmd["insights"]="$edx_ansible_cmd insights.yml -e 'INSIGHTS_VERSION=$2'"
 repos_to_cmd["ecommerce"]="$edx_ansible_cmd ecommerce.yml -e 'ECOMMERCE_VERSION=$2'"
-repos_to_cmd["programs"]="$edx_ansible_cmd programs.yml -e 'PROGRAMS_VERSION=$2'"
 repos_to_cmd["discovery"]="$edx_ansible_cmd discovery.yml -e 'DISCOVERY_VERSION=$2'"
 repos_to_cmd["notifier"]="$edx_ansible_cmd notifier.yml -e 'NOTIFIER_VERSION=$2'"
 

playbooks/roles/edxapp/defaults/main.yml

@@ -534,6 +534,12 @@ EDXAPP_ECOMMERCE_API_URL: "http://localhost:8002/api/v2"
 EDXAPP_ECOMMERCE_API_SIGNING_KEY: "{{ EDXAPP_JWT_SECRET_KEY }}"
 EDXAPP_COURSE_CATALOG_API_URL: "http://localhost:8008/api/v1"
 
+# which access.py permission name to check in order to determine if a course about page is
+# visible. We default this to the legacy permission 'see_exists'.
+
+EDXAPP_COURSE_CATALOG_VISIBILITY_PERMISSION: 'see_exists'
+EDXAPP_COURSE_ABOUT_VISIBILITY_PERMISSION: 'see_exists'
+
 # Mailchimp Settings
 EDXAPP_MAILCHIMP_NEW_USER_LIST_ID: null
 
@@ -629,6 +635,21 @@ EDXAPP_SESSION_SAVE_EVERY_REQUEST: false
 
 EDXAPP_SESSION_COOKIE_SECURE: false
 
+# Course Block Structures
+EDXAPP_BLOCK_STRUCTURES_SETTINGS:
+  # Delay, in seconds, after a new edit of a course is published
+  # before updating the block structures cache.  This is needed
+  # for a better chance at getting the latest changes when there
+  # are secondary reads in sharded mongoDB clusters. See TNL-5041
+  # for more info.
+  COURSE_PUBLISH_TASK_DELAY: 30
+
+  # Delay, in seconds, between retry attempts if a task fails.
+  TASK_DEFAULT_RETRY_DELAY: 30
+
+  # Maximum number of retries per task.
+  TASK_MAX_RETRIES: 5
+
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
@@ -809,6 +830,8 @@ generic_env_config:  &edxapp_generic_env
   ECOMMERCE_PUBLIC_URL_ROOT: "{{ EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT }}"
   ECOMMERCE_API_URL: "{{ EDXAPP_ECOMMERCE_API_URL }}"
   COURSE_CATALOG_API_URL: "{{ EDXAPP_COURSE_CATALOG_API_URL }}"
+  COURSE_CATALOG_VISIBILITY_PERMISSION: "{{ EDXAPP_COURSE_CATALOG_VISIBILITY_PERMISSION }}"
+  COURSE_ABOUT_VISIBILITY_PERMISSION: "{{ EDXAPP_COURSE_ABOUT_VISIBILITY_PERMISSION }}"
   FINANCIAL_REPORTS: "{{ EDXAPP_FINANCIAL_REPORTS }}"
   ONLOAD_BEACON_SAMPLE_RATE: "{{ EDXAPP_ONLOAD_BEACON_SAMPLE_RATE }}"
   CORS_ORIGIN_WHITELIST: "{{ EDXAPP_CORS_ORIGIN_WHITELIST }}"
@@ -951,6 +974,7 @@ generic_env_config:  &edxapp_generic_env
   REGISTRATION_EXTRA_FIELDS: "{{ EDXAPP_REGISTRATION_EXTRA_FIELDS }}"
   XBLOCK_SETTINGS: "{{ EDXAPP_XBLOCK_SETTINGS }}"
   EDXMKTG_USER_INFO_COOKIE_NAME: "{{ EDXAPP_EDXMKTG_USER_INFO_COOKIE_NAME }}"
+  BLOCK_STRUCTURES_SETTINGS: "{{ EDXAPP_BLOCK_STRUCTURES_SETTINGS }}"
 
   # Deprecated, maintained for backward compatibility
   COMPREHENSIVE_THEME_DIR: "{{ EDXAPP_COMPREHENSIVE_THEME_DIR }}"

playbooks/roles/edxapp/templates/workers.conf.j2

@@ -10,6 +10,11 @@ stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 command={{ edxapp_venv_dir + '/bin/newrelic-admin run-program ' if w.monitor and COMMON_ENABLE_NEWRELIC_APP else ''}}{{ edxapp_venv_bin }}/python {{ edxapp_code_dir }}/manage.py {{ w.service_variant }} --settings={{ worker_django_settings_module }} celery worker --loglevel=info --queues=edx.{{ w.service_variant }}.core.{{ w.queue }} --hostname=edx.{{ w.service_variant }}.core.{{ w.queue }}.%%h --concurrency={{ w.concurrency }} {{ '--maxtasksperchild ' + w.max_tasks_per_child|string if w.max_tasks_per_child is defined else '' }}
 killasgroup=true
 stopwaitsecs={{ w.stopwaitsecs | default(EDXAPP_WORKER_DEFAULT_STOPWAITSECS) }}
+; Set autorestart to `true`. The default value for autorestart is `unexpected`, but celery < 4.x will exit
+; with an exit code of zero for certain types of unrecoverable errors, so we must make sure that the workers
+; are auto restarted even when exiting with code 0.
+; The Celery bug was reported in https://github.com/celery/celery/issues/2024, and is fixed in Celery 4.0.0.
+autorestart=true
 
 {% endfor %}
 

playbooks/roles/edxlocal/defaults/main.yml

@@ -10,7 +10,6 @@ edxlocal_databases:
   - "{{ EDXAPP_MYSQL_DB_NAME | default(None) }}"
   - "{{ EDXAPP_MYSQL_CSMH_DB_NAME | default(None) }}"
   - "{{ EDX_NOTES_API_MYSQL_DB_NAME | default(None) }}"
-  - "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}"
   - "{{ ANALYTICS_API_DEFAULT_DB_NAME | default(None) }}"
   - "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}"
   - "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}"
@@ -24,8 +23,8 @@ edxlocal_database_users:
     }
   - {
       db: "{{ INSIGHTS_DATABASE_NAME | default(None) }}",
-      user: "{{ INSIGHTS_MYSQL_USER | default(None) }}",
-      pass: "{{ INSIGHTS_MYSQL_USER | default(None) }}"
+      user: "{{ INSIGHTS_DATABASE_USER | default(None) }}",
+      pass: "{{ INSIGHTS_DATABASE_PASSWORD | default(None) }}"
     }
   - {
       db: "{{ XQUEUE_MYSQL_DB_NAME | default(None) }}",
@@ -43,11 +42,6 @@ edxlocal_database_users:
       pass: "{{ EDXAPP_MYSQL_CSMH_PASSWORD | default(None) }}"
     }
   - {
-      db: "{{ PROGRAMS_DEFAULT_DB_NAME | default(None) }}",
-      user: "{{ PROGRAMS_DATABASE_USER | default(None) }}",
-      pass: "{{ PROGRAMS_DATABASE_PASSWORD | default(None) }}"
-    }
-  - {
       db: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_NAME | default(None) }}",
       user: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_USER | default(None) }}",
       pass: "{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE_PASSWORD | default(None) }}"

playbooks/roles/elasticsearch/meta/main.yml

 ---
 dependencies:
   - common
+  - oraclejdk

playbooks/roles/go-server/defaults/main.yml

@@ -53,6 +53,14 @@ GO_SERVER_BACKUP_PASSWORD: ""
 GO_SERVER_GOMATIC_USERNAME: ""
 GO_SERVER_GOMATIC_PASSWORD: ""
 
+GO_SERVER_USERS:
+    - username: "{{ GO_SERVER_ADMIN_USERNAME }}"
+      password: "{{ GO_SERVER_ADMIN_PASSWORD }}"
+    - username: "{{ GO_SERVER_BACKUP_USERNAME }}"
+      password: "{{ GO_SERVER_BACKUP_PASSWORD }}"
+    - username: "{{ GO_SERVER_GOMATIC_USERNAME }}"
+      password: "{{ GO_SERVER_GOMATIC_PASSWORD }}"
+
 # go-server network settings
 GO_SERVER_PORT: 8153
 GO_SERVER_SECURE_PORT: 8154

playbooks/roles/go-server/tasks/main.yml

@@ -74,31 +74,6 @@
     - { url: "{{ GO_SERVER_OAUTH_LOGIN_JAR_URL }}", md5: "{{ GO_SERVER_OAUTH_LOGIN_MD5 }}" }
     - { url: "{{ GO_SERVER_GITHUB_PR_PLUGIN_JAR_URL }}", md5: "{{ GO_SERVER_GITHUB_PR_PLUGIN_MD5 }}" }
 
-- name: generate line for go-server password file for admin user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_ADMIN_USERNAME }}\" \"{{ GO_SERVER_ADMIN_PASSWORD }}\""
-  register: admin_user_password_line
-  when: GO_SERVER_ADMIN_USERNAME and GO_SERVER_ADMIN_PASSWORD
-
-- name: generate line for go-server password file for backup user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_BACKUP_USERNAME }}\" \"{{ GO_SERVER_BACKUP_PASSWORD }}\""
-  register: backup_user_password_line
-  when: GO_SERVER_BACKUP_USERNAME and GO_SERVER_BACKUP_PASSWORD
-
-- name: generate line for go-server password file for gomatic user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_GOMATIC_USERNAME }}\" \"{{ GO_SERVER_GOMATIC_PASSWORD }}\""
-  register: gomatic_user_password_line
-  when: GO_SERVER_GOMATIC_USERNAME and GO_SERVER_GOMATIC_PASSWORD
-
-- name: setup password file
-  template:
-    src: edx/app/go-server/password.txt.j2
-    dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
-    mode: 0600
-    owner: "{{ GO_SERVER_USER }}"
-    group: "{{ GO_SERVER_GROUP }}"
-    force: no
-  when: GO_SERVER_ADMIN_PASSWORD and GO_SERVER_BACKUP_PASSWORD and GO_SERVER_GOMATIC_PASSWORD
-
 - name: install go-server configuration
   template:
     src: edx/app/go-server/cruise-config.xml.j2
@@ -108,19 +83,43 @@
     group: "{{ GO_SERVER_GROUP }}"
     force: no
 
+- name: ensure everyone can read go-server log files
+  file:
+    path: "/var/log/go-server"
+    state: directory
+    mode: "0755"
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+  tags:
+    - install
+    - install:base
+
 - include: download_backup.yml
   when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT and GO_SERVER_RESTORE_BACKUP
 
-- name: replace the password lines in the password file post-restore
+- name: generate lines for go-server password file for users
+  command: "/usr/bin/htpasswd -nbs '{{ item.username }}' '{{ item.password }}'"
+  with_items: "{{ GO_SERVER_USERS }}"
+  when: item.username and item.password
+  register: go_server_users_htpasswds
+  tags:
+    - install
+    - install:app-configuration
+
+- name: replace the password lines in the password file
   lineinfile:
     dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
-    regexp: "^{{ item.username }}"
-    line: "{{ item.password_hash.stdout }}"
-  with_items:
-    - { username: "{{ GO_SERVER_ADMIN_USERNAME }}", password: "{{ GO_SERVER_ADMIN_PASSWORD }}", password_hash: "{{ admin_user_password_line }}" }
-    - { username: "{{ GO_SERVER_BACKUP_USERNAME }}", password: "{{ GO_SERVER_BACKUP_PASSWORD }}", password_hash: "{{ backup_user_password_line }}" }
-    - { username: "{{ GO_SERVER_GOMATIC_USERNAME }}", password: "{{ GO_SERVER_GOMATIC_PASSWORD }}", password_hash: "{{ gomatic_user_password_line }}" }
-  when: item.username and item.password and item.password_hash
+    regexp: "^{{ item.item.username }}"
+    line: "{{ item.stdout }}"
+    create: yes
+    mode: "0600"
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+  with_items: "{{ go_server_users_htpasswds.results }}"
+  when: item.item.username and item.item.password and item.stdout
+  tags:
+    - install
+    - install:app-configuration
 
 - name: install ssh key for the secure repos
   copy:

playbooks/roles/go-server/templates/edx/app/go-server/password.txt.j2

-{{ admin_user_password_line.stdout }}
-{{ backup_user_password_line.stdout }}
-{{ gomatic_user_password_line.stdout }}

playbooks/roles/graphite/LICENSE

+Originally authored by Hector Castro (https://github.com/azavea/ansible-graphite)
+
+-------------------------------------------------------------------------------
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

playbooks/roles/graphite/defaults/main.yml

+---
+GRAPHITE_CARBON_GIT_URL: "https://github.com/graphite-project/carbon.git"
+GRAPHITE_WHISPER_GIT_URL: "https://github.com/graphite-project/whisper.git"
+GRAPHITE_API_GIT_URL: "https://github.com/brutasse/graphite-api.git"
+
+GRAPHITE_CARBON_VERSION: "816ac631efae33c7b47ecbe79ca9e1f907e5efe8"
+GRAPHITE_WHISPER_VERSION: "0.9.15"
+GRAPHITE_API_VERSION: "1.1.3"
+
+graphite_root: "/opt/graphite"
+graphite_storage_root: "/var/opt/graphite"
+graphite_common_dirs:
+  root:
+    path: "{{ graphite_root }}"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  src:
+    path: "{{ graphite_root }}/src"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  conf:
+    path: "{{ graphite_root }}/conf"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+  run:
+    path: "{{ graphite_root }}/run"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0775"
+  storage:
+    path: "{{ graphite_storage_root }}"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0775"
+  log:
+    path: "/var/log/carbon"
+    owner: "{{ carbon_user }}"
+    group: "{{ graphite_group }}"
+    mode: "0755"
+
+graphite_common_deb_deps:
+  - libcairo2-dev
+  - libffi-dev
+  - pkg-config
+  - fontconfig
+
+carbon_user: "carbon"
+graphite_user: "graphite"
+graphite_group: "graphite"
+
+GRAPHITE_NGINX_PORT: 6000
+GRAPHITE_GUNICORN_WORKERS: 3
+
+CARBON_LISTEN_ADDR: 0.0.0.0
+
+CARBON_CONF:
+  cache:
+    USER: "{{ carbon_user }}"
+    STORAGE_DIR: "{{ graphite_storage_root }}"
+    CACHE_WRITE_STRATEGY: max
+    MAX_CACHE_SIZE: inf
+    USE_FLOW_CONTROL:  True
+    WHISPER_FALLOCATE_CREATE: True
+    MAX_CREATES_PER_MINUTE: 100
+    MAX_UPDATES_PER_SECOND: 1000
+    LINE_RECEIVER_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    PICKLE_RECEIVER_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    USE_INSECURE_UNPICKLER: False
+    CACHE_QUERY_INTERFACE: "{{ CARBON_LISTEN_ADDR }}"
+    LOG_CACHE_HITS: False
+    LOG_CACHE_QUEUE_SORTS: True
+    LOG_LISTENER_CONNECTIONS: True
+    LOG_UPDATES: False
+    ENABLE_LOGROTATION: True
+    WHISPER_AUTOFLUSH: False
+
+  "cache:1":
+    LINE_RECEIVER_PORT: 2003
+    PICKLE_RECEIVER_PORT: 2004
+    CACHE_QUERY_PORT: 7002
+
+CARBON_STORAGE_SCHEMAS:
+  carbon:
+    description: "Catches all of Carbon's internal metrics"
+    pattern: "carbon.*"
+    retentions: "10s:1y"
+  default:
+    pattern: ".*"
+    retentions: "60s:1y"
+
+CARBON_STORAGE_AGGREGATIONS:
+  min:
+    pattern: "\\.min$"
+    xFilesFactor: "0.1"
+    aggregationMethod: "min"
+  max:
+    pattern: "\\.max$"
+    xFilesFactor: "0.1"
+    aggregationMethod: "max"
+  sum:
+    pattern: "\\.count$"
+    xFilesFactor: "0"
+    aggregationMethod: "sum"
+  default_average:
+    pattern: ".*"
+    xFilesFactor: "0.5"
+    aggregationMethod: "average"
+
+GRAPHITE_API_CONF:
+  search_index: "{{ graphite_root }}/api/index"
+  whisper:
+    directories:
+      - "{{ graphite_storage_root }}"

playbooks/roles/graphite/meta/main.yml

+---
+dependencies:
+  - role: common
+    tags:
+      - always
+  - config-encoders

playbooks/roles/graphite/tasks/carbon.yml

+---
+- name: checkout carbon
+  git:
+    repo: "{{ GRAPHITE_CARBON_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/carbon"
+    version: "{{ GRAPHITE_CARBON_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install carbon dependencies
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    requirements: "{{ graphite_root }}/src/carbon/requirements.txt"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install carbon
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/carbon"
+    creates: "{{ graphite_root }}/bin/carbon-cache.py"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: configure carbon
+  template:
+    src: "carbon/conf/{{ item }}.conf.j2"
+    dest: "{{ graphite_root }}/conf/{{ item }}.conf"
+  with_items:
+    - carbon
+    - storage-schemas
+    - storage-aggregation
+  tags:
+    - install
+    - install:app-configuration
+
+- name: configure carbon service definition
+  template:
+    src: carbon/systemd/carbon-cache.service.j2
+    dest: /etc/systemd/system/carbon-cache.service
+  tags:
+    - install
+    - install:configuration
+
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+  tags:
+    - install
+    - install:configuration
+
+- name: enable carbon
+  service:
+    name: carbon-cache
+    enabled: yes
+    state: started
+  tags:
+    - install
+    - install:configuration
+
+- name: stop carbon
+  service:
+    name: carbon-cache
+    state: stopped
+  tags:
+    - manage:stop
+
+- name: restart carbon
+  service:
+    name: carbon-cache
+    state: restarted
+  tags:
+    - manage:start
+    - manage:restart

playbooks/roles/graphite/tasks/graphite-api.yml

+---
+- name: checkout graphite-api
+  git:
+    repo: "{{ GRAPHITE_API_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/api"
+    version: "{{ GRAPHITE_API_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install graphite-api dependencies
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    requirements: "{{ graphite_root }}/src/api/requirements.txt"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install graphite-api
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/api"
+  tags:
+    - install
+    - install:app-requirements
+
+- name: install gunicorn
+  pip:
+    virtualenv: "{{ graphite_root }}"
+    name: gunicorn
+  tags:
+    - install
+    - install:app-requirements
+
+- name: configure graphite-api
+  template:
+    src: graphite-api/conf/graphite-api.yml.j2
+    dest: "{{ graphite_root }}/conf/graphite-api.yml"
+    owner: "{{ graphite_user }}"
+    group: "{{ graphite_group }}"
+  tags:
+    - install
+    - install:app-configuration
+
+- name: configure graphite-api service definitions
+  template:
+    src: "graphite-api/systemd/{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+  with_items:
+    - graphite-api.socket
+    - graphite-api.service
+  tags:
+    - install
+    - install:configuration
+
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+  tags:
+    - install
+    - install:configuration
+
+- name: enable graphite-api
+  service:
+    name: "{{ item }}"
+    enabled: yes
+  with_items:
+    - graphite-api.socket
+    - graphite-api.service
+
+- name: stop graphite-api
+  service:
+    name: graphite-api
+    state: stopped
+  tags:
+    - manage:stop
+
+- name: restart graphite-api
+  service:
+    name: graphite-api
+    state: restarted
+  tags:
+    - manage:start
+    - manage:restart

playbooks/roles/graphite/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Tasks for role graphite
+#
+# Overview:
+#
+# This task is to install "Graphite", or more specifically,
+# its constituent parts: carbon and graphite-api.
+#
+# For our needs at edX, we do not need the full-fledged
+# graphite-web, instead opting to use the light graphite-api,
+# which can be pointed to by other tools such as Grafana.
+#
+# Dependencies:
+# - common
+# - config-encoders
+#
+
+- name: create Graphite user group
+  group:
+    name: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create service account for Graphite
+  user:
+    name: "{{ graphite_user }}"
+    system: yes
+    home: "{{ graphite_root }}"
+    shell: /bin/false
+    group: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create service account for Carbon
+  user:
+    name: "{{ carbon_user }}"
+    system: yes
+    home: "{{ graphite_root }}"
+    shell: /bin/false
+    group: "{{ graphite_group }}"
+    state: present
+  tags:
+    - install
+    - install:base
+
+- name: create common Graphite directories
+  file:
+    path: "{{ item.value.path }}"
+    state: directory
+    owner: "{{ item.value.owner }}"
+    group: "{{ item.value.group }}"
+    mode: "{{ item.value.mode }}"
+  with_dict: "{{ graphite_common_dirs }}"
+  tags:
+    - install
+    - install:base
+
+- name: install Graphite dependencies
+  apt:
+    pkg: "{{ item }}"
+    state: present
+  with_items: "{{ graphite_common_deb_deps }}"
+  tags:
+    - install
+    - install:system-requirements
+
+- name: initialize virtualenv for Graphite
+  command: "virtualenv {{ graphite_root }}"
+  args:
+    creates: "{{ graphite_root }}/bin/activate"
+  tags:
+    - install
+    - install:base
+
+- include: whisper.yml
+- include: carbon.yml
+- include: graphite-api.yml

playbooks/roles/graphite/tasks/whisper.yml

+---
+- name: checkout Whisper
+  git:
+    repo: "{{ GRAPHITE_WHISPER_GIT_URL }}"
+    dest: "{{ graphite_root }}/src/whisper"
+    version: "{{ GRAPHITE_WHISPER_VERSION }}"
+  tags:
+    - install
+    - install:code
+
+- name: install Whisper
+  command: "{{ graphite_root }}/bin/python setup.py install"
+  args:
+    chdir: "{{ graphite_root }}/src/whisper"
+    creates: "{{ graphite_root }}/bin/whisper-create.py"
+  tags:
+    - install
+    - install:app-requirements

playbooks/roles/graphite/templates/carbon/conf/carbon.conf.j2

+{{ CARBON_CONF | encode_ini }}

playbooks/roles/graphite/templates/carbon/conf/storage-aggregation.conf.j2

+# Aggregation methods for whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds
+#
+#  [name]
+#  pattern = <regex>
+#  xFilesFactor = <float between 0 and 1>
+#  aggregationMethod = <average|sum|last|max|min>
+#
+#  name: Arbitrary unique name for the rule
+#  pattern: Regex pattern to match against the metric name
+#  xFilesFactor: Ratio of valid data points required for aggregation to the next retention to occur
+#  aggregationMethod: function to apply to data points for aggregation
+
+{{ CARBON_STORAGE_AGGREGATIONS | encode_ini }}

playbooks/roles/graphite/templates/carbon/conf/storage-schemas.conf.j2

+# Schema definitions for Whisper files. Entries are scanned in order,
+# and first match wins. This file is scanned for changes every 60 seconds.
+
+{{ CARBON_STORAGE_SCHEMAS | encode_ini }}

playbooks/roles/graphite/templates/carbon/systemd/carbon-cache.service.j2

+[Unit]
+Description=carbon-cache
+After=network.target
+
+[Service]
+Type=forking
+StandardOutput=syslog
+StandardError=syslog
+User={{ carbon_user }}
+Group={{ graphite_group }}
+ExecStart={{ graphite_root }}/bin/carbon-cache.py --config={{ graphite_root }}/conf/carbon.conf --pidfile={{ graphite_root }}/run/carbon-cache.pid --logdir=/var/log/carbon/ start
+ExecReload=/bin/kill -USR1 $MAINPID
+ExecStop={{ graphite_root }}/bin/carbon-cache.py --pidfile={{ graphite_root }}/run/carbon-cache.pid stop
+PIDFile={{ graphite_root }}/run/carbon-cache.pid
+PrivateTmp=true
+LimitNOFILE=128000
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/graphite/templates/graphite-api/conf/graphite-api.yml.j2

+{{ GRAPHITE_API_CONF | encode_yaml }}

playbooks/roles/graphite/templates/graphite-api/systemd/graphite-api.service.j2

+[Unit]
+Description=graphite-api service
+After=network.target
+Requires=graphite-api.socket
+
+[Service]
+StandardOutput=syslog
+StandardError=syslog
+User={{ graphite_user }}
+Group={{ graphite_group }}
+WorkingDirectory={{ graphite_root }}
+Environment=GRAPHITE_API_CONFIG={{ graphite_root }}/conf/graphite-api.yml
+ExecStart={{ graphite_root }}/bin/gunicorn -w {{ GRAPHITE_GUNICORN_WORKERS }} graphite_api.app:app
+Restart=on-failure
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+LimitNOFILE=128000
+
+[Install]
+WantedBy=multi-user.target

playbooks/roles/graphite/templates/graphite-api/systemd/graphite-api.socket.j2

+[Unit]
+Description=graphite-api socket
+
+[Socket]
+SocketUser={{ graphite_user }}
+SocketGroup={{ graphite_group }}
+ListenStream={{ graphite_root }}/run/graphite-api.sock
+
+[Install]
+WantedBy=sockets.target

playbooks/roles/hadoop_common/defaults/main.yml

@@ -77,6 +77,8 @@ YARN_SITE_DEFAULT_CONFIG:
   yarn.log-aggregation-enable: "true"
   # 24 hour log retention
   yarn.log-aggregation.retain-seconds: 86400
+  # Checking virtual memory usage causes too many spurious failures.
+  yarn.nodemanager.vmem-check-enabled: false
 
 HADOOP_CORE_SITE_DEFAULT_CONFIG:
   fs.default.name: "hdfs://localhost:9000"

playbooks/roles/insights/tasks/main.yml

@@ -106,15 +106,6 @@
     - assets
     - assets:gather
 
-- name: compile translations
-  shell: ". {{ insights_venv_dir }}/bin/activate && i18n_tool generate -v"
-  args:
-    chdir: "{{ insights_code_dir }}/analytics_dashboard"
-  become_user: "{{ insights_user }}"
-  tags:
-    - assets
-    - assets:gather
-
 - name: write out the supervisior wrapper
   template:
     src: "edx/app/insights/insights.sh.j2"

playbooks/roles/jenkins_master/tasks/main.yml

@@ -146,6 +146,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 # We first download the plugins to a temp directory and include
 # the version in the file name.  That way, if we increment
@@ -159,6 +160,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Install Jenkins plugins
   command: "cp {{ item.dest }} {{ jenkins_home }}/plugins/{{ item.item.name }}.hpi"
@@ -167,6 +169,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Set Jenkins plugin permissions
   file:
@@ -179,6 +182,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 # We had to fork some plugins to workaround
 # certain issues.  If these changes get merged
@@ -195,6 +199,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Compile custom plugins
   command: "mvn -Dmaven.test.skip=true install"
@@ -205,6 +210,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Install custom plugins
   command: mv /tmp/{{ item.item.repo_name }}/target/{{ item.item.package }}
@@ -214,6 +220,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Set custom plugin permissions
   file:
@@ -226,6 +233,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 # Plugins that are bundled with Jenkins are "pinned".
 # Jenkins will overwrite updated plugins with its built-in version
@@ -239,6 +247,7 @@
   tags:
     - install
     - install:base
+    - install:plugins
 
 - name: Setup nginix vhost
   template:

playbooks/roles/launch_ec2/tasks/main.yml

@@ -79,7 +79,7 @@
     value: "{{ item[0].public_dns_name }}"
   with_nested:
     - "{{ ec2.instances }}"
-    - ['studio', 'ecommerce', 'preview', 'programs', 'discovery', 'credentials']
+    - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials']
 
 - name: Add new instance to host group
   local_action:

playbooks/roles/local_dev/defaults/main.yml

@@ -56,13 +56,6 @@ localdev_accounts:
       }
 
     - {
-        user: "{{ programs_user|default('None') }}",
-        home: "{{ programs_home|default('None') }}",
-        env: "programs_env",
-        repo: "programs"
-      }
-
-    - {
         user: "{{ credentials_user|default('None') }}",
         home: "{{ credentials_home|default('None') }}",
         env: "credentials_env",

playbooks/roles/locust/defaults/main.yml

@@ -17,21 +17,13 @@
 locust_service_name: "locust"
 locust_home: "{{ COMMON_APP_DIR }}/{{ locust_service_name }}"
 locust_user: "locust"
-locust_code_dir: "{{ locust_home }}/load-tests"
-locust_requirements_base: "{{ locust_code_dir }}"
-locust_requirements:
-  - "requirements.txt"
-locust_run_dir: "{{ locust_code_dir }}/{{ LOCUST_LOADTEST_DIR }}"
+locust_code_dir: "{{ locust_home }}/edx-load-tests"
+locust_requirements_base: "{{ locust_code_dir }}/requirements"
 
 LOCUST_GIT_IDENTITY: !!null
 
-LOCUST_LOADTEST_REPO: 'load-tests'
+LOCUST_LOADTEST_REPO: 'edx-load-tests'
 LOCUST_LOADTEST_REPO_VERSION: 'master'
-LOCUST_LOADTEST_DIR: 'locust/lms'
-
-LOCUST_MASTER: !!null
-
-LOCUST_TARGET_HOST: 'http://localhost/'
 
 LOCUST_SERVICE_CONFIG: ''
 

playbooks/roles/locust/meta/main.yml

@@ -12,7 +12,6 @@
 # 
 dependencies:
   - common
-  - supervisor
   - role: edx_service
     edx_service_name: "{{ locust_service_name }}"
     edx_service_config: "{{ LOCUST_SERVICE_CONFIG }}"

playbooks/roles/locust/tasks/main.yml

@@ -12,9 +12,10 @@
 # Tasks for role locust
 #
 # Overview:
-#  - Expects a load-tests repo that declares locust.io as a requirement
-#  - Installs load-tests repo using standard edx service roles
-#  - configures locust with defaults in supervisor
+#  - Expects an edx-load-tests repo that declares locust.io as a requirement
+#  - Installs edx-load-tests repo using standard edx service roles
+#  - Makes the locust user suitable for running an interactive shell
+#  - Adds a MOTD message for self-help resources related to loadtesting
 #
 # Dependencies:
 #  - edx-service role
@@ -22,61 +23,62 @@
 #
 # Example play:
 #
-#   # To run: ansible-playbook locust.yml -i "locustmaster.sandbox.edx.org," -e "LOCUST_LOADTEST_DIR='locust/lms'" -e "LOCUST_TARGET_HOST='https://courses-loadtest.edx.org'" -e "@/Users/derf/workspace/sandbox-secure/ansible/vars/developer-sandbox.yml"
+#   # To run: ansible-playbook locust.yml -i "locustdriver.sandbox.edx.org," -e "@/Users/derf/workspace/sandbox-secure/ansible/vars/developer-sandbox.yml"
 #   - name: Deploy Locust
 #     hosts: all
-#     sudo: True
+#     become: True
 #     gather_facts: True
 #     roles:
 #       - locust
 
-- name: Install system packages
-  apt:
-    pkg: "{{ item }}"
-    state: "present"
-    update_cache: true
-    cache_valid_time: 3600
-  with_items: "{{ locust_debian_pkgs }}"
-
-- name: Install application requirements
+- name: Install base requirements used by all load tests
   pip:
-    requirements: "{{ locust_requirements_base }}/{{ item }}"
+    requirements: "{{ locust_requirements_base }}/base.txt"
     virtualenv: "{{ locust_home }}/venvs/{{ locust_service_name }}"
     state: present
   become_user: "{{ locust_user }}"
-  with_items: "{{ locust_requirements }}"
-
-- name: Write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/locust.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ locust_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: "0644"
-
-- name: Enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ locust_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ locust_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-
-- name: Update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-
-- name: Restart the applicaton
-  supervisorctl:
-    name: "{{ locust_service_name }}"
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
 
+# Specifically, we are concerned about allowing as many open connections as
+# possible, to simulate more locust clients.
 - name: Increase file descriptor limit of the system (Session Logout and Login would be required)
   lineinfile:
     dest: "{{ ulimit_conf_file }}"
     line: "{{ item.domain }}  {{ item.type }}  {{ item.item }}  {{ item.value }}"
   with_items: "{{ ulimit_config }}"
+
+- name: Configure locust user with an interactive shell
+  user:
+    name: "{{ locust_user }}"
+    shell: /bin/bash
+
+# This is a hack because we're not using .bash_aliases for its stated purpose.
+# As of this writing, the script will activate a preinstalled virtualenv and
+# change to the edx-load-tests directory.
+- name: Hack .bash_aliases to make it more convenient to start loadtests
+  template:
+    src: locust_home/bash_aliases.j2
+    dest: "{{ locust_home }}/.bash_aliases"
+    owner: "{{ locust_user }}"
+    group: root
+    mode: '644'
+
+- name: Setup a loadtest-specific MOTD
+  template:
+    src: etc/motd.tail.locust.j2
+    dest: /etc/motd.tail.locust
+    owner: root
+    group: root
+    mode: '644'
+  # the directory /etc/update-motd.d is a xenial-ism only
+  when: ansible_distribution_release == 'xenial'
+
+- name: Add motd.tail.locust to update-motd.d
+  copy:
+    dest: "/etc/update-motd.d/76-motd-tail-locust"
+    content: "#!/bin/sh\necho\ncat /etc/motd.tail.locust\n"
+    force: true
+    owner: root
+    group: root
+    mode: "0755"
+  # the directory /etc/update-motd.d is a xenial-ism only
+  when: ansible_distribution_release == 'xenial'

playbooks/roles/locust/templates/edx/app/supervisor/conf.d.available/locust.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ locust_service_name }}]
-
-{% set locust_venv_bin = locust_home + "/venvs/" + locust_service_name + "/bin" %}
-{% set executable = locust_venv_bin + '/locust' %}
-{% if LOCUST_MASTER %}
-{% set locust_extra_args = '--slave --master-host ' + LOCUST_MASTER %}
-{% else %}
-{% set locust_extra_args = '' %}
-{% endif %}
-
-command={{ executable }} --host {{ LOCUST_TARGET_HOST }} {{ locust_extra_args }}
-user={{ common_web_user }}
-directory={{ locust_run_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/locust/templates/etc/motd.tail.locust.j2

+*******************************************************************
+*                 Message regarding loadtests                     *
+*                                                                 *
+* Start your loadtesting journey by switching to the locust user: *
+*                                                                 *
+*   sudo su locust                                                *
+*                                                                 *
+* Please consult the loadtest environment queue before running    *
+* load tests against our shared loadtest environment:             *
+* https://openedx.atlassian.net/wiki/x/B4M3AQ                     *
+*                                                                 *
+* For troubleshooting or other help, see our documentation:       *
+* https://openedx.atlassian.net/wiki/x/-QEsAQ                     *
+*******************************************************************

playbooks/roles/locust/templates/locust_home/bash_aliases.j2

+cd {{ locust_code_dir }}
+source {{ locust_home }}/venvs/{{ locust_service_name }}/bin/activate

playbooks/roles/nginx/defaults/main.yml

@@ -82,7 +82,6 @@ NGINX_EDXAPP_ERROR_PAGES:
 
 CMS_HOSTNAME: '~^((stage|prod)-)?studio.*'
 ECOMMERCE_HOSTNAME: '~^((stage|prod)-)?ecommerce.*'
-PROGRAMS_HOSTNAME: '~^((stage|prod)-)?programs.*'
 CREDENTIALS_HOSTNAME: '~^((stage|prod)-)?credentials.*'
 
 nginx_template_dir: "edx/app/nginx/sites-available"
@@ -107,8 +106,6 @@ nginx_edx_notes_api_gunicorn_hosts:
   - 127.0.0.1
 nginx_ecommerce_gunicorn_hosts:
   - 127.0.0.1
-nginx_programs_gunicorn_hosts:
-  - 127.0.0.1
 nginx_credentails_gunicorn_hosts:
   - 127.0.0.1
 
@@ -135,7 +132,6 @@ EDXAPP_CMS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 EDXAPP_LMS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 EDXAPP_LMS_PREVIEW_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 KIBANA_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
-PROGRAMS_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 XQUEUE_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 XSERVER_ENABLE_BASIC_AUTH: "{{ COMMON_ENABLE_BASIC_AUTH }}"
 
@@ -147,7 +143,6 @@ NGINX_CREATE_HTPASSWD_FILE: >
     EDXAPP_LMS_ENABLE_BASIC_AUTH|bool or
     EDXAPP_LMS_PREVIEW_ENABLE_BASIC_AUTH|bool or
     KIBANA_ENABLE_BASIC_AUTH|bool or
-    PROGRAMS_ENABLE_BASIC_AUTH|bool or
     XQUEUE_ENABLE_BASIC_AUTH|bool or
     XSERVER_ENABLE_BASIC_AUTH|bool
   }}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/graphite.j2

+#
+# {{ ansible_managed }}
+#
+{# This prevents the injected comment from eating the server
+   directive.  There's probably a better way of doing this,
+   but I don't know it currently.
+#}
+{% raw %}
+
+{% endraw %}
+
+{%- if "graphite" in nginx_default_sites -%}
+  {%- set default_site = "default_server" -%}
+{%- else -%}
+  {%- set default_site = "" -%}
+{%- endif -%}
+
+upstream graphite_app_server {
+  server unix:{{ graphite_root }}/run/graphite-api.sock fail_timeout=0;
+}
+
+server {
+
+  server_name graphite.*;
+  listen {{ GRAPHITE_NGINX_PORT }} {{ default_site }};
+  client_max_body_size 1M;
+  keepalive_timeout 5;
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  location @proxy_to_app {
+    {% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    {% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    {% endif %}
+
+    # newrelic-specific header records the time when nginx handles a request.
+    proxy_set_header X-Queue-Start "t=${msec}";
+
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://graphite_app_server;
+  }
+}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/programs.j2

-#
-# {{ ansible_managed }}
-#
-
-
-{% if "programs" in nginx_default_sites %}
-  {% set default_site = "default_server" %}
-{% else %}
-  {% set default_site = "" %}
-{% endif %}
-
-upstream programs_app_server {
-{% for host in nginx_programs_gunicorn_hosts %}
-    server {{ host }}:{{ programs_gunicorn_port }} fail_timeout=0;
-{% endfor %}
-}
-
-server {
-  server_name {{ PROGRAMS_HOSTNAME }};
-  
-  listen {{ PROGRAMS_NGINX_PORT }} {{ default_site }};
-
-  {% include "common-settings.j2" %}
-  
-  {% if NGINX_ENABLE_SSL %}
-
-  listen {{ PROGRAMS_SSL_NGINX_PORT }} ssl;
-
-  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
-  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # Request that the browser use SSL for all connections.
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-
-  {% endif %}
-
-  # Nginx does not support nested condition or or conditions so
-  # there is an unfortunate mix of conditonals here.
-  {% if NGINX_REDIRECT_TO_HTTPS %}
-     {% if NGINX_HTTPS_REDIRECT_STRATEGY == "scheme" %}
-  # Redirect http to https over single instance
-  if ($scheme != "https") 
-  { 
-   set $do_redirect_to_https "true";
-  }
-
-     {% elif NGINX_HTTPS_REDIRECT_STRATEGY == "forward_for_proto" %}
-
-  # Forward to HTTPS if we're an HTTP request... and the server is behind ELB 
-  if ($http_x_forwarded_proto = "http") 
-  {
-   set $do_redirect_to_https "true";
-  }
-     {% endif %}
-
-  # Execute the actual redirect
-  if ($do_redirect_to_https = "true")
-  {
-  return 301 https://$host$request_uri;
-  }
-  {% endif %}
-
-  location ~ ^/static/(?P<file>.*) {
-    root {{ PROGRAMS_DATA_DIR  }};
-    try_files /staticfiles/$file =404;
-
-    # Request that the browser use SSL for these connections. Repeated here
-    # because add_header directives are only inherited from the previous level
-    # if there are no add_header directives defined on the current level.
-    # See: http://nginx.org/en/docs/http/ngx_http_headers_module.html#add_header.
-    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-    
-    # Instruct the browser to cache static assets for one hour.
-    add_header Cache-Control "public; max-age=3600";
-  }
-
-  location ~ ^/media/(?P<file>.*) {
-    root {{ PROGRAMS_DATA_DIR }};
-    try_files /media/$file =404;
-    # django / app always assigns new filenames so these can be cached forever.
-    add_header Cache-Control "public; max-age=31536000";
-  }
-
-  location / {
-    {% if PROGRAMS_ENABLE_BASIC_AUTH|bool %}
-      {% include "basic-auth.j2" %}
-    {% endif %}
-    try_files $uri @proxy_to_app;
-  }
-
-  # The API should be secured with OAuth 2.0 or or JWT.
-  location /api {
-    try_files $uri @proxy_to_app;
-  }
-
-  {% include "robots.j2" %}
-
-location @proxy_to_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
-
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://programs_app_server;
-  }
-
-}

playbooks/roles/oauth_client_setup/defaults/main.yml

@@ -32,13 +32,6 @@ oauth_client_setup_oauth2_clients:
         logout_uri: "{{ INSIGHTS_LOGOUT_URL | default('None') }}"
       }
     - {
-        name: "{{ programs_service_name | default('None') }}",
-        url_root: "{{ PROGRAMS_URL_ROOT | default('None') }}",
-        id: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
-        secret: "{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
-        logout_uri: "{{ PROGRAMS_LOGOUT_URL | default('None') }}"
-      }
-    - {
         name: "{{ credentials_service_name | default('None') }}",
         url_root: "{{ CREDENTIALS_URL_ROOT | default('None') }}",
         id: "{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",

playbooks/roles/programs/defaults/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-##
-# Defaults for role programs
-# 
-PROGRAMS_GIT_IDENTITY: !!null
-
-# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
-# and a key being provided via NEWRELIC_LICENSE_KEY
-PROGRAMS_NEWRELIC_APPNAME: "{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ programs_service_name }}"
-PROGRAMS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-PROGRAMS_NGINX_PORT: 18140
-PROGRAMS_SSL_NGINX_PORT: 48140
-
-PROGRAMS_DEFAULT_DB_NAME: 'programs'
-
-PROGRAMS_DATABASE_USER: 'programs001'
-PROGRAMS_DATABASE_PASSWORD: 'password'
-PROGRAMS_DATABASE_HOST: 'localhost'
-PROGRAMS_DATABASE_PORT: 3306
-
-PROGRAMS_DATABASES:
-  # rw user
-  default:
-    ENGINE: 'django.db.backends.mysql'
-    NAME: '{{ PROGRAMS_DEFAULT_DB_NAME }}'
-    USER: '{{ PROGRAMS_DATABASE_USER }}'
-    PASSWORD: '{{ PROGRAMS_DATABASE_PASSWORD }}'
-    HOST: '{{ PROGRAMS_DATABASE_HOST }}'
-    PORT: '{{ PROGRAMS_DATABASE_PORT }}'
-    ATOMIC_REQUESTS: true
-    CONN_MAX_AGE: 60
-
-PROGRAMS_VERSION: "master"
-PROGRAMS_DJANGO_SETTINGS_MODULE: "programs.settings.production"
-PROGRAMS_URL_ROOT: 'http://localhost:8004'
-PROGRAMS_LOGOUT_URL: '{{ PROGRAMS_URL_ROOT }}/accounts/logout/'
-PROGRAMS_LMS_URL_ROOT: 'http://127.0.0.1:8000'
-PROGRAMS_ORGANIZATIONS_API_URL_ROOT: '{{ PROGRAMS_LMS_URL_ROOT }}/api/organizations/v0/'
-
-PROGRAMS_SECRET_KEY: 'Your secret key here'
-PROGRAMS_TIME_ZONE: 'UTC'
-PROGRAMS_LANGUAGE_CODE: 'en-us'
-
-# Used to automatically configure OAuth2 Client
-PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY : 'programs-key'
-PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET : 'programs-secret'
-PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
-PROGRAMS_OAUTH_URL_ROOT: '{{ PROGRAMS_LMS_URL_ROOT }}/oauth2'
-PROGRAMS_SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ PROGRAMS_LMS_URL_ROOT }}/logout'
-
-PROGRAMS_JWT_AUDIENCE: '{{ EDXAPP_JWT_AUDIENCE | default("SET-ME-PLEASE") }}'
-PROGRAMS_JWT_ISSUER: '{{ PROGRAMS_OAUTH_URL_ROOT }}'
-PROGRAMS_JWT_SECRET_KEY: '{{ EDXAPP_JWT_SECRET_KEY | default("lms-secret") }}'
-
-PROGRAMS_JWT_AUTH: 
-  JWT_ISSUERS:
-    - AUDIENCE: '{{ PROGRAMS_JWT_AUDIENCE }}'
-      ISSUER: '{{ PROGRAMS_JWT_ISSUER }}'
-      SECRET_KEY: '{{ PROGRAMS_JWT_SECRET_KEY }}'
-    - AUDIENCE: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-      ISSUER: '{{ PROGRAMS_OAUTH_URL_ROOT }}'
-      SECRET_KEY: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-
-PROGRAMS_PLATFORM_NAME: 'Your Platform Name Here'
-
-# CORS
-# See: https://github.com/ottoyiu/django-cors-headers/.
-# the whitelist should always contain the public hostname for Studio in the given environment.
-PROGRAMS_CORS_ORIGIN_WHITELIST:
-  - '127.0.0.1:8001'
-
-PROGRAMS_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ programs_service_name }}'
-PROGRAMS_MEDIA_ROOT: '{{ PROGRAMS_DATA_DIR }}/media'
-PROGRAMS_MEDIA_URL: '/media/'
-
-# Example settings to use Amazon S3 as a storage backend for user-uploaded files
-# https://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html#amazon-s3
-#
-# This is only for user-uploaded files and does not cover static assets that ship
-# with the code.
-#
-# Note, AWS_S3_CUSTOM_DOMAIN is required, otherwise boto will generate non-working
-# querystring URLs for assets (see https://github.com/boto/boto/issues/1477)
-#
-# Note, set AWS_S3_CUSTOM_DOMAIN to the cloudfront domain instead, when that is in use.
-#
-# PROGRAMS_BUCKET: mybucket
-# programs_s3_domain: s3.amazonaws.com
-# PROGRAMS_MEDIA_ROOT: 'media'   # NOTE use '$source_ip/media' for an edx sandbox
-#
-# PROGRAMS_MEDIA_STORAGE_BACKEND:
-#   DEFAULT_FILE_STORAGE: 'programs.apps.core.s3utils.MediaS3BotoStorage'
-#   MEDIA_ROOT: '{{ PROGRAMS_MEDIA_ROOT }}'
-#   MEDIA_URL: 'https://{{ PROGRAMS_BUCKET }}.{{ programs_s3_domain }}/{{ PROGRAMS_MEDIA_ROOT }}/'
-#   AWS_STORAGE_BUCKET_NAME: '{{ PROGRAMS_BUCKET }}'
-#   AWS_S3_CUSTOM_DOMAIN: '{{ PROGRAMS_BUCKET }}.{{ programs_s3_domain }}'
-#   AWS_QUERYSTRING_AUTH: false
-#   AWS_QUERYSTRING_EXPIRE: false
-#   AWS_DEFAULT_ACL: ''
-#   AWS_HEADERS:
-#     Cache-Control: max-age=31536000
-#
-#
-PROGRAMS_MEDIA_STORAGE_BACKEND:
-  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
-  MEDIA_ROOT: '{{ PROGRAMS_MEDIA_ROOT }}'
-  MEDIA_URL: '{{ PROGRAMS_MEDIA_URL }}'
-
-PROGRAMS_SERVICE_CONFIG:
-  SECRET_KEY: '{{ PROGRAMS_SECRET_KEY }}'
-  TIME_ZONE: '{{ PROGRAMS_TIME_ZONE }}'
-  LANGUAGE_CODE: '{{ PROGRAMS_LANGUAGE_CODE }}'
-
-  SOCIAL_AUTH_EDX_OIDC_KEY: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
-  SOCIAL_AUTH_EDX_OIDC_SECRET: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_ID_TOKEN_DECRYPTION_KEY: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
-  SOCIAL_AUTH_EDX_OIDC_URL_ROOT: '{{ PROGRAMS_OAUTH_URL_ROOT }}'
-  SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL: '{{ PROGRAMS_SOCIAL_AUTH_EDX_OIDC_LOGOUT_URL }}'
-  SOCIAL_AUTH_REDIRECT_IS_HTTPS: '{{ PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
-
-  STATIC_ROOT: '{{ PROGRAMS_DATA_DIR }}/staticfiles'
-  # db config
-  DATABASE_OPTIONS:
-    connect_timeout: 10
-  DATABASES: '{{ PROGRAMS_DATABASES }}'
-
-  PLATFORM_NAME: '{{ PROGRAMS_PLATFORM_NAME }}'
-
-  CORS_ORIGIN_WHITELIST: '{{ PROGRAMS_CORS_ORIGIN_WHITELIST }}'
-
-  PUBLIC_URL_ROOT: '{{ PROGRAMS_URL_ROOT }}'
-  ORGANIZATIONS_API_URL_ROOT: '{{ PROGRAMS_ORGANIZATIONS_API_URL_ROOT }}'
-  
-  # This hash contains top-level Django settings and must be unpacked
-  # when loading/overriding settings.
-  MEDIA_STORAGE_BACKEND: '{{ PROGRAMS_MEDIA_STORAGE_BACKEND }}'
-  
-  JWT_AUTH: '{{ PROGRAMS_JWT_AUTH }}'
-
-PROGRAMS_REPOS:
-  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
-    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
-    PATH: "{{ COMMON_GIT_PATH }}"
-    REPO: programs.git
-    VERSION: "{{ PROGRAMS_VERSION }}"
-    DESTINATION: "{{ programs_code_dir }}"
-    SSH_KEY: "{{ PROGRAMS_GIT_IDENTITY }}"
-
-
-PROGRAMS_GUNICORN_WORKERS: "2"
-PROGRAMS_GUNICORN_EXTRA: ""
-PROGRAMS_GUNICORN_EXTRA_CONF: ""
-PROGRAMS_GUNICORN_WORKER_CLASS: "gevent"
-
-#
-# vars are namespace with the module name.
-#
-programs_role_name: programs
-programs_venv_dir: "{{ programs_home }}/venvs/{{ programs_service_name }}"
-
-programs_service_name: "programs"
-programs_user: "{{ programs_service_name }}"
-programs_home: "{{ COMMON_APP_DIR }}/{{ programs_service_name }}"
-programs_code_dir: "{{ programs_home }}/{{ programs_service_name }}"
-
-programs_environment:
-  DJANGO_SETTINGS_MODULE: "{{ PROGRAMS_DJANGO_SETTINGS_MODULE }}"
-  PROGRAMS_CFG: "{{ COMMON_CFG_DIR }}/{{ programs_service_name }}.yml"
-  PATH: "{{ programs_venv_dir }}/bin:{{ ansible_env.PATH }}"
-
-programs_gunicorn_host: "127.0.0.1"
-programs_gunicorn_port: 8140
-programs_gunicorn_timeout: 300
-
-programs_log_dir: "{{ COMMON_LOG_DIR }}/{{ programs_service_name }}"
-
-programs_requirements_base: "{{ programs_code_dir }}/requirements"
-programs_requirements:
-  - production.txt
-  - optional.txt
-
-#
-# OS packages
-#
-
-programs_debian_pkgs:
-  - libjpeg-dev
-  - libmysqlclient-dev
-  - libssl-dev
-  - gettext
-# Temporarily, while we get 16.04 rolled out everywhere
-# Then this goes back in common_vars
-  - python2.7
-
-programs_redhat_pkgs: []

playbooks/roles/programs/meta/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-##
-# Role includes for role programs
-# 
-# Example:
-#
-# dependencies:
-#   - {
-#   role: my_role 
-#   my_role_var0: "foo"
-#   my_role_var1: "bar"
-# }
-dependencies:
-  - common
-  - supervisor
-  - role: edx_service
-    edx_service_name: "{{ programs_service_name }}"
-    edx_service_config: "{{ PROGRAMS_SERVICE_CONFIG }}"
-    edx_service_repos: "{{ PROGRAMS_REPOS }}"
-    edx_service_user: "{{ programs_user }}"
-    edx_service_home: "{{ programs_home }}"
-    edx_service_packages:
-      debian: "{{ programs_debian_pkgs }}"
-      redhat: "{{ programs_redhat_pkgs }}"
-

playbooks/roles/programs/tasks/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role programs
-# 
-# Overview:
-# 
-#
-# Dependencies:
-#
-# 
-# Example play:
-#
-#
-
-- name: add gunicorn configuration file
-  template:
-    src: edx/app/programs/programs_gunicorn.py.j2
-    dest: "{{ programs_home }}/programs_gunicorn.py"
-  become_user: "{{ programs_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: install application requirements
-  pip:
-    requirements: "{{ programs_requirements_base }}/{{ item }}"
-    # Ansible will initialize this virtualenv if it's missing.
-    virtualenv: "{{ programs_venv_dir }}"
-    state: present
-  become_user: "{{ programs_user }}"
-  with_items: "{{ programs_requirements }}"
-  tags:
-    - install
-    - install:app-requirements
-
-- name: migrate
-  shell: >
-    chdir={{ programs_code_dir }}
-    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
-    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ programs_venv_dir }}/bin/python ./manage.py migrate --noinput
-  become_user: "{{ programs_user }}"
-  environment: "{{ programs_environment }}"
-  when: migrate_db is defined and migrate_db|lower == "yes"
-  tags:
-    - migrate
-    - migrate:db
-
-- name: run collectstatic
-  shell: "{{ programs_venv_dir }}/bin/python manage.py collectstatic --noinput"
-  args:
-    chdir: "{{ programs_code_dir }}"
-  become_user: "{{ programs_user }}"
-  environment: "{{ programs_environment }}"
-  when: not devstack
-  tags:
-    - assets
-    - assets:gather
-
-# NOTE this isn't used or needed when s3 is used for PROGRAMS_MEDIA_STORAGE_BACKEND
-- name: create programs media dir
-  file:
-    path: "{{ item }}"
-    state: directory
-    mode: 0775
-    owner: "{{ programs_user }}"
-    group: "{{ common_web_group }}"
-  with_items:
-    - "{{ PROGRAMS_MEDIA_ROOT }}"
-  tags:
-    - install
-    - install:base
-
-- name: write out the supervisor wrapper
-  template:
-    src: "edx/app/programs/programs.sh.j2"
-    dest: "{{ programs_home }}/{{ programs_service_name }}.sh"
-    mode: 0650
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-  tags:
-    - install
-    - install:configuration
-
-- name: write supervisord config
-  template:
-    src: "edx/app/supervisor/conf.d.available/programs.conf.j2"
-    dest: "{{ supervisor_available_dir }}/{{ programs_service_name }}.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ common_web_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: setup the programs env file
-  template:
-    src: "./{{ programs_home }}/{{ programs_service_name }}_env.j2"
-    dest: "{{ programs_home }}/programs_env"
-    owner: "{{ programs_user }}"
-    group: "{{ programs_user }}"
-    mode: 0644
-  tags:
-    - install
-    - install:configuration
-
-- name: enable supervisor script
-  file:
-    src: "{{ supervisor_available_dir }}/{{ programs_service_name }}.conf"
-    dest: "{{ supervisor_cfg_dir }}/{{ programs_service_name }}.conf"
-    state: link
-    force: yes
-  when: not disable_edx_services
-  tags:
-    - install
-    - install:configuration
-
-- name: update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  when: not disable_edx_services
-  tags:
-    - manage
-    - manage:start
-
-- name: create symlinks from the venv bin dir
-  file:
-    src: "{{ programs_venv_dir }}/bin/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.programs"
-    state: link
-  with_items:
-  - python
-  - pip
-  - django-admin.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: create symlinks from the repo dir
-  file:
-    src: "{{ programs_code_dir }}/{{ item }}"
-    dest: "{{ COMMON_BIN_DIR }}/{{ item.split('.')[0] }}.programs"
-    state: link
-  with_items:
-  - manage.py
-  tags:
-    - install
-    - install:app-requirements
-
-- name: restart the application
-  supervisorctl: 
-    state: restarted
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-    name: "{{ programs_service_name }}"
-  when: not disable_edx_services
-  become_user: "{{ supervisor_service_user }}"
-  tags:
-    - manage
-    - manage:start

playbooks/roles/programs/templates/edx/app/programs/programs.sh.j2

-#!/usr/bin/env bash
-
-# {{ ansible_managed }}
-
-{% set programs_venv_bin = programs_home + "/venvs/" + programs_service_name + "/bin" %}
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = programs_venv_bin + '/newrelic-admin run-program ' + programs_venv_bin + '/gunicorn' %}
-{% else %}
-{% set executable = programs_venv_bin + '/gunicorn' %}
-{% endif %}
-
-{% if COMMON_ENABLE_NEWRELIC_APP %}
-export NEW_RELIC_APP_NAME="{{ PROGRAMS_NEWRELIC_APPNAME }}"
-export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
-{% endif -%}
-
-source {{ programs_home }}/programs_env
-{{ executable }} -c {{ programs_home }}/programs_gunicorn.py {{ PROGRAMS_GUNICORN_EXTRA }} programs.wsgi:application

playbooks/roles/programs/templates/edx/app/programs/programs_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in programs_environment.items() -%}
-{%- if value -%}
-export {{ name }}="{{ value }}"
-{% endif %}
-{%- endfor %}

playbooks/roles/programs/templates/edx/app/programs/programs_gunicorn.py.j2

-"""
-gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html
-{{ ansible_managed }}
-"""
-
-timeout = {{ programs_gunicorn_timeout }}
-bind = "{{ programs_gunicorn_host }}:{{ programs_gunicorn_port }}"
-pythonpath = "{{ programs_code_dir }}"
-workers = {{ PROGRAMS_GUNICORN_WORKERS }}
-worker_class = "{{ PROGRAMS_GUNICORN_WORKER_CLASS }}"
-
-{{ PROGRAMS_GUNICORN_EXTRA_CONF }}

playbooks/roles/programs/templates/edx/app/supervisor/conf.d.available/programs.conf.j2

-#
-# {{ ansible_managed }}
-# 
-[program:{{ programs_service_name }}]
-
-command={{ programs_home }}/{{ programs_service_name }}.sh
-user={{ common_web_user }}
-directory={{ programs_code_dir }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true

playbooks/roles/rabbitmq/tasks/main.yml

@@ -51,6 +51,8 @@
     owner: "{{ rabbitmq_user }}"
     group: "{{ rabbitmq_group }}"
     mode: "0755"
+  tags:
+    - "monitoring"
 
 - name: Add RabbitMQ memory usage script
   template:
@@ -59,16 +61,22 @@
     owner: "{{ rabbitmq_user }}"
     group: "{{ rabbitmq_group }}"
     mode: "0775"
+  tags:
+    - "monitoring"
 
 - name: Set up a cron job to run queue script
   cron:
     name: "log-queue-lenghts"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh >/dev/null 2>&1"
+  tags:
+    - "monitoring"
 
 - name: Set up a cron job to run the script
   cron:
     name: "log-rabbitmq-memory-usage"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-memory.sh >/dev/null 2>&1"
+  tags:
+    - "monitoring"
 
 - name: install logrotate configuration
   template:

playbooks/roles/supervisor/defaults/main.yml

@@ -67,10 +67,6 @@ supervisor_spec:
     python: python.analytics_api
     code: "{{ analytics_api_code_dir | default(None) }}"
     env: "{{ analytics_api_home | default(None) }}/analytics_api_env"
-  - service: programs
-    python: python.programs
-    code: "{{ programs_code_dir | default(None) }}"
-    env: "{{ programs_home | default(None) }}/programs_env"
   - service: credentials
     python: python.credentials
     code: "{{ credentials_code_dir | default(None) }}"

playbooks/roles/supervisor/files/pre_supervisor_checks.py

@@ -15,7 +15,6 @@ MIGRATION_COMMANDS = {
         'cms':     "/edx/bin/edxapp-migrate-cms --noinput --list",
         'xqueue':  "SERVICE_VARIANT=xqueue sudo -E -u xqueue {python} {code_dir}/manage.py migrate --noinput --list --settings=xqueue.aws_settings",
         'ecommerce':     ". {env_file}; sudo -E -u ecommerce {python} {code_dir}/manage.py showmigrations",
-        'programs':      ". {env_file}; sudo -E -u programs {python} {code_dir}/manage.py showmigrations",
         'insights':      ". {env_file}; sudo -E -u insights {python} {code_dir}/manage.py showmigrations",
         'analytics_api': ". {env_file}; sudo -E -u analytics_api {python} {code_dir}/manage.py showmigrations",
         'credentials':   ". {env_file}; sudo -E -u credentials {python} {code_dir}/manage.py showmigrations",
@@ -96,15 +95,6 @@ if __name__ == '__main__':
     ecom_migration_args.add_argument("--ecommerce-code-dir",
         help="Location of the ecommerce code.")
 
-    programs_migration_args = parser.add_argument_group("programs_migrations",
-            "Args for running programs migration checks.")
-    programs_migration_args.add_argument("--programs-python",
-        help="Path to python to use for executing migration check.")
-    programs_migration_args.add_argument("--programs-env",
-        help="Location of the programs environment file.")
-    programs_migration_args.add_argument("--programs-code-dir",
-        help="Location of the programs code.")
-
     credentials_migration_args = parser.add_argument_group("credentials_migrations",
             "Args for running credentials migration checks.")
     credentials_migration_args.add_argument("--credentials-python",
@@ -242,7 +232,6 @@ if __name__ == '__main__':
                         "lms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir},
                         "cms": {'python': args.edxapp_python, 'env_file': args.edxapp_env, 'code_dir': args.edxapp_code_dir},
                         "ecommerce": {'python': args.ecommerce_python, 'env_file': args.ecommerce_env, 'code_dir': args.ecommerce_code_dir},
-                        "programs": {'python': args.programs_python, 'env_file': args.programs_env, 'code_dir': args.programs_code_dir},
                         "credentials": {'python': args.credentials_python, 'env_file': args.credentials_env, 'code_dir': args.credentials_code_dir},
                         "discovery": {'python': args.discovery_python, 'env_file': args.discovery_env, 'code_dir': args.discovery_code_dir},
                         "insights": {'python': args.insights_python, 'env_file': args.insights_env, 'code_dir': args.insights_code_dir},

playbooks/roles/supervisor/templates/etc/init/pre_supervisor.conf.j2

@@ -3,12 +3,6 @@ description     "Tasks before supervisord"
 start on runlevel [2345]
 task
 
-{% if programs_code_dir is defined %}
-  {% set programs_command = "--programs-env " + programs_home + "/programs_env --programs-code-dir " + programs_code_dir + " --programs-python " + COMMON_BIN_DIR + "/python.programs" %}
-{% else %}
-  {% set programs_command = "" %}
-{% endif %}
-
 {% if credentials_code_dir is defined %}
   {% set credentials_command = "--credentials-env " + credentials_home + "/credentials_env --credentials-code-dir " + credentials_code_dir + " --credentials-python " + COMMON_BIN_DIR + "/python.credentials" %}
 {% else %}
@@ -22,4 +16,4 @@ task
 {% endif %}
 
 
-exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{ supervisor_available_dir }} --enabled={{ supervisor_cfg_dir }} {% if SUPERVISOR_HIPCHAT_API_KEY is defined %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %} {% if edxapp_code_dir is defined %}--edxapp-python {{ COMMON_BIN_DIR }}/python.edxapp --edxapp-code-dir {{ edxapp_code_dir }} --edxapp-env {{ edxapp_app_dir }}/edxapp_env{% endif %} {% if xqueue_code_dir is defined %}--xqueue-code-dir {{ xqueue_code_dir }} --xqueue-python {{ COMMON_BIN_DIR }}/python.xqueue {% endif %} {% if ecommerce_code_dir is defined %}--ecommerce-env {{ ecommerce_home }}/ecommerce_env --ecommerce-code-dir {{ ecommerce_code_dir }} --ecommerce-python {{ COMMON_BIN_DIR }}/python.ecommerce {% endif %} {% if insights_code_dir is defined %}--insights-env {{ insights_home }}/insights_env --insights-code-dir {{ insights_code_dir }} --insights-python {{ COMMON_BIN_DIR }}/python.insights {% endif %} {% if analytics_api_code_dir is defined %}--analytics-api-env {{ analytics_api_home }}/analytics_api_env --analytics-api-code-dir {{ analytics_api_code_dir }} --analytics-api-python {{ COMMON_BIN_DIR }}/python.analytics_api {% endif %} {{ programs_command }} {{ discovery_command }} {{ credentials_command }}
+exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{ supervisor_available_dir }} --enabled={{ supervisor_cfg_dir }} {% if SUPERVISOR_HIPCHAT_API_KEY is defined %}--hipchat-api-key {{ SUPERVISOR_HIPCHAT_API_KEY }} --hipchat-room {{ SUPERVISOR_HIPCHAT_ROOM }} {% endif %} {% if edxapp_code_dir is defined %}--edxapp-python {{ COMMON_BIN_DIR }}/python.edxapp --edxapp-code-dir {{ edxapp_code_dir }} --edxapp-env {{ edxapp_app_dir }}/edxapp_env{% endif %} {% if xqueue_code_dir is defined %}--xqueue-code-dir {{ xqueue_code_dir }} --xqueue-python {{ COMMON_BIN_DIR }}/python.xqueue {% endif %} {% if ecommerce_code_dir is defined %}--ecommerce-env {{ ecommerce_home }}/ecommerce_env --ecommerce-code-dir {{ ecommerce_code_dir }} --ecommerce-python {{ COMMON_BIN_DIR }}/python.ecommerce {% endif %} {% if insights_code_dir is defined %}--insights-env {{ insights_home }}/insights_env --insights-code-dir {{ insights_code_dir }} --insights-python {{ COMMON_BIN_DIR }}/python.insights {% endif %} {% if analytics_api_code_dir is defined %}--analytics-api-env {{ analytics_api_home }}/analytics_api_env --analytics-api-code-dir {{ analytics_api_code_dir }} --analytics-api-python {{ COMMON_BIN_DIR }}/python.analytics_api {% endif %} {{ discovery_command }} {{ credentials_command }}

playbooks/roles/testcourses/tasks/deploy.yml

 ---
-- name: import the test courses from github
-  shell: "{{ demo_edxapp_venv_bin }}/python /edx/bin/manage.edxapp lms git_add_course --settings={{ demo_edxapp_settings }} \"{{ item.github_url }}\""
-  become_user: "{{ common_web_user }}"
-  when: item.install == True
+- include: import_course.yml
+  when: course.install == True
   with_items: "{{ TESTCOURSES_EXPORTS }}"
+  loop_control:
+    loop_var: course
 
 - name: enroll test users in the testcourses
   shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} --service-variant lms enroll_user_in_course -e {{ item[0].email }} -c {{ item[1].course_id }}"

playbooks/roles/testcourses/tasks/import_course.yml

+- name: check out testcourses
+  git:
+    dest: "{{ demo_app_dir }}/{{ course.course_id }}"
+    repo: "{{ course.github_url }}"
+    accept_hostkey: yes
+  become_user: "{{ demo_edxapp_user }}"
+  register: testcourse_checkout
+
+- name: import testcourse
+  shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_app_dir }} {{ course.course_id }}"
+  args:
+    chdir: "{{ demo_edxapp_code_dir }}"
+  when: testcourse_checkout.changed
+  become_user: "{{ common_web_user }}"

playbooks/roles/tools_jenkins/defaults/main.yml

@@ -46,7 +46,10 @@ jenkins_tools_plugins:
   - { name: "github-oauth", version: "0.24" }
   - { name: "gradle", version: "1.25" }
   - { name: "credentials-binding", version: "1.9" }
+
+# ec2 + dependencies, used by the android build workers + any additional workers we build
   - { name: "ec2", version: "1.36" }
   - { name: "node-iterator-api", version: "1.5" }
   - { name: "aws-java-sdk", version: "1.11.37" }
+  - { name: "jackson2-api", version: "2.5.4" }
   - { name: "aws-credentials", version: "1.11" }

playbooks/roles/xsy/defaults/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-##
-# Defaults for role xsy
-# 
-
-#
-# vars are namespace with the module name.
-#
-
-
-XSY_USERNAME: '1234_1234@chat.hipchat.com'
-XSY_PASSWORD: 'password'
-XSY_V1_TOKEN: 'HIPCHAT_V1_TOKEN'
-XSY_V2_TOKEN: 'HIPCHAT_V2_TOKEN'
-XSY_WILL_ROOMS: 'Hammer'
-XSY_NAME: 'xsy'
-XSY_HANDLE: 'xsy'
-XSY_REDIS_URL: 'redis://fakeuser:redispassword@localhost:6379'
-XSY_HTTPSERVER_PORT: '8081'
-XSY_WORLD_WEATHER_KEY: !!null
-
-
-xsy_service_name: xsy
-
-xsy_user: xsy
-xsy_app_dir: "{{ COMMON_APP_DIR }}/xsy"
-xsy_code_dir: "{{ xsy_app_dir }}/xsy"
-xsy_venvs_dir: "{{ xsy_app_dir }}/venvs"
-xsy_venv_dir: "{{ xsy_venvs_dir }}/xsy"
-xsy_venv_bin: "{{ xsy_venv_dir }}/bin"
-
-xsy_source_repo: "https://github.com/edx/xsy.git"
-xsy_version: "HEAD"
-xsy_requirements_file: "{{ xsy_code_dir }}/requirements.txt"
-
-xsy_supervisor_wrapper: "{{ xsy_app_dir }}/xsy-supervisor.sh"
-
-xsy_environment:
-  WILL_USERNAME: "{{ XSY_USERNAME }}"
-  WILL_PASSWORD: "{{ XSY_PASSWORD }}"
-  WILL_V1_TOKEN: "{{ XSY_V1_TOKEN }}"
-  WILL_V2_TOKEN: "{{ XSY_V2_TOKEN }}"
-  WILL_NAME: "{{ XSY_NAME }}"
-  WILL_HANDLE: "{{ XSY_HANDLE }}"
-  WILL_REDIS_URL: "{{ XSY_REDIS_URL }}"
-  WILL_HTTPSERVER_PORT: "{{ XSY_HTTPSERVER_PORT }}"
-  WILL_WORLD_WEATHER_ONLINE_KEY: "{{ XSY_WORLD_WEATHER_KEY }}"
-  WILL_ROOMS: "{{ XSY_WILL_ROOMS }}"
-
-
-#
-# OS packages
-#
-
-xsy_debian_pkgs: []
-
-xsy_redhat_pkgs: []
-

playbooks/roles/xsy/handlers/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Handlers for role xsy
-# 
-# Overview:
-# 
-#
-- name: restart xsy
-  supervisorctl:
-    name: xsy
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-    state: restarted
-  when: not disable_edx_services

playbooks/roles/xsy/meta/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-##
-# Role includes for role xsy
-# 
-# Example:
-#
-# dependencies:
-#   - {
-#   role: my_role 
-#   my_role_var0: "foo"
-#   my_role_var1: "bar"
-# }
-
-dependencies:
-  - common
-  - supervisor

playbooks/roles/xsy/tasks/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-#
-#
-# Tasks for role xsy
-# 
-# Overview:
-# 
-#
-# Dependencies:
-#
-# 
-# Example play:
-#
-#
-- name: Create application user
-  user:
-    name: "{{ xsy_user }}"
-    home: "{{ xsy_app_dir }}"
-    createhome: no
-    shell: /bin/false
-
-- name: Create xsy user dirs
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ xsy_user }}"
-    group: "{{ common_web_group }}"
-  with_items:
-    - "{{ xsy_app_dir }}"
-    - "{{ xsy_venvs_dir }}"
-
-- name: Setup the xsy env
-  template:
-    src: "xsy_env.j2"
-    dest: "{{ xsy_app_dir }}/xsy_env"
-    owner: "{{ xsy_user }}"
-    group: "{{ common_web_user }}"
-    mode: "0644"
-  notify: restart xsy
-
-- name: Checkout the code
-  git:
-    dest: "{{ xsy_code_dir }}"
-    repo: "{{ xsy_source_repo }}"
-    version: "{{ xsy_version }}"
-    accept_hostkey: yes
-  become_user: "{{ xsy_user }}"
-  register: xsy_checkout
-  notify: restart xsy
-
-- name: Install the requirements
-  pip:
-    requirements: "{{ xsy_requirements_file }}"
-    virtualenv: "{{ xsy_venv_dir }}"
-    state: present
-    extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
-  become_user: "{{ xsy_user }}"
-  notify: restart xsy
-
-- name: Create the supervisor wrapper
-  template:
-    src: "{{ xsy_supervisor_wrapper | basename }}.j2"
-    dest: "{{ xsy_supervisor_wrapper }}"
-    mode: "0755"
-  become_user: "{{ xsy_user }}"
-  notify: restart xsy
-
-- name: Create a supervisor config
-  template: 
-    src: "xsy.conf.j2"
-    dest: "{{ supervisor_available_dir }}/xsy.conf"
-    owner: "{{ supervisor_user }}"
-    group: "{{ supervisor_user }}"
-  become_user: "{{ supervisor_user }}"
-  notify: restart xsy
-
-- name: Enable the supervisor config
-  file:
-    src: "{{ supervisor_available_dir }}/xsy.conf"
-    dest: "{{ supervisor_cfg_dir }}/xsy.conf"
-    state: link
-    force: yes
-    mode: "0644"
-  become_user: "{{ supervisor_user }}"
-  when: not disable_edx_services
-  notify: restart xsy
-
-- name: Update supervisor configuration
-  shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
-  register: supervisor_update
-  changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
-  when: not disable_edx_services
-
-- name: Ensure xsy is started
-  supervisorctl:
-    name: xsy
-    supervisorctl_path: "{{ supervisor_ctl }}"
-    config: "{{ supervisor_cfg }}"
-    state: started
-  when: not disable_edx_services
-
-- include: tag_ec2.yml
-  when: COMMON_TAG_EC2_INSTANCE
-  tags:
-    - deploy

playbooks/roles/xsy/tasks/tag_ec2.yml

----
-
-- name: get instance information
-  action: ec2_facts
-  tags:
-    - deploy
-
-- name: Tag instance
-  ec2_tag:
-    resource: "{{ ansible_ec2_instance_id }}"
-    region: "{{ ansible_ec2_placement_region }}"
-    tags:
-      "version:xsy" : "{{ xsy_source_repo }} {{ xsy_checkout.after }}"
-  when: xsy_checkout.after is defined
-  tags:
-    - deploy

playbooks/roles/xsy/templates/xsy-supervisor.sh.j2

-#!/bin/bash
-
-source {{ xsy_app_dir }}/xsy_env
-cd {{ xsy_code_dir }}
-
-{{ xsy_venv_bin }}/python run_will.py

playbooks/roles/xsy/templates/xsy.conf.j2

-[program:xsy]
-
-command={{ xsy_supervisor_wrapper }}
-priority=999
-user={{ common_web_user }}
-stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
-stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
-killasgroup=true
-stopasgroup=true
-stopsignal=QUIT

playbooks/roles/xsy/templates/xsy_env.j2

-# {{ ansible_managed }}
-
-{% for name,value in xsy_environment.items() -%}
-{%- if value -%}
-export {{ name }}='{{ value }}'
-{% endif %}
-{%- endfor %}
-

playbooks/sample_vars/passwords.yml

@@ -45,7 +45,6 @@ NOTIFIER_COMMENT_SERVICE_API_KEY: '{{ FORUM_API_KEY }}'
 NOTIFIER_EMAIL_PASS: !!null
 NOTIFIER_USER_SERVICE_HTTP_AUTH_PASS: !!null
 POSTFIX_QUEUE_EXTERNAL_SMTP_PASSWORD: !!null
-PROGRAMS_DATABASE_PASSWORD: !!null
 REDIS_PASSWORD: !!null
 SPLUNKFORWARDER_PASSWORD: !!null
 SPLUNK_SMTP_PASSWORD: !!null
@@ -55,7 +54,6 @@ XQUEUE_BASIC_AUTH_PASSWORD: !!null
 XQUEUE_BASIC_AUTH_USER: !!null
 XQUEUE_MYSQL_PASSWORD: !!null
 XQUEUE_RABBITMQ_PASS: !!null
-XSY_PASSWORD: !!null
 
 EDXAPP_MONGO_PASSWORD: !!null
 EDXAPP_MONGO_USER: 'edxapp'
@@ -75,7 +73,6 @@ EDXAPP_EDXAPP_SECRET_KEY: !!null #SECRET_KEY
 EDXAPP_PROFILE_IMAGE_SECRET_KEY:  !!null #SECRET_KEY
 INSIGHTS_SECRET_KEY: !!null #SECRET_KEY
 NOTIFIER_LMS_SECRET_KEY: !!null #SECRET_KEY
-PROGRAMS_SECRET_KEY: !!null #SECRET_KEY
 
 MONGO_USERS:
   - user: "{{ FORUM_MONGO_USER }}"

playbooks/sample_vars/server_vars.yml

@@ -31,10 +31,6 @@
 #ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
 #ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 #
-#PROGRAMS_LMS_URL_ROOT: "https://${deploy_host}"
-#PROGRAMS_URL_ROOT: "https://programs-${deploy_host}"
-#PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
-#
 #CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
 #CREDENTIALS_URL_ROOT: "https://credentials-${deploy_host}"
 #CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

playbooks/vagrant-analytics.yml

@@ -28,7 +28,6 @@
     - forum
     - ecommerce
     - ecomworker
-    - programs
     - role: notifier
       NOTIFIER_DIGEST_TASK_INTERVAL: "5"
     - analytics_api

playbooks/vagrant-devstack.yml

@@ -16,6 +16,7 @@
     # When provisioning your devstack, we apply security updates
     COMMON_SECURITY_UPDATES: true
     SECURITY_UPGRADE_ON_ANSIBLE: true
+    MONGO_AUTH: false
   vars_files:
     - roles/edxapp/vars/devstack.yml
   roles:
@@ -35,7 +36,6 @@
     - ecommerce
     - role: ecomworker
       ECOMMERCE_WORKER_BROKER_HOST: 127.0.0.1
-    - programs
     - role: notifier
       NOTIFIER_DIGEST_TASK_INTERVAL: "5"
     - browsers

playbooks/vagrant-fullstack.yml

-- name: Configure instance(s)
-  hosts: all
-  become: True
-  gather_facts: True
-  vars:
-    migrate_db: 'yes'
-    EDXAPP_LMS_NGINX_PORT: '80'
-    EDX_ANSIBLE_DUMP_VARS: true
-    CERTS_DOWNLOAD_URL: 'http://192.168.33.10:18090'
-    CERTS_VERIFY_URL: 'http://192.168.33.10:18090'
-    # used for releases
-    edx_platform_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    certs_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    forum_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    xqueue_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    demo_version: '{{ OPENEDX_RELEASE | default("master") }}'
-  roles:
-    - common
-    - vhost
-    - edx_ansible
-    - user
-    - role: nginx
-      nginx_sites:
-      - cms
-      - lms
-      - forum
-      - xqueue
-      - certs
-      nginx_default_sites:
-      - lms
-      - cms
-    - mysql
-    - edxlocal
-    - memcache
-    - mongo
-    - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
-    - edxapp
-    - { role: 'edxapp', celery_worker: True }
-    - demo
-    - oraclejdk
-    - elasticsearch
-    - forum
-    - { role: "xqueue", update_users: True }
-    - certs
-    - role: analytics_api
-      when: ANALYTICS_API_GIT_IDENTITY

requirements.txt

@@ -12,7 +12,7 @@ python-simple-hipchat==0.2
 prettytable==0.7.2
 # Don't forget to update the version of this installed in
 # the aws role as well.
-awscli==1.10.28
+awscli==1.11.58
 requests==2.9.1
 datadog==0.8.0
 networkx==1.11

util/install/sandbox.sh

@@ -52,7 +52,6 @@ VERSION_VARS=(
   ANALYTICS_API_VERSION
   ECOMMERCE_VERSION
   ECOMMERCE_WORKER_VERSION
-  PROGRAMS_VERSION
 )
 
 EXTRA_VARS="-e SANDBOX_ENABLE_ECOMMERCE=True $EXTRA_VARS"

util/jenkins/ansible-provision.sh

@@ -193,10 +193,6 @@ ECOMMERCE_NGINX_PORT: 80
 ECOMMERCE_SSL_NGINX_PORT: 443
 ECOMMERCE_VERSION: $ecommerce_version
 
-PROGRAMS_NGINX_PORT: 80
-PROGRAMS_SSL_NGINX_PORT: 443
-PROGRAMS_VERSION: $programs_version
-
 CREDENTIALS_NGINX_PORT: 80
 CREDENTIALS_SSL_NGINX_PORT: 443
 CREDENTIALS_VERSION: $credentials_version
@@ -287,12 +283,6 @@ ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
 ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 
-PROGRAMS_LMS_URL_ROOT: "https://${deploy_host}"
-PROGRAMS_URL_ROOT: "https://programs-${deploy_host}"
-PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
-PROGRAMS_CORS_ORIGIN_WHITELIST:
-  - studio-${deploy_host}
-
 CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
 CREDENTIALS_DOMAIN: "credentials-${deploy_host}"
 CREDENTIALS_URL_ROOT: "https://{{ CREDENTIALS_DOMAIN }}"
@@ -301,7 +291,6 @@ COURSE_DISCOVERY_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
 
 DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
 DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
-DISCOVERY_PROGRAMS_API_URL: "{{ PROGRAMS_URL_ROOT }}/api/v1/"
 
 EOF
 fi
@@ -348,7 +337,7 @@ EOF
 fi
 
 declare -A deploy
-roles="edxapp forum ecommerce programs credentials discovery notifier xqueue xserver certs demo testcourses"
+roles="edxapp forum ecommerce credentials discovery notifier xqueue xserver certs demo testcourses"
 
 for role in $roles; do
     deploy[$role]=${!role}

util/parsefiles_config.yml

@@ -22,7 +22,6 @@ weights:
   - ecommerce: 6
   - rabbitmq: 2
   - automated: 1
-  - programs: 4
   - mysql: 2
   - elasticsearch: 7
   - docker-tools: 3

util/rabbitmq/shovel.py

+import argparse
+import subprocess
+import requests
+from requests.exceptions import HTTPError
+import sys
+
+parser=argparse.ArgumentParser(description='Shovels between RabbitMQ Clusters')
+parser.add_argument('--src_host',action='store',dest='src_host')
+parser.add_argument('--dest_host',action='store',dest='dest_host',default='127.0.0.1')
+parser.add_argument('--src_user',action='store',dest='src_user')
+parser.add_argument('--src_user_pass',action='store',dest='src_user_pass')
+parser.add_argument('--dest_user',action='store',dest='dest_user')
+parser.add_argument('--dest_user_pass',action='store',dest='dest_user_pass')
+
+args=parser.parse_args()
+
+src_uri='amqp://{}:{}@{}'.format(args.src_user,args.src_user_pass,args.src_host)
+dest_uri='amqp://{}:{}@{}'.format(args.dest_user,args.dest_user_pass,args.dest_host)
+port=15672
+
+def list_vhosts():
+    url='http://{}:{}/api/vhosts'.format(args.src_host,port)
+    try:
+        response=requests.get(url,auth=(args.src_user,args.src_user_pass))
+        response.raise_for_status()
+        vhosts=[v['name'] for v in response.json() if v['name'].startswith('/')]
+    except Exception as ex:
+        print "Failed to get vhosts: {}".format(ex)
+        sys.exit(1)
+    return vhosts
+
+def list_queues():
+    for vhost in list_vhosts():
+        url='http://{}:{}/api/queues/{}'.format(args.src_host,port,vhost)
+        try:
+            response=requests.get(url,auth=(args.src_user,args.src_user_pass))
+            response.raise_for_status()
+            queues=[q['name'] for q in response.json()]
+        except Exception as ex:
+            print "Failed to get queues: {}".format(ex)
+            sys.exit(1)
+        return queues
+
+def create_shovel(shovel,arg):
+    cmd="/usr/sbin/rabbitmqctl set_parameter shovel {} '{}'".format(shovel,arg)
+    try:
+        subprocess.check_output(
+                              cmd,stderr=subprocess.STDOUT,shell=True)
+    except subprocess.CalledProcessError as ex:
+        return ex.output
+
+if __name__=='__main__':
+
+    """
+    command line arguments are expected to be in following format
+    python shovel.py --src_host <src_host_IP> --src_user <src_rabbitmq_user> --src_user_pass <user_pass>  \
+    --dest_host <dest_host_IP> --dest_user <dest_rabbitmq_user> --dest_user_pass <user_pass>
+    """
+    output={}    
+    for queue in list_queues():
+        """ 
+        Ignore queues celeryev and *.pidbox to shovel
+        """
+        q=queue.split('.')
+        if (q[0]!='celeryev' and q[-1]!='pidbox'):
+            args='{{"src-uri": "{}", "src-queue": "{}","dest-uri": "{}","dest-queue": "{}"}}'.format(src_uri,queue,dest_uri,queue)
+            print "Running shovel for queue:{}".format(queue)
+            shovel_output=create_shovel(queue,args)
+            if shovel_output is not None:
+               content=unicode(shovel_output,"utf-8")
+               output[queue]=content
+    for k,v in output.items():
+          print k,v

util/vagrant/upgrade.sh

@@ -333,7 +333,6 @@ echo "demo_version: $TARGET" >> vars.yml
 echo "NOTIFIER_VERSION: $TARGET" >> vars.yml
 echo "ECOMMERCE_VERSION: $TARGET" >> vars.yml
 echo "ECOMMERCE_WORKER_VERSION: $TARGET" >> vars.yml
-echo "PROGRAMS_VERSION: $TARGET" >> vars.yml
 $ANSIBLE_PLAYBOOK \
     --extra-vars="@vars.yml" \
     $SERVER_VARS \

vagrant/base/analyticstack/Vagrantfile

@@ -26,7 +26,6 @@ VERSION_VARS = [
   'NOTIFIER_VERSION',
   'ECOMMERCE_VERSION',
   'ECOMMERCE_WORKER_VERSION',
-  'PROGRAMS_VERSION',
   'ANALYTICS_API_VERSION',
   'INSIGHTS_VERSION',
 ]
@@ -46,7 +45,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.network :forwarded_port, guest: 8001, host: 8001  # Studio
     config.vm.network :forwarded_port, guest: 8002, host: 8002  # Ecommerce
     config.vm.network :forwarded_port, guest: 8003, host: 8003  # LMS for Bok Choy
-    config.vm.network :forwarded_port, guest: 8004, host: 8004  # Programs
     config.vm.network :forwarded_port, guest: 8031, host: 8031  # Studio for Bok Choy
     config.vm.network :forwarded_port, guest: 8120, host: 8120  # edX Notes Service
     config.vm.network :forwarded_port, guest: 8765, host: 8765

vagrant/base/devstack/Vagrantfile

@@ -26,7 +26,6 @@ VERSION_VARS = [
     'NOTIFIER_VERSION',
     'ECOMMERCE_VERSION',
     'ECOMMERCE_WORKER_VERSION',
-    'PROGRAMS_VERSION',
 ]
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
@@ -43,7 +42,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.network :forwarded_port, guest: 8001, host: 8001  # Studio
     config.vm.network :forwarded_port, guest: 8002, host: 8002  # Ecommerce
     config.vm.network :forwarded_port, guest: 8003, host: 8003  # LMS for Bok Choy
-    config.vm.network :forwarded_port, guest: 8004, host: 8004  # Programs
     config.vm.network :forwarded_port, guest: 8031, host: 8031  # Studio for Bok Choy
     config.vm.network :forwarded_port, guest: 8120, host: 8120  # edX Notes Service
     config.vm.network :forwarded_port, guest: 8765, host: 8765

vagrant/base/fullstack/Vagrantfile

@@ -23,7 +23,6 @@ VERSION_VARS = [
     'NOTIFIER_VERSION',
     'ECOMMERCE_VERSION',
     'ECOMMERCE_WORKER_VERSION',
-    'PROGRAMS_VERSION',
 ]
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|

vagrant/release/analyticstack/Vagrantfile

@@ -21,7 +21,6 @@ VERSION_VARS = [
   'NOTIFIER_VERSION',
   'ECOMMERCE_VERSION',
   'ECOMMERCE_WORKER_VERSION',
-  'PROGRAMS_VERSION',
   'ANALYTICS_API_VERSION',
   'INSIGHTS_VERSION',
 ]
@@ -32,7 +31,6 @@ MOUNT_DIRS = {
   :forum => {:repo => "cs_comments_service", :local => "/edx/app/forum/cs_comments_service", :owner => "forum"},
   :ecommerce => {:repo => "ecommerce", :local => "/edx/app/ecommerce/ecommerce", :owner => "ecommerce"},
   :ecommerce_worker => {:repo => "ecommerce-worker", :local => "/edx/app/ecommerce_worker/ecommerce_worker", :owner => "ecommerce_worker"},
-  :programs => {:repo => "programs", :local => "/edx/app/programs/programs", :owner => "programs"},
   :insights => {:repo => "insights", :local => "/edx/app/insights/edx_analytics_dashboard", :owner => "insights"},
   :analytics_api => {:repo => "analytics_api", :local => "/edx/app/analytics_api/analytics_api", :owner => "analytics_api"},
   :analytics_pipeline => {:repo => "edx-analytics-pipeline", :local => "/edx/app/analytics_pipeline/analytics_pipeline", :owner => "hadoop"},
@@ -86,7 +84,7 @@ EXTRA_VARS="#{extra_vars_lines}"
 CONFIG_VER="#{ENV['CONFIGURATION_VERSION'] || openedx_release || 'master'}"
 
 ansible-playbook -i localhost, -c local run_role.yml -e role=edx_ansible -e configuration_version=$CONFIG_VER $EXTRA_VARS
-ansible-playbook -i localhost, -c local vagrant-analytics.yml -e configuration_version=$CONFIG_VER $EXTRA_VARS -e ELASTICSEARCH_CLUSTER_MEMBERS=[]
+ansible-playbook -i localhost, -c local vagrant-analytics.yml -e configuration_version=$CONFIG_VER $EXTRA_VARS
 
 SCRIPT
 

vagrant/release/devstack/Vagrantfile

@@ -21,7 +21,6 @@ VERSION_VARS = [
     'NOTIFIER_VERSION',
     'ECOMMERCE_VERSION',
     'ECOMMERCE_WORKER_VERSION',
-    'PROGRAMS_VERSION',
 ]
 
 MOUNT_DIRS = {
@@ -30,7 +29,6 @@ MOUNT_DIRS = {
   :forum => {:repo => "cs_comments_service", :local => "/edx/app/forum/cs_comments_service", :owner => "forum"},
   :ecommerce => {:repo => "ecommerce", :local => "/edx/app/ecommerce/ecommerce", :owner => "ecommerce"},
   :ecommerce_worker => {:repo => "ecommerce-worker", :local => "/edx/app/ecommerce_worker/ecommerce_worker", :owner => "ecommerce_worker"},
-  :programs => {:repo => "programs", :local => "/edx/app/programs/programs", :owner => "programs"},
   # This src directory won't have useful permissions. You can set them from the
   # vagrant user in the guest OS. "sudo chmod 0777 /edx/src" is useful.
   :src => {:repo => "src", :local => "/edx/src", :owner => "root"},