Initial support for journals service

playbooks/edx_continuous_integration.yml

@@ -35,6 +35,9 @@
     - analytics_api
     - ecommerce
     - credentials
+    - discovery
+    - role: journals
+      when: JOURNALS_ENABLED
     - oauth_client_setup
     - role: datadog
       when: COMMON_ENABLE_DATADOG

playbooks/edx_sandbox.yml

@@ -71,6 +71,8 @@
     - forum
     - role: discovery
       when: SANDBOX_ENABLE_DISCOVERY
+    - role: journals
+      when: JOURNALS_ENABLED
     - role: notifier
       NOTIFIER_DIGEST_TASK_INTERVAL: 5
     - role: xqueue

playbooks/journals.yml

+- name: Deploy Journals
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_NEWRELIC: False
+    CLUSTER_NAME: 'journals'
+  roles:
+    - aws
+    - role: nginx
+      nginx_default_sites:
+        - journals
+    - journals
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC
+    - role: newrelic_infrastructure
+      when: COMMON_ENABLE_NEWRELIC_INFRASTRUCTURE
+    - role: datadog-uninstall
+      when: not COMMON_ENABLE_DATADOG

playbooks/oauth_client_setup.yml

@@ -9,6 +9,7 @@
     - "roles/ecommerce/defaults/main.yml"
     - "roles/credentials/defaults/main.yml"
     - "roles/discovery/defaults/main.yml"
+    - "roles/journals/defaults/main.yml"
     - "roles/veda_web_frontend/defaults/main.yml"
   roles:
     - oauth_client_setup

playbooks/roles/docker-tools/tasks/main.yml

@@ -61,7 +61,8 @@
 
 - name: install docker-compose
   pip:
-    name: "docker-compose"
+    # lock to this version to avoid https://github.com/ansible/ansible/issues/20492
+    name: "docker-compose==1.9.0"
   tags:
     - install
     - install:system-requirements

playbooks/roles/edx_ansible/templates/update.j2

@@ -70,6 +70,7 @@ repos_to_cmd["edx-ora2"]="$edx_ansible_cmd ora2.yml -e 'ora2_version=$2'"
 repos_to_cmd["insights"]="$edx_ansible_cmd insights.yml -e 'INSIGHTS_VERSION=$2'"
 repos_to_cmd["ecommerce"]="$edx_ansible_cmd ecommerce.yml -e 'ECOMMERCE_VERSION=$2'"
 repos_to_cmd["discovery"]="$edx_ansible_cmd discovery.yml -e 'DISCOVERY_VERSION=$2'"
+repos_to_cmd["journals"]="$edx_ansible_cmd journals.yml -e 'JOURNALS_VERSION=$2'"
 repos_to_cmd["notifier"]="$edx_ansible_cmd notifier.yml -e 'NOTIFIER_VERSION=$2'"
 repos_to_cmd["video_web_frontend"]="$edx_ansible_cmd veda_web_frontend.yml -e 'VEDA_WEB_FRONTEND_VERSION=$2'"
 repos_to_cmd["video_delivery_worker"]="$edx_ansible_cmd veda_delivery_worker.yml -e 'VEDA_DELIVERY_WORKER_VERSION=$2'"

playbooks/roles/edxapp/defaults/main.yml

@@ -667,6 +667,10 @@ EDXAPP_COURSE_CATALOG_API_URL: "http://localhost:8008/api/v1"
 EDXAPP_CREDENTIALS_INTERNAL_SERVICE_URL: "http://localhost:8005"
 EDXAPP_CREDENTIALS_PUBLIC_SERVICE_URL: "http://localhost:8005"
 
+# Journals Related Settings
+EDXAPP_JOURNALS_URL_ROOT: "https://journals-{{ EDXAPP_LMS_BASE }}"
+EDXAPP_JOURNALS_API_URL: "https://journals-{{ EDXAPP_LMS_BASE }}/api/v1/"
+
 # which access.py permission name to check in order to determine if a course about page is
 # visible. We default this to the legacy permission 'see_exists'.
 
@@ -833,6 +837,27 @@ EDXAPP_BLOCK_STRUCTURES_SETTINGS:
 
   PRUNING_ACTIVE: false
 
+# Configuration needed for LMS to communicate with the Journals service
+JOURNALS_SERVICE_USER_EMAIL: "journals_worker@example.com"
+JOURNALS_SERVICE_USER_NAME: "journals_worker"
+
+# Configuration needed for LMS to communicate with the Discovery service
+DISCOVERY_SERVICE_USER_EMAIL: "discovery_worker@example.com"
+DISCOVERY_SERVICE_USER_NAME: "discovery_worker"
+
+# Configuration needed for LMS to communicate with the Ecommerce service
+ECOMMERCE_SERVICE_USER_EMAIL: "ecommerce_worker@example.com"
+ECOMMERCE_SERVICE_USER_NAME: "ecommerce_worker"
+
+# Configuration needed for LMS to communicate with the Credentials service
+CREDENTIALS_SERVICE_USER_EMAIL: "credentials_worker@example.com"
+CREDENTIALS_SERVICE_USER_NAME: "credentials_worker"
+
+# Configuration needed for LMS to communicate with the Insights service
+INSIGHTS_SERVICE_USER_EMAIL: "insights_worker@example.com"
+INSIGHTS_SERVICE_USER_NAME: "insights_worker"
+
+
 # Configuration settings needed for the LMS to communicate with the Enterprise service.
 EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/enterprise/api/v1"
 
@@ -1085,6 +1110,8 @@ generic_env_config:  &edxapp_generic_env
   CREDENTIALS_PUBLIC_SERVICE_URL: "{{ EDXAPP_CREDENTIALS_PUBLIC_SERVICE_URL }}"
   ECOMMERCE_PUBLIC_URL_ROOT: "{{ EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT }}"
   ECOMMERCE_API_URL: "{{ EDXAPP_ECOMMERCE_API_URL }}"
+  JOURNALS_URL_ROOT: "{{ EDXAPP_JOURNALS_URL_ROOT }}"
+  JOURNALS_API_URL: "{{ EDXAPP_JOURNALS_API_URL }}"
   EDX_PLATFORM_REVISION: "{{ edx_platform_version }}"
   ENTERPRISE_API_URL: "{{ EDXAPP_ENTERPRISE_API_URL }}"
   COURSE_CATALOG_API_URL: "{{ EDXAPP_COURSE_CATALOG_API_URL }}"
@@ -1527,5 +1554,21 @@ SERVICE_WORKER_USERS:
     username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME }}"
     is_staff: true
     is_superuser: false
+  - email: "{{ JOURNALS_SERVICE_USER_EMAIL }}"
+    username: "{{ JOURNALS_SERVICE_USER_NAME }}"
+    is_staff: true
+    is_superuser: false
+  - email: "{{ DISCOVERY_SERVICE_USER_EMAIL }}"
+    username: "{{ DISCOVERY_SERVICE_USER_NAME }}"
+    is_staff: true
+    is_superuser: false
+  - email: "{{ INSIGHTS_SERVICE_USER_EMAIL }}"
+    username: "{{ INSIGHTS_SERVICE_USER_NAME }}"
+    is_staff: true
+    is_superuser: false
+  - email: "{{ CREDENTIALS_SERVICE_USER_EMAIL }}"
+    username: "{{ CREDENTIALS_SERVICE_USER_NAME }}"
+    is_staff: true
+    is_superuser: false
 
 EDXAPP_ENABLE_DJANGO_ADMIN_RESTRICTION: false

playbooks/roles/edxlocal/defaults/main.yml

@@ -14,6 +14,7 @@ edxlocal_databases:
   - "{{ ANALYTICS_API_REPORTS_DB_NAME | default(None) }}"
   - "{{ CREDENTIALS_DEFAULT_DB_NAME | default(None) }}"
   - "{{ DISCOVERY_DEFAULT_DB_NAME | default(None) }}"
+  - "{{ JOURNALS_DEFAULT_DB_NAME | default(None) }}"
   - "{{ VEDA_WEB_FRONTEND_DEFAULT_DB_NAME | default(None) }}"
 
 edxlocal_database_users:
@@ -63,6 +64,11 @@ edxlocal_database_users:
       pass: "{{ DISCOVERY_MYSQL_PASSWORD | default(None) }}"
     }
   - {
+      db: "{{ JOURNALS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ JOURNALS_MYSQL_USER | default(None) }}",
+      pass: "{{ JOURNALS_MYSQL_PASSWORD | default(None) }}"
+    }
+  - {
       db: "{{ VEDA_WEB_FRONTEND_DEFAULT_DB_NAME | default(None) }}",
       user: "{{ VEDA_WEB_FRONTEND_MYSQL_USER | default(None) }}",
       pass: "{{ VEDA_WEB_FRONTEND_MYSQL_PASSWORD | default(None) }}"

playbooks/roles/journals/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role journals
+#
+
+JOURNALS_GIT_IDENTITY: !!null
+
+#
+# vars are namespace with the module name.
+#
+journals_service_name: "journals"
+journals_gunicorn_port: 8606
+
+journals_environment:
+  JOURNALS_CFG: "{{ COMMON_CFG_DIR }}/{{ journals_service_name }}.yml"
+
+journals_user: "{{ journals_service_name }}"
+journals_home: "{{ COMMON_APP_DIR }}/{{ journals_service_name }}"
+journals_code_dir: "{{ journals_home }}/{{ journals_service_name }}"
+
+#
+# OS packages
+#
+
+journals_debian_pkgs:
+  - libxml2-dev
+  - libxslt-dev
+  - libjpeg-dev
+
+
+JOURNALS_NGINX_PORT: "1{{ journals_gunicorn_port }}"
+JOURNALS_SSL_NGINX_PORT: "4{{ journals_gunicorn_port }}"
+
+JOURNALS_DEFAULT_DB_NAME: 'journals'
+JOURNALS_MYSQL: 'localhost'
+# MySQL usernames are limited to 16 characters
+JOURNALS_MYSQL_USER: 'journ001'
+JOURNALS_MYSQL_PASSWORD: 'password'
+
+# use port 9500 as elasticsearch5 will be run in docker container on this port
+JOURNALS_ELASTICSEARCH_URL: 'http://127.0.0.1:9500'
+JOURNALS_ELASTICSEARCH_INDEX_NAME: 'journals'
+
+JOURNALS_MEMCACHE: [ 'memcache' ]
+
+JOURNALS_VERSION: "master"
+JOURNALS_DJANGO_SETTINGS_MODULE: "journals.settings.production"
+JOURNALS_URL_ROOT: 'http://journals:{{ JOURNALS_NGINX_PORT }}'
+JOURNALS_LOGOUT_URL: '{{ JOURNALS_URL_ROOT }}/logout/'
+
+JOURNALS_SECRET_KEY: 'Your secret key here'
+
+JOURNALS_LANGUAGE_CODE: 'en'
+
+## Configuration for django-parler package. For more information visit
+## https://django-parler.readthedocs.io/en/latest/configuration.html#parler-languages
+JOURNALS_PARLER_DEFAULT_LANGUAGE_CODE: '{{JOURNALS_LANGUAGE_CODE}}'
+JOURNALS_PARLER_LANGUAGES :
+  1:
+    - code: 'en'
+  default:
+    fallbacks:
+      - '{{JOURNALS_PARLER_DEFAULT_LANGUAGE_CODE}}'
+    hide_untranslated: 'False'
+
+JOURNALS_DEFAULT_PARTNER_ID: 1
+JOURNALS_SESSION_EXPIRE_AT_BROWSER_CLOSE: false
+
+# Used to automatically configure OAuth2 Client
+JOURNALS_SOCIAL_AUTH_EDX_OIDC_KEY : 'journals-key'
+JOURNALS_SOCIAL_AUTH_EDX_OIDC_SECRET : 'journals-secret'
+JOURNALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
+
+JOURNALS_PLATFORM_NAME: 'Your Platform Name Here'
+
+JOURNALS_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ journals_service_name }}'
+JOURNALS_MEDIA_ROOT: '{{ JOURNALS_DATA_DIR }}/media'
+JOURNALS_MEDIA_URL: '/media/'
+
+JOURNALS_MEDIA_STORAGE_BACKEND:
+  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
+  MEDIA_ROOT: '{{ JOURNALS_MEDIA_ROOT }}'
+  MEDIA_URL: '{{ JOURNALS_MEDIA_URL }}'
+
+JOURNALS_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage'
+
+# You can set different email backends with django:
+# https://docs.djangoproject.com/en/1.9/topics/email/#email-backends
+JOURNALS_EMAIL_BACKEND: 'django_ses.SESBackend'
+
+# For email backend django-ses, the following settings are required
+JOURNALS_AWS_SES_REGION_NAME: 'us-east-1'
+JOURNALS_AWS_SES_REGION_ENDPOINT: 'email.us-east-1.amazonaws.com'
+
+# For default email backend SMTP, following settings are required
+JOURNALS_EMAIL_HOST: 'localhost'
+JOURNALS_EMAIL_PORT: 25
+JOURNALS_EMAIL_USE_TLS: False
+JOURNALS_EMAIL_HOST_USER: ''
+JOURNALS_EMAIL_HOST_PASSWORD: ''
+
+JOURNALS_PUBLISHER_FROM_EMAIL: !!null
+
+JOURNALS_OPENEXCHANGERATES_API_KEY: ''
+
+JOURNALS_GUNICORN_EXTRA: ''
+
+JOURNALS_EXTRA_APPS: []
+
+JOURNALS_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: 'journals.git'
+    VERSION: "{{ JOURNALS_VERSION }}"
+    DESTINATION: "{{ journals_code_dir }}"
+    SSH_KEY: "{{ JOURNALS_GIT_IDENTITY }}"
+
+journals_service_config_overrides:
+  ELASTICSEARCH_URL: '{{ JOURNALS_ELASTICSEARCH_URL }}'
+  ELASTICSEARCH_INDEX_NAME: '{{ JOURNALS_ELASTICSEARCH_INDEX_NAME }}'
+
+  PLATFORM_NAME: '{{ JOURNALS_PLATFORM_NAME }}'
+
+  DEFAULT_PARTNER_ID: '{{ JOURNALS_DEFAULT_PARTNER_ID }}'
+
+  EMAIL_BACKEND: '{{ JOURNALS_EMAIL_BACKEND }}'
+
+  # Settings for django-ses email backend
+  AWS_SES_REGION_NAME: '{{ JOURNALS_AWS_SES_REGION_NAME }}'
+  AWS_SES_REGION_ENDPOINT: '{{ JOURNALS_AWS_SES_REGION_ENDPOINT }}'
+
+  # Settings for default django SMTP email backend
+  EMAIL_HOST: '{{ JOURNALS_EMAIL_HOST }}'
+  EMAIL_PORT: '{{ JOURNALS_EMAIL_PORT }}'
+  EMAIL_USE_TLS: '{{ JOURNALS_EMAIL_USE_TLS }}'
+  EMAIL_HOST_USER: '{{ JOURNALS_EMAIL_HOST_USER }}'
+  EMAIL_HOST_PASSWORD: '{{ JOURNALS_EMAIL_HOST_PASSWORD }}'
+
+  PUBLISHER_FROM_EMAIL: '{{ JOURNALS_PUBLISHER_FROM_EMAIL }}'
+
+  OPENEXCHANGERATES_API_KEY: '{{ JOURNALS_OPENEXCHANGERATES_API_KEY }}'
+
+  LANGUAGE_CODE: '{{JOURNALS_LANGUAGE_CODE}}'
+  PARLER_DEFAULT_LANGUAGE_CODE: '{{JOURNALS_PARLER_DEFAULT_LANGUAGE_CODE}}'
+  PARLER_LANGUAGES : '{{JOURNALS_PARLER_LANGUAGES}}'
+  CSRF_COOKIE_SECURE: "{{ JOURNALS_CSRF_COOKIE_SECURE }}"
+
+# See edx_django_service_automated_users for an example of what this should be
+JOURNALS_AUTOMATED_USERS: {}
+
+JOURNALS_CSRF_COOKIE_SECURE: false
+
+journals_create_demo_data: false
+
+journals_post_migrate_commands:
+  - command: >
+      ./manage.py create_site
+      --sitename "DemoSite"
+      --hostname "journals-{{ EDXAPP_SITE_NAME }}"
+      --port "80"
+      --lms-url-root "{{ JOURNALS_LMS_URL_ROOT }}"
+      --lms-public-url-root-override "{{ JOURNALS_LMS_URL_ROOT }}"
+      --discovery-api-url "{{ JOURNALS_DISCOVERY_API_URL }}"
+      --ecommerce-api-url "{{ JOURNALS_ECOMMERCE_API_URL }}"
+      --discovery-partner-id "edX"
+      --ecommerce-partner-id "edX"
+      --currency-codes USD
+      --client-secret "{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}"
+      --client-id "{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_KEY }}"
+      --discovery-journal-api-url "{{ JOURNALS_DISCOVERY_JOURNALS_API_URL }}"
+      --ecommerce-journal-api-url "{{ JOURNALS_ECOMMERCE_JOURNALS_API_URL }}"
+      --ecommerce-public-url-root "{{ JOURNALS_ECOMMERCE_BASE_URL }}"
+    when: true
+  - command: './manage.py create_org --key "edX" --sitename "DemoSite"'
+    when: true
+  - command: >
+      {{ COMMON_BIN_DIR }}/python.journals {{ COMMON_BIN_DIR }}/manage.journals publish_journals --create "Demo Journal" --org "edX" --price "100.00"
+    become_user: "{{ journals_user }}"
+    environment: "{{ journals_environment }}"
+    # when: "{{ JOURNALS_ENABLED }}"
+    # re-enable once we create automatically create an Organization in Discovery
+    when: false
+
+

playbooks/roles/journals/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role journals
+#
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role
+#   my_role_var0: 'foo'
+#   my_role_var1: 'bar'
+# }
+dependencies:
+  - role: docker
+  - role: docker-tools
+  - role: edx_django_service
+    edx_django_service_version: '{{ JOURNALS_VERSION }}'
+    edx_django_service_repos: '{{ JOURNALS_REPOS }}'
+    edx_django_service_name: '{{ journals_service_name }}'
+    edx_django_service_user: '{{ journals_user }}'
+    edx_django_service_home: '{{ COMMON_APP_DIR }}/{{ journals_service_name }}'
+    edx_django_service_config_overrides: '{{ journals_service_config_overrides }}'
+    edx_django_service_debian_pkgs_extra: '{{ journals_debian_pkgs }}'
+    edx_django_service_gunicorn_port: '{{ journals_gunicorn_port }}'
+    edx_django_service_django_settings_module: '{{ JOURNALS_DJANGO_SETTINGS_MODULE }}'
+    edx_django_service_environment_extra: '{{ journals_environment }}'
+    edx_django_service_gunicorn_extra: '{{ JOURNALS_GUNICORN_EXTRA }}'
+    edx_django_service_wsgi_name: 'journals'
+    edx_django_service_nginx_port: '{{ JOURNALS_NGINX_PORT }}'
+    edx_django_service_ssl_nginx_port: '{{ JOURNALS_SSL_NGINX_PORT }}'
+    edx_django_service_language_code: '{{ JOURNALS_LANGUAGE_CODE }}'
+    edx_django_service_secret_key: '{{ JOURNALS_SECRET_KEY }}'
+    edx_django_service_staticfiles_storage: '{{ JOURNALS_STATICFILES_STORAGE }}'
+    edx_django_service_media_storage_backend: '{{ JOURNALS_MEDIA_STORAGE_BACKEND }}'
+    edx_django_service_memcache: '{{ JOURNALS_MEMCACHE }}'
+    edx_django_service_default_db_host: '{{ JOURNALS_MYSQL }}'
+    edx_django_service_default_db_name: '{{ JOURNALS_DEFAULT_DB_NAME }}'
+    edx_django_service_default_db_atomic_requests: false
+    edx_django_service_db_user: '{{ JOURNALS_MYSQL_USER }}'
+    edx_django_service_db_password: '{{ JOURNALS_MYSQL_PASSWORD }}'
+    edx_django_service_social_auth_edx_oidc_key: '{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_KEY }}'
+    edx_django_service_social_auth_edx_oidc_secret: '{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
+    edx_django_service_social_auth_redirect_is_https: '{{ JOURNALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
+    edx_django_service_extra_apps: '{{ JOURNALS_EXTRA_APPS }}'
+    edx_django_service_session_expire_at_browser_close: '{{ JOURNALS_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_automated_users: '{{ JOURNALS_AUTOMATED_USERS }}'
+    edx_django_service_use_python3: true
+    edx_django_service_post_migrate_commands: '{{ journals_post_migrate_commands }}'
+    edx_django_service_has_static_assets: true

playbooks/roles/journals/tasks/Dockerfile

+FROM elasticsearch:5.6.9
+RUN bin/elasticsearch-plugin install ingest-attachment

playbooks/roles/journals/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role discovery
+#
+# Overview: This role's tasks come from edx_django_service.
+#
+#
+# Dependencies:
+#
+#
+# Example play:
+#
+#
+- name: Create LMS catalog integration
+  shell: >
+    {{ COMMON_BIN_DIR }}/python.edxapp {{ COMMON_BIN_DIR }}/manage.edxapp lms --settings={{ COMMON_EDXAPP_SETTINGS }}
+    create_catalog_integrations --enabled --internal_api_url {{ EDXAPP_DISCOVERY_API_URL }}
+    --service_username {{ DISCOVERY_SERVICE_USER_NAME }}
+  become_user: "{{ edxapp_user }}"
+  environment: "{{ edxapp_environment }}"
+  tags:
+    - manage
+    - manage:db
+
+- name: create web-writable journal data dirs
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ common_web_user }}"
+    group: "{{ journals_user }}"
+    mode: "0775"
+  with_items:
+    - "{{ JOURNALS_MEDIA_ROOT }}"
+  tags:
+    - install
+    - install:base
+
+- name: build elasticsearch5 image
+  docker_image:
+     path: /edx/app/edx_ansible/edx_ansible/playbooks/roles/journals/tasks/
+     name: elasticsearch5
+     state: present
+  tags:
+    - install
+    - install:base
+
+- name: create elasticsearch5 docker container
+  docker_container:
+    name: elasticsearch5
+    image: elasticsearch5
+    state: started
+    ports:
+     - "9500:9200"
+  tags:
+    - install
+    - install:base
+
+- name: wait for elasticsearch container to start
+  wait_for:
+    port: 9500
+    delay: 10
+  tags:
+    - install
+    - install:base
+
+- name: update journal elasticsearch index
+  shell: >
+    {{ COMMON_BIN_DIR }}/python.journals {{ COMMON_BIN_DIR }}/manage.journals update_index --settings={{ JOURNALS_DJANGO_SETTINGS_MODULE }}
+  become_user: "{{ journals_user }}"
+  environment: "{{ journals_environment }}"
+  tags:
+    - install
+    - install:app-requirements

playbooks/roles/launch_ec2/tasks/main.yml

@@ -89,7 +89,7 @@
   delay: 30
   with_nested:
     - "{{ ec2.instances }}"
-    - ['studio', 'ecommerce', 'preview', 'discovery', 'credentials', 'veda', 'analytics-api']
+    - ['studio', 'ecommerce', 'preview', 'discovery', 'journals', 'credentials', 'veda', 'analytics-api']
 
 - name: Add DNS name for whitelabel sites
   local_action:

playbooks/roles/local_dev/defaults/main.yml

@@ -69,6 +69,13 @@ localdev_accounts:
         repo: "discovery"
       }
 
+    - {
+        user: "{{ journals_user|default('None') }}",
+        home: "{{ journals_home|default('None') }}",
+        env: "journals_env",
+        repo: "journals"
+      }
+
 # Helpful system packages for local dev
 local_dev_pkgs:
     - vim

playbooks/roles/mariadb/defaults/main.yml

@@ -76,6 +76,10 @@ MARIADB_DATABASES:
       encoding: "utf8"
     }
   - {
+      db: "{{ JOURNALS_DEFAULT_DB_NAME | default(None) }}",
+      encoding: "utf8"
+    }
+  - {
       db: "{{ HIVE_METASTORE_DATABASE_NAME | default(None) }}",
       encoding: "latin1"
     }
@@ -132,6 +136,11 @@ MARIADB_USERS:
       user: "{{ DISCOVERY_MYSQL_USER | default(None) }}",
       pass: "{{ DISCOVERY_MYSQL_PASSWORD | default(None) }}"
     }
+  - {
+      db: "{{ JOURNALS_DEFAULT_DB_NAME | default(None) }}",
+      user: "{{ JOURNALS_MYSQL_USER | default(None) }}",
+      pass: "{{ JOURNALS_MYSQL_PASSWORD | default(None) }}"
+    }
 
 #
 # OS packages

playbooks/roles/oauth_client_setup/defaults/main.yml

@@ -22,35 +22,48 @@ oauth_client_setup_oauth2_clients:
         url_root: "{{ ECOMMERCE_ECOMMERCE_URL_ROOT | default('None') }}",
         id: "{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
         secret: "{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
-        logout_uri: "{{ ECOMMERCE_LOGOUT_URL | default('None') }}"
+        logout_uri: "{{ ECOMMERCE_LOGOUT_URL | default('None') }}",
+        username: "{{ ECOMMERCE_SERVICE_USER_NAME | default('None') }}",
       }
     - {
         name: "{{ INSIGHTS_OAUTH2_APP_CLIENT_NAME | default('None') }}",
         url_root: "{{ INSIGHTS_BASE_URL | default('None') }}",
         id: "{{ INSIGHTS_OAUTH2_KEY | default('None') }}",
         secret: "{{ INSIGHTS_OAUTH2_SECRET | default('None') }}",
-        logout_uri: "{{ INSIGHTS_LOGOUT_URL | default('None') }}"
+        logout_uri: "{{ INSIGHTS_LOGOUT_URL | default('None') }}",
+        username: "{{ INSIGHTS_SERVICE_USER_NAME | default('None') }}",
       }
     - {
         name: "{{ credentials_service_name | default('None') }}",
         url_root: "{{ CREDENTIALS_URL_ROOT | default('None') }}",
         id: "{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
         secret: "{{ CREDENTIALS_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
-        logout_uri: "{{ CREDENTIALS_LOGOUT_URL | default('None') }}"
+        logout_uri: "{{ CREDENTIALS_LOGOUT_URL | default('None') }}",
+        username: "{{ CREDENTIALS_SERVICE_USER_NAME | default('None') }}",
       }
     - {
         name: "{{ discovery_service_name | default('None') }}",
         url_root: "{{ DISCOVERY_URL_ROOT | default('None') }}",
         id: "{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
         secret: "{{ DISCOVERY_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
-        logout_uri: "{{ DISCOVERY_LOGOUT_URL | default('None') }}"
+        logout_uri: "{{ DISCOVERY_LOGOUT_URL | default('None') }}",
+        username: "{{ DISCOVERY_SERVICE_USER_NAME | default('None') }}",
+      }
+    - {
+        name: "{{ journals_service_name | default('None') }}",
+        url_root: "{{ JOURNALS_URL_ROOT | default('None') }}",
+        id: "{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
+        secret: "{{ JOURNALS_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
+        logout_uri: "{{ JOURNALS_LOGOUT_URL | default('None') }}",
+        username: "{{ JOURNALS_SERVICE_USER_NAME | default('None') }}",
       }
     - {
         name: "{{ veda_web_frontend_service_name | default('None') }}",
         url_root: "{{ VEDA_WEB_FRONTEND_OAUTH2_URL | default('None') }}",
         id: "{{ VEDA_WEB_FRONTEND_SOCIAL_AUTH_EDX_OIDC_KEY | default('None') }}",
         secret: "{{ VEDA_WEB_FRONTEND_SOCIAL_AUTH_EDX_OIDC_SECRET | default('None') }}",
-        logout_uri: "{{ VEDA_WEB_FRONTEND_LOGOUT_URL | default('None') }}"
+        logout_uri: "{{ VEDA_WEB_FRONTEND_LOGOUT_URL | default('None') }}",
+        username: "{{ EDXAPP_VEDA_SERVICE_USER_NAME | default('None') }}"
       }
 
 #

playbooks/roles/oauth_client_setup/tasks/main.yml

@@ -33,6 +33,7 @@
     --client_secret {{ item.secret }}
     --trusted
     --logout_uri {{ item.logout_uri | default("") }}
+    --username {{ item.username }}
   become_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
   with_items: "{{ oauth_client_setup_oauth2_clients }}"

playbooks/roles/supervisor/defaults/main.yml

@@ -76,3 +76,7 @@ supervisor_spec:
     python: python.discovery
     code: "{{ discovery_code_dir | default(None) }}"
     env: "{{ discovery_home | default(None) }}/discovery_env"
+  - service: journals
+    python: python.journals
+    code: "{{ journals_code_dir | default(None) }}"
+    env: "{{ journals_home | default(None) }}/journals_env"

playbooks/roles/supervisor/files/pre_supervisor_checks.py

@@ -19,6 +19,7 @@ MIGRATION_COMMANDS = {
         'analytics_api': ". {env_file}; sudo -E -u analytics_api {python} {code_dir}/manage.py showmigrations",
         'credentials':   ". {env_file}; sudo -E -u credentials {python} {code_dir}/manage.py showmigrations",
         'discovery':     ". {env_file}; sudo -E -u discovery {python} {code_dir}/manage.py showmigrations",
+        'journals':     ". {env_file}; sudo -E -u journals {python} {code_dir}/manage.py showmigrations",
     }
 NGINX_ENABLE = {
         'lms':  "sudo ln -sf /edx/app/nginx/sites-available/lms /etc/nginx/sites-enabled/lms",
@@ -119,6 +120,16 @@ if __name__ == '__main__':
     discovery_migration_args.add_argument("--discovery-code-dir",
         help="Location of the discovery code.")
 
+    journals_migration_args = parser.add_argument_group("journals_migrations",
+            "Args for running journals migration checks.")
+    journals_migration_args.add_argument("--journals-python",
+        help="Path to python to use for executing migration check.")
+    journals_migration_args.add_argument("--journals-env",
+        help="Location of the journals environment file.")
+    journals_migration_args.add_argument("--journals-code-dir",
+        help="Location of the journals code.")
+
+
     insights_migration_args = parser.add_argument_group("insights_migrations",
             "Args for running insights migration checks.")
     insights_migration_args.add_argument("--insights-python",
@@ -235,6 +246,7 @@ if __name__ == '__main__':
                     "ecommerce": {'python': args.ecommerce_python, 'env_file': args.ecommerce_env, 'code_dir': args.ecommerce_code_dir},
                     "credentials": {'python': args.credentials_python, 'env_file': args.credentials_env, 'code_dir': args.credentials_code_dir},
                     "discovery": {'python': args.discovery_python, 'env_file': args.discovery_env, 'code_dir': args.discovery_code_dir},
+                    "journals": {'python': args.journals_python, 'env_file': args.journals_env, 'code_dir': args.journals_code_dir},                    
                     "insights": {'python': args.insights_python, 'env_file': args.insights_env, 'code_dir': args.insights_code_dir},
                     "analytics_api": {'python': args.analytics_api_python, 'env_file': args.analytics_api_env, 'code_dir': args.analytics_api_code_dir},
                     "xqueue": {'python': args.xqueue_python, 'env_file': args.xqueue_env, 'code_dir': args.xqueue_code_dir},

playbooks/roles/whitelabel/tasks/main.yml

@@ -35,3 +35,16 @@
    --dns-name {{ dns_name }} --settings=ecommerce.settings.production --theme-path {{ WHITELABEL_THEME_DIR }}
   become_user: "{{ ecommerce_user }}"
   environment: "{{ ecommerce_environment }}"
+
+# Journals
+
+- name: Create Sites and Partners for Journals
+  shell: >
+      {{ COMMON_BIN_DIR }}/python.journals {{ COMMON_BIN_DIR }}/manage.journals create_themed_sites
+      --settings={{ JOURNALS_DJANGO_SETTINGS_MODULE }}
+      --dns-name {{ dns_name }}
+      --theme-path {{ WHITELABEL_THEME_DIR }}
+      --port "80"
+  become_user: "{{ journals_user }}"
+  environment: "{{ journals_environment }}"
+  when: "{{ JOURNALS_ENABLED }}"

playbooks/whitelabel.yml

@@ -10,6 +10,7 @@
     - "roles/edxapp/defaults/main.yml"
     - "roles/discovery/defaults/main.yml"
     - "roles/ecommerce/defaults/main.yml"
+    - "roles/journals/defaults/main.yml"
   roles:
     - aws
     - edxlocal

requirements.txt

@@ -49,3 +49,5 @@ simplejson==3.15.0        # via datadog
 six==1.11.0               # via bcrypt, cryptography, pathlib2, pynacl, python-dateutil
 urllib3==1.22             # via requests
 wsgiref==0.1.2
+# lock to this version to avoid https://github.com/ansible/ansible/issues/20492
+docker-compose==1.9.0

util/jenkins/ansible-provision.sh

@@ -173,6 +173,15 @@ if [[ -z $set_whitelabel ]]; then
   set_whitelabel="true"
 fi
 
+if [[ -z $journals ]]; then
+  journals="false"
+fi
+
+if [[ -z $journals_version ]]; then
+  journals_version="master"
+fi
+
+
 # Lowercase the dns name to deal with an ansible bug
 dns_name="${dns_name,,}"
 
@@ -191,6 +200,7 @@ certs_version: $certs_version
 configuration_version: $configuration_version
 demo_version: $demo_version
 THEMES_VERSION: $themes_version
+journals_version: $journals_version
 
 edx_ansible_source_repo: ${configuration_source_repo}
 edx_platform_repo: ${edx_platform_repo}
@@ -213,6 +223,11 @@ ANALYTICS_API_NGINX_PORT: 80
 ANALYTICS_API_SSL_NGINX_PORT: 443
 ANALYTICS_API_VERSION: $analytics_api_version
 
+JOURNALS_NGINX_PORT: 80
+JOURNALS_SSL_NGINX_PORT: 443
+JOURNALS_VERSION: $journals_version
+JOURNALS_ENABLED: $journals
+
 VIDEO_PIPELINE_BASE_NGINX_PORT: 80
 VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: 443
 
@@ -298,7 +313,8 @@ FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
 SANDBOX_USERNAME: $github_username
 EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
 EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
-EDXAPP_COURSE_CATALOG_API_URL: "https://catalog-${deploy_host}/api/v1"
+EDXAPP_DISCOVERY_API_URL: "https://discovery-${deploy_host}/api/v1"
+EDXAPP_COURSE_CATALOG_API_URL: "{{ EDXAPP_DISCOVERY_API_URL }}"
 
 ANALYTICS_API_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}/"
 
@@ -309,6 +325,18 @@ ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
 ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 ecommerce_create_demo_data: true
 
+JOURNALS_URL_ROOT: "https://journals-{{ EDXAPP_LMS_BASE }}"
+JOURNALS_API_URL: "https://journals-{{ EDXAPP_LMS_BASE }}/api/v1/"
+JOURNALS_DISCOVERY_SERVICE_URL: "https://discovery-{{ EDXAPP_LMS_BASE }}"
+JOURNALS_LMS_URL_ROOT: "https://{{ EDXAPP_LMS_BASE }}"
+JOURNALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
+JOURNALS_DISCOVERY_API_URL: "{{ JOURNALS_DISCOVERY_SERVICE_URL }}/api/v1/"
+JOURNALS_DISCOVERY_JOURNALS_API_URL: "{{ JOURNALS_DISCOVERY_SERVICE_URL }}/journal/api/v1/"
+JOURNALS_ECOMMERCE_BASE_URL: "{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}"
+JOURNALS_ECOMMERCE_API_URL: "{{ JOURNALS_ECOMMERCE_BASE_URL }}/api/v2/"
+JOURNALS_ECOMMERCE_JOURNALS_API_URL: "{{ JOURNALS_ECOMMERCE_BASE_URL }}/journal/api/v1"
+journals_create_demo_data: true
+
 DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
 DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
 
@@ -378,7 +406,7 @@ veda_encode_worker=${video_encode_worker:-false}
 video_pipeline_integration=${video_pipeline:-false}
 
 declare -A deploy
-plays="edxapp forum ecommerce credentials discovery analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration notifier xqueue xserver certs demo testcourses"
+plays="edxapp forum ecommerce credentials discovery journals analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration notifier xqueue xserver certs demo testcourses"
 
 for play in $plays; do
     deploy[$play]=${!play}