First round of review comments

playbooks/edx-east/analytics_single.yml

@@ -6,6 +6,9 @@
     migrate_db: "yes"
     disable_edx_services: true
     insights_register_oauth_app: yes
+    EDXAPP_LMS_BASE: 127.0.0.1:8000
+    EDXAPP_OAUTH_ENFORCE_SECURE: false
+    EDXAPP_LMS_BASE_SCHEME: http
   roles:
     - edxlocal
     - analytics_api

playbooks/roles/edxapp/defaults/main.yml

@@ -687,7 +687,7 @@ generic_env_config:  &edxapp_generic_env
   CROSS_DOMAIN_CSRF_COOKIE_NAME: "{{ EDXAPP_CROSS_DOMAIN_CSRF_COOKIE_NAME }}"
   VIDEO_UPLOAD_PIPELINE: "{{ EDXAPP_VIDEO_UPLOAD_PIPELINE }}"
   DEPRECATED_ADVANCED_COMPONENT_TYPES: "{{ EDXAPP_DEPRECATED_ADVANCED_COMPONENT_TYPES }}"
-  OAUTH_OIDC_ISSUER: "{{ EDXAPP_LMS_BASE_SCHEME | default('http') }}://{{ EDXAPP_LMS_BASE }}/oauth2"
+  OAUTH_OIDC_ISSUER: "{{ EDXAPP_LMS_BASE_SCHEME | default('https') }}://{{ EDXAPP_LMS_BASE }}/oauth2"
   XBLOCK_FS_STORAGE_BUCKET: "{{ EDXAPP_XBLOCK_FS_STORAGE_BUCKET }}"
   XBLOCK_FS_STORAGE_PREFIX: "{{ EDXAPP_XBLOCK_FS_STORAGE_PREFIX }}"
   ANALYTICS_DATA_URL: "{{ EDXAPP_ANALYTICS_DATA_URL }}"

playbooks/roles/hadoop_common/defaults/main.yml

@@ -13,9 +13,9 @@
 
 HADOOP_COMMON_VERSION: 2.3.0
 HADOOP_COMMON_USER_HOME: /home/hadoop
-HADOOP_COMMON_HOME: "{{ HADOOP_COMMON_USER_HOME }}/hadoop"
+HADOOP_COMMON_HOME: "{{ COMMON_APP_DIR }}/hadoop"
+HADOOP_COMMON_DATA: "{{ COMMON_DATA_DIR }}/hadoop"
 HADOOP_COMMON_LOGS: "{{ HADOOP_COMMON_HOME }}/logs"
-HADOOP_COMMON_DATA: /var/lib/hadoop/hdfs
 HADOOP_COMMON_CONF_DIR: "{{ HADOOP_COMMON_HOME }}/etc/hadoop"
 HADOOP_COMMON_PROTOBUF_VERSION: 2.5.0
 HADOOP_COMMON_SERVICES_DIR: "{{ HADOOP_COMMON_USER_HOME }}/services.d"
@@ -37,8 +37,8 @@ hadoop_common_native_dist:
   filename: "release-{{ HADOOP_COMMON_VERSION }}.tar.gz"
   url: "https://github.com/apache/hadoop-common/archive/release-{{ HADOOP_COMMON_VERSION }}.tar.gz"
   sha256sum: a8e1b49d4e891255d465e9449346ac7fb259bb35dce07d9f0df3b46fac3e9bd0
-hadoop_common_java_home: /usr/lib/jvm/java-7-oracle
-hadoop_common_env: "{{ HADOOP_COMMON_USER_HOME }}/.hadoop_env"
+hadoop_common_java_home: "{{ oraclejdk_link }}"
+hadoop_common_env: "{{ HADOOP_COMMON_HOME }}/.hadoop_env"
 
 #
 # OS packages

playbooks/roles/insights/defaults/main.yml

@@ -121,7 +121,7 @@ INSIGHTS_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
 INSIGHTS_NGINX_PORT: "18110"
 INSIGHTS_GUNICORN_WORKERS: "2"
 INSIGHTS_GUNICORN_EXTRA: ""
-INSIGHTS_COURSE_API_URL: "{{ INSIGHTS_LMS_BASE}}/api/course_structure/v0/"
+INSIGHTS_COURSE_API_URL: "{{ INSIGHTS_LMS_BASE }}/api/course_structure/v0/"
 
 INSIGHTS_VERSION: "master"
 INSIGHTS_GIT_IDENTITY: !!null

playbooks/roles/sqoop/defaults/main.yml

@@ -12,6 +12,9 @@
 # 
 
 SQOOP_VERSION: 1.4.6
+
+# There is no non-alpha version here, this is just the version of sqoop that is compatible with Hadoop 2
+SQOOP_HADOOP_VERSION: 2.0.4-alpha
 SQOOP_MYSQL_CONNECTOR_VERSION: 5.1.29
 SQOOP_HOME: "{{ HADOOP_COMMON_USER_HOME }}/sqoop"
 SQOOP_CONF: "{{ SQOOP_HOME }}/conf"
@@ -21,9 +24,11 @@ SQOOP_LIB: "{{ SQOOP_HOME }}/lib"
 # vars are namespace with the module name.
 #
 sqoop_role_name: sqoop
+sqoop_temporary_dir: /var/tmp
+sqoop_base_filename: "sqoop-{{ SQOOP_VERSION }}.bin__hadoop-{{ SQOOP_HADOOP_VERSION }}"
 sqoop_dist:
-  filename: "sqoop-{{ SQOOP_VERSION }}.bin__hadoop-2.0.4-alpha.tar.gz"
-  url: "http://www.carfab.com/apachesoftware/sqoop/{{ SQOOP_VERSION }}/sqoop-{{ SQOOP_VERSION }}.bin__hadoop-2.0.4-alpha.tar.gz"
+  filename: "{{ sqoop_base_filename }}.tar.gz"
+  url: "http://www.carfab.com/apachesoftware/sqoop/{{ SQOOP_VERSION }}/{{ sqoop_base_filename }}.tar.gz"
   sha256sum: d582e7968c24ff040365ec49764531cb76dfa22c38add5f57a16a57e70d5d496
 sqoop_mysql_connector_dist:
   filename: "mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}.tar.gz"

playbooks/roles/sqoop/tasks/main.yml

@@ -29,18 +29,18 @@
     url={{ sqoop_dist.url }}
     sha256sum={{ sqoop_dist.sha256sum }}
     validate_certs=no
-    dest=/tmp
+    dest={{ sqoop_temporary_dir }}
   when: not installed.stat.exists
 
 - name: distribution extracted
   shell: >
     chdir={{ HADOOP_COMMON_USER_HOME }}
-    tar -xzf /tmp/{{ sqoop_dist.filename }} && chown -R {{ hadoop_common_user }}:{{ hadoop_common_group }} sqoop-{{ SQOOP_VERSION }}.bin__hadoop-2.0.4-alpha
+    tar -xzf {{ sqoop_temporary_dir }}/{{ sqoop_dist.filename }} && chown -R {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ sqoop_base_filename }}
   when: not installed.stat.exists
 
 - name: versioned directory symlink created
   file: >
-    src={{ HADOOP_COMMON_USER_HOME }}/sqoop-{{ SQOOP_VERSION }}.bin__hadoop-2.0.4-alpha
+    src={{ HADOOP_COMMON_USER_HOME }}/{{ sqoop_base_filename }}
     dest={{ SQOOP_HOME }}
     owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=link
 
@@ -49,13 +49,13 @@
     url={{ sqoop_mysql_connector_dist.url }}
     sha256sum={{ sqoop_mysql_connector_dist.sha256sum }}
     validate_certs=no
-    dest=/tmp
+    dest={{ sqoop_temporary_dir }}
   when: not installed.stat.exists
 
 - name: mysql connector distribution extracted
   shell: >
-    chdir=/tmp
-    tar -xzf /tmp/{{ sqoop_mysql_connector_dist.filename }}
+    chdir={{ sqoop_temporary_dir }}
+    tar -xzf {{ sqoop_temporary_dir }}/{{ sqoop_mysql_connector_dist.filename }}
   when: not installed.stat.exists
 
 - name: sqoop lib exists
@@ -65,7 +65,7 @@
 
 - name: mysql connector installed
   shell: >
-    chdir=/tmp/mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}
+    chdir=/{{ sqoop_temporary_dir }}/mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}
     cp mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}-bin.jar {{ SQOOP_LIB }} &&
     chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ SQOOP_LIB }}/mysql-connector-java-{{ SQOOP_MYSQL_CONNECTOR_VERSION }}-bin.jar
   when: not installed.stat.exists

playbooks/vagrant-analytics.yml

@@ -15,6 +15,7 @@
     ENABLE_LEGACY_ORA: !!null
     EDXAPP_LMS_BASE: 127.0.0.1:8000
     EDXAPP_OAUTH_ENFORCE_SECURE: false
+    EDXAPP_LMS_BASE_SCHEME: http
   roles:
     - edx_ansible
     - edxlocal

playbooks/vagrant-devstack.yml

@@ -14,6 +14,7 @@
     ENABLE_LEGACY_ORA: !!null
     EDXAPP_LMS_BASE: 127.0.0.1:8000
     EDXAPP_OAUTH_ENFORCE_SECURE: false
+    EDXAPP_LMS_BASE_SCHEME: http
   roles:
     - edx_ansible
     - edxlocal