push in luigi configuration

TODO.txt

-* fix auto-registration of new oauth2 client in LMS (insights role)
-    * the new version of edx-oauth2-provider is not being installed for some reason?
-* manual addition of "DEBUG: true" to /edx/etc/insights.yml in order to get django to serve static assets from "runserver"
 * had to manually run "python manage.py migrate" on the insights app
-* EXTRA_SCOPES = ['permissions'] in insights base.py settings
-    * LMS master rejects this scope?
 
 
 * document instructions

playbooks/roles/analytics_pipeline/defaults/main.yml

@@ -24,7 +24,11 @@ ANALYTICS_PIPELINE_INPUT_DATABASE:
   port: 3306
 
 ANALYTICS_PIPELINE_CONFIG_DIR: "{{ COMMON_CFG_DIR }}/edx-analytics-pipeline"
-ANALYTICS_PIPELINE_HDFS_DATA_DIR: "hdfs://localhost:9000/data/"
+ANALYTICS_PIPELINE_HDFS_DATA_DIR: "hdfs://localhost:9000/data"
+
+ANALYTICS_PIPELINE_LUIGI_HADOOP_VERSION: cdh4
+ANALYTICS_PIPELINE_LUIGI_HADOOP_COMMAND: "{{ HADOOP_COMMON_HOME }}/bin/hadoop"
+ANALYTICS_PIPELINE_LUIGI_HADOOP_STREAMING_JAR: "{{ HADOOP_COMMON_HOME }}/share/hadoop/tools/lib/hadoop-streaming-{{ HADOOP_COMMON_VERSION }}.jar"
 
 #
 # vars are namespaced with the module name.

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -55,6 +55,18 @@
     dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
     mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
 
+- name: luigi configuration directory created
+  file: >
+    path=/etc/luigi
+    state=directory
+    mode=755
+
+- name: luigi configuration file written
+  template: >
+    src=client.cfg.j2
+    dest=/etc/luigi/client.cfg
+    mode=644
+
 - name: util library source checked out
   git: >
     dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
@@ -112,4 +124,4 @@
   cron: >
     user={{ hadoop_common_user }}
     name="Sync tracking log to HDFS"
-    job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
+    job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"

playbooks/roles/analytics_pipeline/templates/client.cfg.j2

+[hadoop]
+version: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_VERSION }}
+command: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_COMMAND }}
+streaming-jar: {{ ANALYTICS_PIPELINE_LUIGI_HADOOP_STREAMING_JAR }}