Refactor how we manage secure data, git-identity changes

* I've moved the example secure directory to playbooks/secure_example,
  the directory structure in your secure repo should mirror that in
  secure_example/
* If you run the lms role without a valid git-identity file it will
  fail, there is an empty file checked into the secure_example dir
* Using GIT_SSH env for git checkout, we will also need to use that
  for pip installs that use git if there are private repos in
  requirements.

README.md

@@ -39,6 +39,46 @@ version instead of the official v1.1 release._
 
 ## Organization
 
+### Secure vs. Insecure data
+
+As a general policy we want to protect the following data:
+
+* Usernames
+* Public keys (keys are ok to be public, but can be used to figure out usernames)
+* Hostnames
+* Passwords, api keys
+
+The folowing yml files and examples serve as templates that should be overridden with your own
+environment specific configuration:
+
+* vars in `vars/secure_default/` 
+* files in `files/secure_default/` 
+
+Directory structure for the secure repo:
+
+```
+
+ansible
+├── files
+├── keys
+└── vars
+
+```
+
+The same directory structure, required yml files and files are 
+in the secure_example dir:
+
+```
+secure_example/
+├── files
+├── keys
+└── vars
+```
+
+The default `secure\_dir` is set in `group\_vars/all` and can be overridden by
+adding another file in group_vars that corresponds to a deploy group name.
+
+
 The directory structure should follow Ansible best practices.
 
 http://ansible.cc/docs/bestpractices.html

playbooks/edxapp_custom.yml

 - hosts: tag_Group_edxapp_custom
   vars_files:
   # using conditional loading to override defaults for site-specific installs
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/edxapp_custom_vars.yml", "vars/secure_default/edxapp_custom_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/edxapp_custom_vars.yml"
+    - "{{ secure_dir }}/users.yml"
+    - "{{ secure_dir }}/edxapp_stage_users.yml"
   roles:
     - common
     - nginx

playbooks/edxapp_prod.yml

 - hosts: tag_Group_edxapp_prod
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_prod_vars.yml", "vars/secure_default/edxapp_prod_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_prod_users.yml", "vars/secure_default/edxapp_prod_users.yml"]
+    - "{{ secure_dir }}/edxapp_prod_vars.yml"
+    - "{{ secure_dir }}/users.yml"
+    - "{{ secure_dir }}/edxapp_prod_users.yml"
   roles:
     - common
     - nginx

playbooks/edxapp_rolling_example.yml

 # ansible-playbook -v --user=ubuntu  edxapp_rolling_example.yml -i ./ec2.py  --private-key=/path/to/deployment.pem 
 
 - hosts: tag_Group_anothermulti
-  serial: 1
+  serial: 2
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
   pre_tasks:
     - name: Gathering ec2 facts 
       ec2_facts: 
@@ -21,7 +20,7 @@
     - common
     - nginx 
     - lms
-    - ruby
+#    - ruby
   post_tasks:
     - local_action: command util/elb_reg.py -e {{ ",".join(elbs[ansible_ec2_instance_id]) }} -i {{ ansible_ec2_instance_id }} register
 # Register will pass in the same elb list and the same instance id 

playbooks/edxapp_stage.yml

 - hosts: tag_Group_edxapp_stage
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/users.yml"
+    - "{{ secure_dir }}/edxapp_stage_users.yml"
   roles:
     - common
     - nginx

playbooks/group_vars/all

 ---
 app_base_dir: /opt/wwc
-
-#where are the secure files on the deploying machine?
-secure_file_dir: ../../edx-secret/ansible/vars/
+# this path is relative to the playbook dir
+secure_dir: 'secure_example'

playbooks/group_vars/tag_Group_anothermulti

+---
+# this path is relative to the playbook dir
+#secure_dir: '../../configuration-secure/ansible'

playbooks/roles/common/tasks/create_users.yml

@@ -9,11 +9,12 @@
   user: name={{ item.user }} append=yes groups={{ "adm,edx,"+",".join(item.groups) }} shell=/bin/bash
   sudo: True
   with_items: admin_users
+  when: admin_users is defined
   tags:
   - users
   - admin_users
 - name: Copying ssh keys for admin users
-  authorized_key: user={{ item.user }} key="{{ lookup('file', item.path) }}"
+  authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
   sudo: True
   with_items: admin_keys
   tags:
@@ -23,11 +24,13 @@
   user: name={{ item.user }} groups={{ ",".join(item.groups) }} shell=/bin/bash
   sudo: True
   with_items: env_users
+  when: env_users is defined
   tags:
   - users
 - name: Copying ssh keys for env users
-  authorized_key: user={{ item.user }} key="{{ lookup('file', item.path) }}"
+  authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
   sudo: True
   with_items: env_keys
+  when: env_keys is defined
   tags:
   - users

playbooks/roles/lms/files/git_ssh.sh

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i /etc/git-identity "$@"

playbooks/roles/lms/tasks/main.yml

@@ -15,12 +15,16 @@
 
 # Install ssh keys for ubuntu account to be able to check out from mitx
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
+# {{ secure_dir }} is relative to the top-level playbooks dir so there is some
+# ugly relative pathing here
+
 - name: install read-only ssh key for mitx repo (private)
-  copy: src={{ secure_file_dir }}/ssh_deploy_private dest=/home/ubuntu/.ssh/id_rsa force=yes owner=ubuntu group=ubuntu mode=600
-- name: install read-only ssh key for mitx repo (public)
-  copy: src={{ secure_file_dir }}/ssh_deploy_public dest=/home/ubuntu/.ssh/id_rsa.pub force=yes owner=ubuntu group=ubuntu mode=644
-- name: install read-only ssh key for mitx repo (host github known)
-  copy: src={{ secure_file_dir }}/ssh_deploy_known_hosts dest=/home/ubuntu/.ssh/known_hosts force=yes owner=ubuntu group=ubuntu mode=600
+  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=root mode=644
+  sudo: True
+
+- name: upload ssh script
+  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=root mode=755
+  sudo: True
 
 # Check out mitx repo to $app_base_dir
 - name: set permissions on $app_base_dir sgid for edx
@@ -32,3 +36,5 @@
   sudo: True
 - name: git checkout mitx repo into $app_base_dir
   git: dest=$app_base_dir/mitx repo=git@github.com:MITx/mitx.git
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh

playbooks/secure_example/files/git-identity

+IDENTITY FILE FOR GIT 

playbooks/vars/secure_default/users.yml → playbooks/secure_example/vars/users.yml

@@ -10,4 +10,4 @@ admin_users:
 
 admin_keys:
 - user: joe
-  path: keys/joe.key
+  path: "{{ secure_dir }}/keys/joe.key"

playbooks/vars/secure_default/.gitignore

-# Ignore git deployment ssh keys, which should never be checked into source
-# control.
-ssh_deploy*