Merge branch 'master' of github.com:edx/configuration into feature/sef/refarch-vpc-dns-ssh

README.md

@@ -39,6 +39,46 @@ version instead of the official v1.1 release._
 
 ## Organization
 
+### Secure vs. Insecure data
+
+As a general policy we want to protect the following data:
+
+* Usernames
+* Public keys (keys are ok to be public, but can be used to figure out usernames)
+* Hostnames
+* Passwords, api keys
+
+The folowing yml files and examples serve as templates that should be overridden with your own
+environment specific configuration:
+
+* vars in `secure_example/vars` 
+* files in `secure_example/files` 
+
+Directory structure for the secure repo:
+
+```
+
+ansible
+├── files
+├── keys
+└── vars
+
+```
+
+The same directory structure, required yml files and files are 
+in the secure_example dir:
+
+```
+secure_example/
+├── files
+├── keys
+└── vars
+```
+
+The default `secure\_dir` is set in `group\_vars/all` and can be overridden by
+adding another file in group_vars that corresponds to a deploy group name.
+
+
 The directory structure should follow Ansible best practices.
 
 http://ansible.cc/docs/bestpractices.html
@@ -84,8 +124,9 @@ Example users are in the `vars/secure` directory:
 
 ```
 cloudformation_templates  <-- official edX cloudformation templates
-│   └── examples          <-- example templates
-└── playbooks
+    └── examples          <-- example templates
+playbooks
+ └──
      edxapp_prod.yml      <-- example production environment playbook
      edxapp_stage.yml     <-- example stage environment playbook
      edxapp_custom.yml    <-- example custom environment playbook
@@ -108,7 +149,7 @@ cloudformation_templates  <-- official edX cloudformation templates
     │       └── templates 
     │   (etc)
     └── vars             <-- public variable definitions
-        └── secure       <-- secure variables (example)
+    └── secure_example   <-- secure variables (example)
 
 ```
 

playbooks/edxapp_custom.yml

 - hosts: tag_Group_edxapp_custom
   vars_files:
   # using conditional loading to override defaults for site-specific installs
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/edxapp_custom_vars.yml", "vars/secure_default/edxapp_custom_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/vars/edxapp_custom_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
     - nginx

playbooks/edxapp_prod.yml

 - hosts: tag_Group_edxapp_prod
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_prod_vars.yml", "vars/secure_default/edxapp_prod_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_prod_users.yml", "vars/secure_default/edxapp_prod_users.yml"]
+    - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
   roles:
     - common
     - nginx

playbooks/edxapp_rolling_example.yml

 # ansible-playbook -v --user=ubuntu  edxapp_rolling_example.yml -i ./ec2.py  --private-key=/path/to/deployment.pem 
 
 - hosts: tag_Group_anothermulti
-  serial: 1
+  serial: 2
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
   pre_tasks:
     - name: Gathering ec2 facts 
       ec2_facts: 
@@ -21,7 +20,7 @@
     - common
     - nginx 
     - lms
-    - ruby
+#    - ruby
   post_tasks:
     - local_action: command util/elb_reg.py -e {{ ",".join(elbs[ansible_ec2_instance_id]) }} -i {{ ansible_ec2_instance_id }} register
 # Register will pass in the same elb list and the same instance id 

playbooks/edxapp_stage.yml

 - hosts: tag_Group_edxapp_stage
   vars_files:
-    - ["{{ secure_file_dir }}/edxapp_stage_vars.yml", "vars/secure_default/edxapp_stage_vars.yml"]
-    - ["{{ secure_file_dir }}/users.yml", "vars/secure_default/users.yml"]
-    - ["{{ secure_file_dir }}/edxapp_stage_users.yml", "vars/secure_default/edxapp_stage_users.yml"]
+    - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/edxapp_stage_users.yml"
   roles:
     - common
     - nginx

playbooks/group_vars/all

 ---
 app_base_dir: /opt/wwc
+# this path is relative to the playbook dir
+secure_dir: 'secure_example'
 venv_dir: /opt/edx
-
-#where are the secure files on the deploying machine?
-secure_file_dir: ../../edx-secret/ansible/vars/

playbooks/group_vars/tag_Group_anothermulti

+---
+# this path is relative to the playbook dir
+#secure_dir: '../../configuration-secure/ansible'

playbooks/group_vars/tag_Group_edxapp_prod

 ---
 edxapp_prod: true
+secure_dir: '../../configuration-secure/ansible'
\ No newline at end of file

playbooks/roles/common/tasks/create_users.yml

@@ -6,11 +6,12 @@
   user: name={{ item.user }} append=yes groups={{ "adm,edx,"+",".join(item.groups) }} shell=/bin/bash
   sudo: True
   with_items: admin_users
+  when: admin_users is defined
   tags:
   - users
   - admin_users
 - name: Copying ssh keys for admin users
-  authorized_key: user={{ item.user }} key="{{ lookup('file', item.path) }}"
+  authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
   sudo: True
   with_items: admin_keys
   tags:
@@ -20,11 +21,13 @@
   user: name={{ item.user }} groups={{ ",".join(item.groups) }} shell=/bin/bash
   sudo: True
   with_items: env_users
+  when: env_users is defined
   tags:
   - users
 - name: Copying ssh keys for env users
-  authorized_key: user={{ item.user }} key="{{ lookup('file', item.path) }}"
+  authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
   sudo: True
   with_items: env_keys
+  when: env_keys is defined
   tags:
   - users

playbooks/roles/lms/files/git_ssh.sh

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i /etc/git-identity "$@"

playbooks/roles/lms/tasks/main.yml

@@ -20,18 +20,19 @@
 
 # Install ssh keys for ubuntu account to be able to check out from mitx
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
+# {{ secure_dir }} is relative to the top-level playbooks dir so there is some
+# ugly relative pathing here
+
 - name: install read-only ssh key for mitx repo (private)
-  copy: src={{ secure_file_dir }}/ssh_deploy_private dest=/home/ubuntu/.ssh/id_rsa force=yes owner=ubuntu group=ubuntu mode=600
-  tags:
-  - lms
-  - cms
-- name: install read-only ssh key for mitx repo (public)
-  copy: src={{ secure_file_dir }}/ssh_deploy_public dest=/home/ubuntu/.ssh/id_rsa.pub force=yes owner=ubuntu group=ubuntu mode=644
+  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=root mode=644
+  sudo: True
   tags:
   - lms
   - cms
-- name: install read-only ssh key for mitx repo (host github known)
-  copy: src={{ secure_file_dir }}/ssh_deploy_known_hosts dest=/home/ubuntu/.ssh/known_hosts force=yes owner=ubuntu group=ubuntu mode=600
+
+- name: upload ssh script
+  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=root mode=755
+  sudo: True
   tags:
   - lms
   - cms
@@ -39,7 +40,6 @@
 # Check out mitx repo to $app_base_dir
 - name: set permissions on $app_base_dir sgid for edx
   file: path=$app_base_dir owner=root group=edx mode=2775 state=directory
-  file: path=$app_base_dir owner=ubuntu group=edx mode=2775 state=directory
   sudo: True
   tags:
   - lms
@@ -52,6 +52,8 @@
   - cms
 - name: git checkout mitx repo into $app_base_dir
   git: dest={{app_base_dir}}/mitx repo={{lms_source_repo}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
   tags:
   - lms
   - cms

playbooks/secure_example/files/git-identity

+IDENTITY FILE FOR GIT 

playbooks/vars/secure_default/users.yml → playbooks/secure_example/vars/users.yml

@@ -10,4 +10,4 @@ admin_users:
 
 admin_keys:
 - user: joe
-  path: keys/joe.key
+  path: "{{ secure_dir }}/keys/joe.key"

playbooks/vars/secure_default/.gitignore

-# Ignore git deployment ssh keys, which should never be checked into source
-# control.
-ssh_deploy*