updating py file for repo sync

playbooks/roles/gh_mirror/defaults/main.yml

@@ -11,9 +11,8 @@ gh_mirror_orgs:
   - eventbrite
   - dementrock
   - mfogel
-gh_mirror_debian_pkgs:
-  - python-yaml
-  - python-requests
+gh_mirror_pip_pkgs:
+  - pyyaml
+  - requests
 gh_mirror_app_files:
-  - gh_mirror_cron.sh
   - repos_from_orgs.py

playbooks/roles/gh_mirror/files/gh_mirror_cron.sh

-#!/usr/bin/env bash
-
-if [[ -z $1 ]]; then
-    echo "Path to data directory required"
-    exit 1
-fi
-
-data_dir=$1
-dir=$(dirname $0)
-
-if [[ ! -f /var/tmp/repos.txt ]]; then
-    python $dir/repos_from_orgs.py
-fi
-
-for repo_url in $(cat /var/tmp/repos.txt); do 
-
-    repo_name=${repo_url##*/}
-    if [[ ! -d $data_dir/$repo_name ]]; then
-        git clone --mirror $repo_url $data_dir/$repo_name
-        cd $data_dir/$repo_name
-        git update-server-info
-    else
-        cd $data_dir/$repo_name
-        git remote update
-        git update-server-info
-    fi
-done

playbooks/roles/gh_mirror/files/repos_from_orgs.py

 #!/usr/bin/python
-# Generates /var/tmp/repos.txt from
+# Generates /var/tmp/repos.json from
 # a yaml file containing a list of
 # github organizations
 
 import yaml
 import sys
 import requests
+import json
+import subprocess
+import os
+import logging
 from os.path import dirname, abspath, join
+from argparse import ArgumentParser
 
-path = dirname(abspath(__file__))
 
-try:
-    with open(join(path, 'orgs.yml')) as f:
-        orgs = yaml.load(f)
-except IOError:
-    print "Unable to read {}/orgs.yml, does it exist?".format(path)
-    sys.exit(1)
+def run_cmd(cmd):
+    logging.debug('running: {}\n'.format(cmd))
+    process = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+        shell=True)
+    for line in iter(process.stdout.readline, ""):
+        logging.debug(line)
 
-repos = []
 
-for org in orgs:
-    r = requests.get('https://api.github.com/orgs/{}/repos'.format(org))
-    org_data = r.json
-    for repo_data in org_data:
-        repos.append(repo_data['html_url'])
+def parse_args():
+    parser = ArgumentParser()
+    parser.add_argument('-r', '--refresh', action='store_true',
+                        help="Refresh the list of repos", default=False)
+    parser.add_argument('-d', '--datadir', help="repo directory")
+    return parser.parse_args()
 
-with open('/var/tmp/repos.txt', 'wb') as f:
-    f.write('\n'.join(repos))
+
+def refresh_cache():
+    path = dirname(abspath(__file__))
+    try:
+        with open(join(path, 'orgs.yml')) as f:
+            orgs = yaml.load(f)
+    except IOError:
+        print "Unable to read {}/orgs.yml, does it exist?".format(path)
+        sys.exit(1)
+
+    repos = []
+
+    for org in orgs:
+        r = requests.get('https://api.github.com/orgs/{}/repos'.format(org))
+        org_data = r.json()
+        for repo_data in org_data:
+            if 'html_url' in repo_data:
+                repos.append({'html_url': repo_data['html_url'],
+                              'name': repo_data['name'],
+                              'org': repo_data['owner']['login']})
+
+    with open('/var/tmp/repos.json', 'wb') as f:
+        f.write(json.dumps(repos))
+
+
+def update_repos():
+    with open('/var/tmp/repos.json') as f:
+        repos = json.load(f)
+    for repo in repos:
+        repo_path = os.path.join(args.datadir, repo['org'], repo['name'])
+        if not os.path.exists(repo_path):
+            run_cmd('mkdir -p {}'.format(repo_path))
+            run_cmd('git clone --mirror {} {}'.format(repo['html_url'], repo_path))
+            run_cmd('cd {} && git update-server-info'.format(repo_path))
+        else:
+            run_cmd('cd {} && git remote-update'.format(repo_path))
+            run_cmd('cd {} && git update-server-info'.format(repo_path))
+
+if __name__ == '__main__':
+    logging.basicConfig(filename='/var/log/repos-from-orgs.log',
+                        level=logging.DEBUG)
+    args = parse_args()
+    if args.refresh:
+        refresh_cache()
+    else:
+        if not args.datadir:
+            print "Please specificy a repository directory"
+            sys.exit(1)
+        if not os.path.exists('/var/tmp/repos.json'):
+            refresh_cache()
+        update_repos()

playbooks/roles/gh_mirror/tasks/main.yml

 ---
-- name: gh_mirror | install debian packages {{ gh_mirror_debian_pkgs }}
-  apt: pkg={{','.join(gh_mirror_debian_pkgs)}} state=present
+- name: gh_mirror | install pip packages
+  pip: name={{ item }} state=present
+  with_items: gh_mirror_pip_pkgs
 
 - name: gh_mirror | create gh_mirror user
   user: >
@@ -22,6 +23,18 @@
 - name: gh_mirror | create org config
   template: src=orgs.yml.j2 dest={{ gh_mirror_app_dir }}/orgs.yml
 
-- name: copying sync scripts {{ gh_mirror_app_files }}
+- name: copying sync scripts
   copy: src={{ item }} dest={{ gh_mirror_app_dir }}/{{ item }}
   with_items: "{{ gh_mirror_app_files }}"
+
+- name: creating cron job to update repos
+  cron:
+    name: "update repos from github"
+    job: "/usr/bin/flock -n /tmp/update-repo.lock -c '/usr/bin/python {{ gh_mirror_app_dir }}/repos_from_orgs.py -d {{ gh_mirror_data_dir }}'"
+    
+- name: creating cron to update github repo list
+  cron: 
+    name: "refresh repo list from github"
+    job: "/usr/bin/flock -n /tmp/update-github-repo.lock -c '/usr/bin/python {{ gh_mirror_app_dir}}/repos_from_orgs.py -r'"
+    minute: 0
+