Merge pull request #2642 from edx/max/package-upgrades

Add script & playbook for reporting on package upgrades

playbooks/check_package_upgrades.yml

+# Spawn an instance from an AMI and then report if any packages need to be upgraded
+#
+# Usage:
+#   ansible-playbook check_package_upgrades.yml -i localhost, -e 'packages="PKG1 PKG2 ..."' -e 'ami=ami-xxxxxxxx'
+#     -e 'key_name=KEY' -e 'security_group=sg-xxxxxxxx' -e 'subnet_id=subnet-xxxxxxxx'
+#
+# Required arguments:
+#   -e 'packages="PKG1 ...": space-separated list of packages to check
+#   -e 'ami=ami-xxxxxxxx': AMI ID to use for the instance
+#   -e 'key_name=KEY': private ssh key to use for the instance
+#   -e 'security_group=sg-xxxxxxxx': security group to use for the instance
+#   -e subnet_id=subnet-xxxxxxxx': subnet to use for the instance
+#
+# Relevant optional arguments:
+#  -e 'script_path=PATH': path to the apt_check_upgrades.py script
+#  -e 'report_dest_path=PATH': path to which the resulting report will be written.
+#  --private-key=PATH_TO_PRIVATE_KEY_FILE: ssh key to use when connecting to the new host
+#  -e 'key_name': AWS key to use for the new instance. This key must be available locally
+#                 either as an ssh profile or as specified with the above option.
+#  -e 'profile=PROFILE': AWS profile to use for AWS API calls
+#  -e 'region=REGION': AWS region to make the instance in
+#  -e 'security_group_id=sg-xxxxxxxx': security group to attach to the new instance
+#  -e 'subnet_id=subnet-xxxxxxxx': subnet to make the new instance in
+#  -e 'instance_type=INSTANCE.TYPE': instance type to use
+
+#Get an AMI ID from an E-D-P:
+#edc=CHANGEME
+#lconfig=$(aws autoscaling describe-auto-scaling-groups |
+#  jq -r ".AutoScalingGroups[] | select(.Tags[] | select(.Key == \"Name\").Value == \"$edc\").LaunchConfigurationName")
+#if [ $(echo $lconfig | wc -l) -ne 1 ]; then
+#  echo "More than 1 ASG found for E-D-P: $edp"
+#  exit 1
+#else
+#  ami=$(aws autoscaling describe-launch-configurations --launch-configuration-names $lconfig |
+#    jq -r '.LaunchConfigurations[].ImageId')
+#fi
+
+- name: Launch instance for checking packages
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    ami: !!null
+    profile: !!null
+    security_group_id: !!null
+    subnet_id: !!null
+    key_name: !!null
+    region: us-east-1
+    instance_type: t2.large
+
+  tasks:
+    - name: Launch instance
+      ec2:
+        image: "{{ ami }}"
+        instance_type: "{{ instance_type }}"
+        profile: "{{ profile }}"
+        region: "{{ region }}"
+        group_id: "{{ security_group_id }}"
+        vpc_subnet_id: "{{ subnet_id }}"
+        key_name: "{{ key_name }}"
+        volumes:
+          - device_name: /dev/sda1
+            delete_on_termination: true
+            volume_size: 50
+        instance_tags:
+          Name: temp-package-checker
+        wait: yes
+      register: instance
+
+    - name: Wait for instance to be ready
+      wait_for:
+        host: "{{ instance.instances.0.private_ip }}"
+        port: 22
+
+    - name: Add new instance to host group
+      add_host:
+        hostname: "{{ instance.instances.0.private_ip }}"
+        id: "{{ instance.instances.0.id }}"
+        groups: instance_group
+        ansible_ssh_user: ubuntu
+
+
+- name: Check for package upgrades
+  hosts: instance_group
+  become: true
+  vars:
+    packages: !!null
+    script_path: ./apt_check_upgrades.py
+    report_dest_path: .
+  tasks:
+    - name: Update apt cache
+      apt:
+        update_cache: yes
+
+    - name: Install pyyaml to allow for yaml script output
+      pip:
+        name: pyyaml
+        state: present
+
+    - name: Transfer package-checking script
+      copy:
+        src: "{{ script_path }}"
+        dest: /tmp/apt_check_upgrades.py
+        mode: 0700
+
+    - name: Run package-checking script
+      shell: /tmp/apt_check_upgrades.py -y {{ packages }} > /tmp/upgrade_results.yml
+
+    - name: Retrieve results
+      fetch:
+        src: /tmp/upgrade_results.yml
+        dest: "{{ report_dest_path}}"
+        flat: true
+
+- name: Clean up instance
+  hosts: localhost
+  connection: local
+  vars:
+    region: us-east-1
+  tasks:
+    - name: Terminate instance
+      ec2:
+        state: absent
+        instance_ids: "{{ hostvars[groups.instance_group.0].id }}"
+        region: "{{ region }}"

util/jenkins/apt_check_upgrades.py

+#!/usr/bin/env python
+import apt
+import argparse
+
+def parse_args():
+  description = "Print machine-readable output detailing available upgrades for specified packages"
+  parser = argparse.ArgumentParser(description=description)
+  parser.add_argument('-y', '--yaml', action='store_true', 
+    help="change output format to YAML (NB: requires pyyaml) [default: JSON]")
+  parser.add_argument('-a', '--all', action='store_true', help="check all packages for upgrade")
+  parser.add_argument('packages', nargs='*', help="packages to check for upgrade")
+  
+  args = parser.parse_args()
+  
+  if args.yaml:
+    import yaml
+    global yaml
+  else:
+    import json
+    global json
+  
+  return args
+
+
+def check_upgrade(pkg):
+  if pkg.installed != pkg.candidate:
+    return {
+      'current_version': pkg.installed.version,
+      'new_version': pkg.candidate.version,
+      'summary': pkg.candidate.summary,
+      'current_md5': pkg.installed.md5,
+      'new_md5': pkg.candidate.md5,
+      'homepage': pkg.candidate.homepage,
+    }
+
+
+def main(args):
+  need_upgrade = {}
+  cache = apt.Cache()
+  
+  if args.all:
+    for pkg in cache:
+      if pkg.is_installed:
+        result = check_upgrade(pkg)
+        if result:
+          need_upgrade[pkg.name] = result
+  else:
+    for pkg_name in args.packages:
+      if pkg_name not in cache:
+        raise Exception('no package named "{}" exists in the cache!'.format(pkg_name))
+      result = check_upgrade(cache[pkg_name])
+      if result:
+        need_upgrade[pkg_name] = result
+
+  if need_upgrade:
+    if args.yaml:
+      output = yaml.dump(need_upgrade, default_flow_style=False)
+    else:
+      output = json.dumps(need_upgrade)
+    print output
+
+
+
+if __name__ == '__main__':
+  main(parse_args())