Merge pull request #3628 from edx/estute/move-sytemctl-checks-to-packer-scripts

temp add stop-automatic-updates to packer scripts

util/packer/jenkins_worker.json

@@ -31,12 +31,17 @@
       "mkdir {{user `playbook_remote_dir`}}"]
   }, {
     "type": "file",
+    "source": "stop-automatic-updates.sh",
+    "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh"
+  }, {
+    "type": "file",
     "source": "../../util/install/ansible-bootstrap.sh",
     "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh"
   }, {
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",
       "export CONFIGURATION_VERSION='{{user `remote_branch`}}'",
+      "sudo bash ./stop-automatic-updates.sh",
       "sudo bash ./ansible-bootstrap.sh" ]
   }, {
     "type": "ansible-local",

util/packer/jenkins_worker_android.json

@@ -30,12 +30,17 @@
       "mkdir {{user `playbook_remote_dir`}}"]
   }, {
     "type": "file",
+    "source": "stop-automatic-updates.sh",
+    "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh"
+  }, {
+    "type": "file",
     "source": "../../util/install/ansible-bootstrap.sh",
     "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh"
   }, {
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",
       "export CONFIGURATION_VERSION='{{user `remote_branch`}}'",
+      "sudo bash ./stop-automatic-updates.sh",
       "sudo bash ./ansible-bootstrap.sh" ]
   }, {
     "type": "ansible-local",

util/packer/jenkins_worker_simple.json

@@ -31,12 +31,17 @@
       "mkdir {{user `playbook_remote_dir`}}"]
   }, {
     "type": "file",
+    "source": "stop-automatic-updates.sh",
+    "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh"
+  }, {
+    "type": "file",
     "source": "../../util/install/ansible-bootstrap.sh",
     "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh"
   }, {
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",
       "export CONFIGURATION_VERSION='{{user `remote_branch`}}'",
+      "sudo bash ./stop-automatic-updates.sh",
       "sudo bash ./ansible-bootstrap.sh" ]
   }, {
     "type": "ansible-local",

util/packer/jenkins_worker_sitespeedio.json

@@ -30,12 +30,17 @@
       "mkdir {{user `playbook_remote_dir`}}"]
   }, {
     "type": "file",
+    "source": "stop-automatic-updates.sh",
+    "destination": "{{user `playbook_remote_dir`}}/stop-automatic-updates.sh"
+  }, {
+    "type": "file",
     "source": "../../util/install/ansible-bootstrap.sh",
     "destination": "{{user `playbook_remote_dir`}}/ansible-bootstrap.sh"
   }, {
     "type": "shell",
     "inline": ["cd {{user `playbook_remote_dir`}}",
       "export CONFIGURATION_VERSION='{{user `remote_branch`}}'",
+      "sudo bash ./stop-automatic-updates.sh",
       "sudo bash ./ansible-bootstrap.sh" ]
   }, {
     "type": "ansible-local",

util/packer/stop-automatic-updates.sh

+#!/usr/bin/env bash
+
+# Kill the apt services holding a dpkg lock, so that the ansible-bootstrap
+# script can run without conflicts.
+
+# NOTE: this is a temporary fix. Instead, we should be doing what devops does,
+# and first run the security+common roles on a vanilla AMI, which will disable
+# unattended-updates and set up users. Then we can feel free to run the
+# ansible bootstrap without any problems.
+
+set -xe
+
+if grep -q 'Xenial Xerus' /etc/os-release; then
+    systemctl stop apt-daily.service
+    systemctl kill --kill-who=all apt-daily.service
+    # Our jenkins job for building AMIs will timeout, even if the lock is
+    # never released.
+    while lsof |grep -q /var/lib/dpkg/lock; do
+        echo "Waiting for apt to release the dpkg lock..."
+        sleep 5
+    done
+fi
+