tasks for pipeline celery workers

playbooks/roles/video_pipeline/defaults/main.yml

@@ -21,11 +21,15 @@ video_pipeline_gunicorn_port: 8555
 video_pipeline_user: "{{ video_pipeline_service_name }}"
 video_pipeline_home: "{{ COMMON_APP_DIR }}/{{ video_pipeline_service_name }}"
 
-# This should be removed once instance_config.yml is moved out side of source code.
-video_pipeline_code_dir: '{{ COMMON_APP_DIR }}/{{ video_pipeline_service_name }}/{{ video_pipeline_service_name }}'
+# TODO! Find some better way to do the below
+video_pipeline_service_home: "{{ COMMON_APP_DIR }}/{{ video_pipeline_service_name }}"
+video_pipeline_code_dir: "{{ video_pipeline_service_home }}/{{ video_pipeline_service_name }}"
 
 video_pipeline_environment: {}
 
+video_pipeline_ingest_worker: "{{ video_pipeline_service_name }}_ingest"
+video_pipeline_deliver_worker: "{{ video_pipeline_service_name }}_deliver"
+
 #
 # OS packages
 #
@@ -37,39 +41,39 @@ video_pipeline_debian_pkgs:
 VIDEO_PIPELINE_NGINX_PORT: "1{{ video_pipeline_gunicorn_port }}"
 VIDEO_PIPELINE_SSL_NGINX_PORT: "4{{ video_pipeline_gunicorn_port }}"
 
-VIDEO_PIPELINE_DEFAULT_DB_NAME: 'pipeline'
-VIDEO_PIPELINE_MYSQL: 'localhost'
+VIDEO_PIPELINE_DEFAULT_DB_NAME: "pipeline"
+VIDEO_PIPELINE_MYSQL: "localhost"
 # MySQL usernames are limited to 16 characters
-VIDEO_PIPELINE_MYSQL_USER: 'pipeline001'
-VIDEO_PIPELINE_MYSQL_PASSWORD: 'password'
+VIDEO_PIPELINE_MYSQL_USER: "pipeline001"
+VIDEO_PIPELINE_MYSQL_PASSWORD: "password"
 
 VIDEO_PIPELINE_MEMCACHE: []
 
 VIDEO_PIPELINE_VERSION: "master"
 VIDEO_PIPELINE_DJANGO_SETTINGS_MODULE: "VEDA.settings"
-VIDEO_PIPELINE_URL_ROOT: 'http://pipeline:{{ VIDEO_PIPELINE_NGINX_PORT }}'
-VIDEO_PIPELINE_LOGOUT_URL: '{{ VIDEO_PIPELINE_URL_ROOT }}/logout/'
+VIDEO_PIPELINE_URL_ROOT: "http://pipeline:{{ VIDEO_PIPELINE_NGINX_PORT }}"
+VIDEO_PIPELINE_LOGOUT_URL: "{{ VIDEO_PIPELINE_URL_ROOT }}/logout/"
 
-VIDEO_PIPELINE_SECRET_KEY: 'Your secret key here'
-VIDEO_PIPELINE_LANGUAGE_CODE: 'en-us'
+VIDEO_PIPELINE_SECRET_KEY: "Your secret key here"
+VIDEO_PIPELINE_LANGUAGE_CODE: "en-us"
 
 # Used to automatically configure OAuth2 Client
-VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_KEY : 'pipeline-key'
-VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_SECRET : 'pipeline-secret'
+VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_KEY : "pipeline-key"
+VIDEO_PIPELINE_SOCIAL_AUTH_EDX_OIDC_SECRET : "pipeline-secret"
 VIDEO_PIPELINE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: false
 
-VIDEO_PIPELINE_DATA_DIR: '{{ COMMON_DATA_DIR }}/{{ video_pipeline_service_name }}'
-VIDEO_PIPELINE_MEDIA_ROOT: '{{ VIDEO_PIPELINE_DATA_DIR }}/media'
-VIDEO_PIPELINE_MEDIA_URL: '/media/'
+VIDEO_PIPELINE_DATA_DIR: "{{ COMMON_DATA_DIR }}/{{ video_pipeline_service_name }}"
+VIDEO_PIPELINE_MEDIA_ROOT: "{{ VIDEO_PIPELINE_DATA_DIR }}/media"
+VIDEO_PIPELINE_MEDIA_URL: "/media/"
 
 VIDEO_PIPELINE_MEDIA_STORAGE_BACKEND:
-  DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
-  MEDIA_ROOT: '{{ VIDEO_PIPELINE_MEDIA_ROOT }}'
-  MEDIA_URL: '{{ VIDEO_PIPELINE_MEDIA_URL }}'
+  DEFAULT_FILE_STORAGE: "django.core.files.storage.FileSystemStorage"
+  MEDIA_ROOT: "{{ VIDEO_PIPELINE_MEDIA_ROOT }}"
+  MEDIA_URL: "{{ VIDEO_PIPELINE_MEDIA_URL }}"
 
-VIDEO_PIPELINE_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage'
+VIDEO_PIPELINE_STATICFILES_STORAGE: "django.contrib.staticfiles.storage.StaticFilesStorage"
 
-VIDEO_PIPELINE_GUNICORN_EXTRA: ''
+VIDEO_PIPELINE_GUNICORN_EXTRA: ""
 
 VIDEO_PIPELINE_EXTRA_APPS: []
 
@@ -77,43 +81,53 @@ VIDEO_PIPELINE_EXTRA_APPS: []
 VIDEO_PIPELINE_AUTOMATED_USERS: {}
 
 # instance_config.yaml variable overrides
-
-VIDEO_PIPELINE_EDX_S3_INGEST_PREFIX: 'video_pipeline_edx_s3_ingest_prefix'
-VIDEO_PIPELINE_EDX_S3_INGEST_BUCKET: 'video_pipeline_edx_s3_ingest_bucket'
-VIDEO_PIPELINE_EDX_S3_ENDPOINT_BUCKET: 'video_pipeline_edx_s3_endpoint_bucket'
-VIDEO_PIPELINE_EDX_CLOUDFRONT_PREFIX: 'video_pipeline_edx_cloudfront_prefix'
-VIDEO_PIPELINE_AWS_VIDEO_IMAGES_BUCKET: 'video_pipeline_aws_video_images_bucket'
-VIDEO_PIPELINE_VEDA_S3_UPLOAD_BUCKET: 'video_pipeline_veda_s3_upload_bucket'
-VIDEO_PIPELINE_VEDA_S3_HOTSTORE_BUCKET: 'video_pipeline_veda_s3_hotstore_bucket'
-VIDEO_PIPELINE_VEDA_DELIVERABLE_BUCKET: 'video_pipeline_veda_deliverable_bucket'
-VIDEO_PIPELINE_VEDA_BASE_URL: 'video_pipeline_veda_base_url'
-VIDEO_PIPELINE_AWS_VIDEO_TRANSCRIPTS_BUCKET: 'video_pipeline_aws_video_transcripts_bucket'
-VIDEO_PIPELINE_CIELO24_API_ENVIRONMENT: 'video_pipeline_cielo24_api_environment'
-VIDEO_PIPELINE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN: 'video_pipeline_transcript_provider_request_token'
-VIDEO_PIPELINE_VEDA_SECRET_ACCESS_KEY: 'video_pipeline_veda_secret_access_key'
-VIDEO_PIPELINE_VEDA_ACCESS_KEY_ID: 'video_pipeline_veda_access_key_id'
-VIDEO_PIPELINE_VEDA_NOREPLY_EMAIL: 'video_pipeline_veda_noreply_email'
-VIDEO_PIPELINE_ADMIN_EMAIL: 'video_pipeline_admin_email'
-VIDEO_PIPELINE_VEDA_API_URL: 'video_pipeline_veda_api_url'
-VIDEO_PIPELINE_VEDA_AUTH_URL: 'video_pipeline_veda_auth_url'
-VIDEO_PIPELINE_VEDA_CLIENT_ID: 'video_pipeline_veda_client_id'
-VIDEO_PIPELINE_VEDA_SECRET_KEY: 'video_pipeline_veda_secret_key'
-VIDEO_PIPELINE_VEDA_TOKEN_URL: 'video_pipeline_veda_token_url'
-VIDEO_PIPELINE_VAL_API_URL: 'video_pipeline_val_api_url'
-VIDEO_PIPELINE_VAL_TOKEN_URL: 'video_pipeline_val_token_url'
-VIDEO_PIPELINE_VAL_VIDEO_IMAGES_URL: 'video_pipeline_val_video_images_url'
-VIDEO_PIPELINE_VAL_CLIENT_ID: 'video_pipeline_val_client_id'
-VIDEO_PIPELINE_VAL_SECRET_KEY: 'video_pipeline_val_secret_key'
-VIDEO_PIPELINE_VAL_PASSWORD: 'video_pipeline_val_password'
-VIDEO_PIPELINE_VAL_USERNAME: 'video_pipeline_val_username'
-VIDEO_PIPELINE_VAL_TRANSCRIPT_CREATE_URL: 'video_pipeline_val_transcript_create_url'
-VIDEO_PIPELINE_VAL_VIDEO_TRANSCRIPT_STATUS_URL: 'video_pipeline_val_video_transcript_status_url'
-VIDEO_PIPELINE_RABBITMQ_BROKER: 'video_pipeline_rabbitmq_broker'
-VIDEO_PIPELINE_RABBITMQ_PASS: 'video_pipeline_rabbitmq_pass'
-VIDEO_PIPELINE_RABBITMQ_USER: 'video_pipeline_rabbitmq_user'
-VIDEO_PIPELINE_SG_SERVER_PATH: 'video_pipeline_sg_server_path'
-VIDEO_PIPELINE_SG_SCRIPT_NAME: 'video_pipeline_sg_script_name'
-VIDEO_PIPELINE_SG_SCRIPT_KEY: 'video_pipeline_sg_script_key'
-VIDEO_PIPELINE_THREEPLAY_FTPHOST: 'video_pipeline_threeplay_ftphost'
-VIDEO_PIPELINE_XUETANG_API_URL: 'video_pipeline_xuetang_api_url'
-VIDEO_PIPELINE_XUETANG_API_SHARED_SECRET: 'video_pipeline_xuetang_api_shared_secret'
+# TODO! add sane default values for below variables
+VIDEO_PIPELINE_EDX_S3_INGEST_PREFIX: "SET-ME-PLEASE"
+VIDEO_PIPELINE_EDX_S3_INGEST_BUCKET: "video_pipeline_edx_s3_ingest_bucket"
+VIDEO_PIPELINE_EDX_S3_ENDPOINT_BUCKET: "video_pipeline_edx_s3_endpoint_bucket"
+
+VIDEO_PIPELINE_EDX_CLOUDFRONT_PREFIX: "video_pipeline_edx_cloudfront_prefix"
+VIDEO_PIPELINE_VEDA_S3_UPLOAD_BUCKET: "video_pipeline_veda_s3_upload_bucket"
+VIDEO_PIPELINE_VEDA_S3_HOTSTORE_BUCKET: "video_pipeline_veda_s3_hotstore_bucket"
+VIDEO_PIPELINE_VEDA_DELIVERABLE_BUCKET: "video_pipeline_veda_deliverable_bucket"
+VIDEO_PIPELINE_VEDA_BASE_URL: "veda.edx.org"
+VIDEO_PIPELINE_VEDA_SECRET_ACCESS_KEY: "video_pipeline_veda_secret_access_key"
+VIDEO_PIPELINE_VEDA_ACCESS_KEY_ID: "video_pipeline_veda_access_key_id"
+VIDEO_PIPELINE_VEDA_NOREPLY_EMAIL: "video_pipeline_veda_noreply_email"
+
+VIDEO_PIPELINE_AWS_VIDEO_IMAGES_BUCKET: "video_pipeline_aws_video_images_bucket"
+VIDEO_PIPELINE_AWS_VIDEO_TRANSCRIPTS_BUCKET: "video_pipeline_aws_video_transcripts_bucket"
+
+VIDEO_PIPELINE_ADMIN_EMAIL: "video_pipeline_admin_email"
+
+VIDEO_PIPELINE_CIELO24_API_ENVIRONMENT: "video_pipeline_cielo24_api_environment"
+VIDEO_PIPELINE_TRANSCRIPT_PROVIDER_REQUEST_TOKEN: "video_pipeline_transcript_provider_request_token"
+
+VIDEO_PIPELINE_VEDA_API_URL: "video_pipeline_veda_api_url"
+VIDEO_PIPELINE_VEDA_AUTH_URL: "video_pipeline_veda_auth_url"
+VIDEO_PIPELINE_VEDA_CLIENT_ID: "video_pipeline_veda_client_id"
+VIDEO_PIPELINE_VEDA_SECRET_KEY: "video_pipeline_veda_secret_key"
+VIDEO_PIPELINE_VEDA_TOKEN_URL: "video_pipeline_veda_token_url"
+
+VIDEO_PIPELINE_VAL_API_URL: "video_pipeline_val_api_url"
+VIDEO_PIPELINE_VAL_TOKEN_URL: "video_pipeline_val_token_url"
+VIDEO_PIPELINE_VAL_VIDEO_IMAGES_URL: "video_pipeline_val_video_images_url"
+VIDEO_PIPELINE_VAL_CLIENT_ID: "video_pipeline_val_client_id"
+VIDEO_PIPELINE_VAL_SECRET_KEY: "video_pipeline_val_secret_key"
+VIDEO_PIPELINE_VAL_PASSWORD: "video_pipeline_val_password"
+VIDEO_PIPELINE_VAL_USERNAME: "video_pipeline_val_username"
+VIDEO_PIPELINE_VAL_TRANSCRIPT_CREATE_URL: "video_pipeline_val_transcript_create_url"
+VIDEO_PIPELINE_VAL_VIDEO_TRANSCRIPT_STATUS_URL: "video_pipeline_val_video_transcript_status_url"
+
+VIDEO_PIPELINE_RABBITMQ_BROKER: "localhost"
+VIDEO_PIPELINE_RABBITMQ_PASS: "veda"
+VIDEO_PIPELINE_RABBITMQ_USER: "veda"
+
+VIDEO_PIPELINE_SG_SERVER_PATH: "video_pipeline_sg_server_path"
+VIDEO_PIPELINE_SG_SCRIPT_NAME: "video_pipeline_sg_script_name"
+VIDEO_PIPELINE_SG_SCRIPT_KEY: "video_pipeline_sg_script_key"
+
+VIDEO_PIPELINE_THREEPLAY_FTPHOST: "video_pipeline_threeplay_ftphost"
+
+VIDEO_PIPELINE_XUETANG_API_URL: "video_pipeline_xuetang_api_url"
+VIDEO_PIPELINE_XUETANG_API_SHARED_SECRET: "video_pipeline_xuetang_api_shared_secret"

playbooks/roles/video_pipeline/tasks/main.yml

@@ -25,3 +25,42 @@
   template:
     src: "templates/instance_config.yaml.j2"
     dest: "{{ video_pipeline_code_dir }}/instance_config.yaml"
+
+- name: write out the supervisor wrapper for celery workers
+  template:
+    src: "templates/celery_worker.sh.j2"
+    dest: "{{ video_pipeline_service_home }}/{{ item.worker_script_wrapper_name }}.sh"
+    mode: 0650
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+  vars:
+    worker_script: "{{ item.worker_script_src_path }}"
+  with_items:
+    - { worker_script_src_path: 'bin/ingest', worker_script_wrapper_name:  "{{ video_pipeline_ingest_worker }}" }
+    - { worker_script_src_path: 'bin/deliver', worker_script_wrapper_name:  "{{ video_pipeline_deliver_worker }}" }
+
+- name: write supervisord config for celery workers
+  template:
+    src: "templates/supervisor.conf.j2"
+    dest: "{{ supervisor_available_dir }}/{{ item }}.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+  vars:
+    worker_script: "{{ item }}"
+  with_items:
+    - "{{ video_pipeline_ingest_worker }}"
+    - "{{ video_pipeline_deliver_worker }}"
+
+- name: enable supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/{{ item }}.conf"
+    dest: "{{ supervisor_cfg_dir }}/{{ item }}.conf"
+    state: link
+    force: yes
+  with_items:
+    - "{{ video_pipeline_ingest_worker }}"
+    - "{{ video_pipeline_deliver_worker }}"
+
+- name: update supervisor configuration
+  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"

playbooks/roles/video_pipeline/templates/celery_worker.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source {{ video_pipeline_service_home }}/{{ video_pipeline_service_name }}_env
+python {{ video_pipeline_service_home }}/{{ video_pipeline_service_name }}/{{ worker_script }}

playbooks/roles/video_pipeline/templates/supervisor.conf.j2

+#
+# {{ ansible_managed }}
+#
+
+[program:{{ worker_script }}]
+command={{ video_pipeline_service_home }}/{{ worker_script }}.sh
+user={{ common_web_user }}
+directory={{ video_pipeline_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true
+

vagrant/base/test_playbook/Vagrantfile

@@ -2,26 +2,33 @@
 # under /playbooks (e.g., "foo.yml"), export its name (e.g., "foo") as the value
 # of the environment variable VAGRANT_ANSIBLE_PLAYBOOK, and execute `vagrant up`
 # from within this directory.
-MEMORY = 2048
+MEMORY = 1024
 CPU_COUNT = 2
 
 Vagrant.configure("2") do |config|
 
-  config.vm.box     = "xenial64"
-  config.vm.box_url = "http://files.vagrantup.com/xenial64.box"
+  config.vm.box     = "ubuntu_xenial"
 
   config.vm.network :private_network, ip: "192.168.33.20"
-  config.vm.network :forwarded_port, guest: 8080, host: 8080
+  config.vm.network :forwarded_port, guest: 8555, host: 8555
+
+  config.ssh.insert_key = true
 
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
 
     # You can adjust this to the amount of CPUs your system has available
     vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
+    vb.customize ['modifyvm', :id, '--nictype1', 'virtio']
   end
 
+  config.ssh.forward_x11 = true
+
   config.vm.provision :ansible do |ansible|
     ansible.playbook = "../../../playbooks/" + ENV['VAGRANT_ANSIBLE_PLAYBOOK'] + ".yml"
-    ansible.verbose = "extra"
+    ansible.verbose = "vvvv"
+    ansible.extra_vars = {
+        edx_django_service_version: 'ammar/changes-for-config'
+    }
   end
 end