Skip to content

C
configuration
Commit 1cfe1ac0 by Jillian Vogel
Options

Create Jenkins jobs for analytics tasks using a job DSL repo. 

* defaults/main.yml: adds analytics schedule variables for configuring the AnalyticsSeedJob, with all analytics tasks enabled by default.
* templates/seed_job_template.xml: use updated analytics seed job config and description
* tasks/main.yml: generates a hashed password for the jenkins user, and adds jenkins-auth and create-seed-job tags.
* README.md: documents seed job parameters and jenkins auth change
parent 0198a560
Showing with 207 additions and 54 deletions
playbooks/roles/jenkins_analytics/defaults/main.yml
......@@ -2,11 +2,11 @@
# See README.md for variable descriptions
JENKINS_ANALYTICS_USER_PASSWORD_HASHED: $6$rAVyI.p2wXVDKk5w$y0G1MQehmHtvaPgdtbrnvAsBqYQ99g939vxrdLXtPQCh/e7GJVwbnqIKZpve8EcMLTtq.7sZwTBYV9Tdjgf1k.
JENKINS_ANALYTICS_USER_PASSWORD_PLAIN: jenkins
JENKINS_ANALYTICS_GITHUB_CREDENTIAL_ID: 'github-deploy-key'
JENKINS_ANALYTICS_CREDENTIALS:
- id: github-deploy-key
- id: "{{ JENKINS_ANALYTICS_GITHUB_CREDENTIAL_ID }}"
scope: GLOBAL
username: git
type: ssh-private-key
......@@ -16,6 +16,39 @@ JENKINS_ANALYTICS_CREDENTIALS:
JENKINS_ANALYTICS_CONCURRENT_JOBS_COUNT: 2
ANALYTICS_SCHEDULE_SECURE_REPO_URL: null
ANALYTICS_SCHEDULE_SECURE_REPO_DEST: "analytics-secure-config"
ANALYTICS_SCHEDULE_SECURE_REPO_VERSION: "master"
ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID: "{{ JENKINS_ANALYTICS_GITHUB_CREDENTIAL_ID }}"
ANALYTICS_SCHEDULE_JOBS_DSL_REPO_URL: "git@github.com:edx-ops/edx-jenkins-job-dsl.git"
ANALYTICS_SCHEDULE_JOBS_DSL_REPO_VERSION: "master"
ANALYTICS_SCHEDULE_JOBS_DSL_REPO_CREDENTIAL_ID: "{{ JENKINS_ANALYTICS_GITHUB_CREDENTIAL_ID }}"
ANALYTICS_SCHEDULE_JOBS_DSL_CLASSPATH: |
src/main/groovy
lib/*.jar
ANALYTICS_SCHEDULE_JOBS_DSL_TARGET_JOBS: "jobs/analytics-edx-jenkins.edx.org/*Jobs.groovy"
ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION: true
ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS:
SECURE_REPO: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL }}"
SECURE_BRANCH: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION }}"
ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY: true
ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY_EXTRA_VARS:
SECURE_REPO: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL }}"
SECURE_BRANCH: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION }}"
ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL: true
ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS:
SECURE_REPO: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL }}"
SECURE_BRANCH: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION }}"
ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO: true
ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS:
SECURE_REPO: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL }}"
SECURE_BRANCH: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION }}"
ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY: true
ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS:
SECURE_REPO: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL }}"
SECURE_BRANCH: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION }}"
jenkins_credentials_root: '/tmp/credentials'
jenkins_credentials_file_dest: "{{ jenkins_credentials_root }}/credentials.json"
jenkins_credentials_script: "{{ jenkins_credentials_root }}/addCredentials.groovy"
......@@ -27,16 +60,27 @@ jenkins_auth_realm:
service: su
# Change this default password: (see README.md to see how you can do it)
plain_password: "{{ JENKINS_ANALYTICS_USER_PASSWORD_PLAIN }}"
hashed_password: "{{ JENKINS_ANALYTICS_USER_PASSWORD_HASHED }}"
# For now only a single seed job is supported, adding more would require
# Ansible 2.+ or converting _execute_jenkins_cli to a module
jenkins_seed_job:
name: analytics-seed-job
time_trigger: "H * * * *"
removed_job_action: "DELETE"
removed_view_action: "IGNORE"
scm:
type: git
url: "git@github.com:edx-ops/edx-jenkins-job-dsl.git"
credential_id: "github-deploy-key"
target_jobs: "jobs/analytics-edx-jenkins.edx.org/*Jobs.groovy"
additional_classpath: "src/main/groovy"
name: AnalyticsSeedJob
multiscm:
- scm:
type: git
url: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_URL }}"
dest: ""
branch: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_VERSION | default('master') }}"
credential_id: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_CREDENTIAL_ID | default('') }}"
- scm:
type: git
url: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL | default('') }}"
dest: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST | default('') }}"
branch: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION | default('master') }}"
credential_id: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID | default('') }}"
dsl:
gradle_tasks: "clean libs test"
removed_view_action: IGNORE
removed_job_action: DELETE
additional_classpath: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_CLASSPATH }}"
target_jobs: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_TARGET_JOBS }}"
playbooks/roles/jenkins_analytics/tasks/main.yml
......@@ -2,16 +2,29 @@
- fail: msg=included unix realm by accident
when: jenkins_auth_realm.name != "unix"
tags:
- jenkins-auth
- fail: msg=Please change default password for jenkins user
when: jenkins_auth_realm.plain_password == 'jenkins'
tags:
- jenkins-auth
- name: generate jenkins user password
shell: "openssl passwd -1 '{{ jenkins_auth_realm.plain_password | quote }}'"
register: jenkins_user_password_hash
no_log: True
tags:
- jenkins-auth
- user:
name: "{{ jenkins_user }}"
state: present
groups: shadow
append: yes
password: "{{ jenkins_auth_realm.hashed_password }}"
update_password: always
password: "{{ jenkins_user_password_hash.stdout }}"
tags:
- jenkins-auth
- name: template config.xml
template:
......@@ -19,6 +32,8 @@
dest: "{{ jenkins_home }}/config.xml"
owner: "{{ jenkins_user }}"
group: "{{ jenkins_group }}"
tags:
- jenkins-auth
# Unconditionally restart Jenkins, this has two side-effects:
# 1. Jenkins uses new auth realm
......@@ -28,17 +43,23 @@
- name: restart Jenkins
service: name=jenkins state=restarted
tags:
- jenkins-auth
# Upload Jenkins credentials
- name: create credentials dir
file: name={{ jenkins_credentials_root }} state=directory
tags:
- jenkins-auth
- name: upload groovy script
template:
src: addCredentials.groovy
dest: "{{ jenkins_credentials_script }}"
mode: "600"
tags:
- jenkins-auth
- name: upload credentials file
template:
......@@ -46,48 +67,66 @@
dest: "{{ jenkins_credentials_file_dest }}"
mode: "600"
owner: "{{ jenkins_user }}"
tags:
- jenkins-auth
- name: add credentials
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "groovy {{ jenkins_credentials_script }}"
tags:
- jenkins-auth
- name: clean up
file: name={{ jenkins_credentials_root }} state=absent
tags:
- jenkins-auth
# Upload seed job
- name: upload job file
template: src=seed_job_template.xml dest=/tmp/{{ jenkins_seed_job.name }} mode="600"
template: src=seed_job_template.xml dest=/tmp/{{ jenkins_seed_job.name }} mode="600"
tags:
- create-seed-job
- name: check if job is present
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "get-job {{ jenkins_seed_job.name }}"
jenkins_command_string: "get-job {{ jenkins_seed_job.name }}"
jenkins_ignore_cli_errors: yes
tags:
- create-seed-job
- set_fact:
get_job_output: "{{ jenkins_command_output }}"
tags:
- create-seed-job
# Upload seed job to Jenkins
- name: Create seed job if absent
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "create-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
jenkins_command_string: "create-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
when: get_job_output.rc != 0
tags:
- create-seed-job
- name: update seed job
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "update-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
jenkins_command_string: "update-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
when: get_job_output.rc == 0
tags:
- create-seed-job
# Build the seed job
- name: Build the seed job
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "build {{ jenkins_seed_job.name }} -s"
jenkins_command_string: "build {{ jenkins_seed_job.name }} -s"
tags:
- create-seed-job
playbooks/roles/jenkins_analytics/templates/seed_job_template.xml
......@@ -2,7 +2,7 @@
<project>
<actions/>
<description>
Seed job autogenerated by ansible, it will be overridden.
Creates and configures the analytics task jobs.
</description>
<keepDependencies>false</keepDependencies>
<properties>
......@@ -10,44 +10,114 @@
<useJobPriority>false</useJobPriority>
<priority>-1</priority>
</jenkins.advancedqueue.AdvancedQueueSorterJobProperty>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<hudson.model.StringParameterDefinition>
<name>GIT_CREDENTIAL_ID</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID | default('') }}</defaultValue>
<description>Jenkins Credential ID used for cloning secure git
repos. Must match a Credential configured in Jenkins.</description>
</hudson.model.StringParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>ANSWER_DISTRIBUTION</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION | default(true) }}</defaultValue>
<description>Create or update the AnswerDistributionWorkflow analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.TextParameterDefinition>
<name>ANSWER_DISTRIBUTION_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS | default('{}') | to_nice_json }}</defaultValue>
<description>Set default values for the AnswerDistributionWorkflow job parameters. Format as YAML.</description>
</hudson.model.TextParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>COURSE_ACTIVITY_WEEKLY</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY | default(true) }}</defaultValue>
<description>Create or update the CourseActivityWeeklyTask analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.TextParameterDefinition>
<name>COURSE_ACTIVITY_WEEKLY_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY_EXTRA_VARS | default('{}') | to_nice_json }}</defaultValue>
<description>Set default values for the CourseActivityWeeklyTask job parameters. Format as YAML.</description>
</hudson.model.TextParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>IMPORT_ENROLLMENTS_INTO_MYSQL</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL | default(true) }}</defaultValue>
<description>Create or update the ImportEnrollmentsIntoMysql analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.TextParameterDefinition>
<name>IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS | default('{}') | to_nice_json }}</defaultValue>
<description>Set default values for the ImportEnrollmentsIntoMysql job parameters. Format as YAML.</description>
</hudson.model.TextParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>INSERT_TO_MYSQL_ALL_VIDEO</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO | default(true) }}</defaultValue>
<description>Create or update the InsertToMysqlAllVideoTask analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.TextParameterDefinition>
<name>INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS | default('{}') | to_nice_json }}</defaultValue>
<description>Set default values for the InsertToMysqlAllVideoTask job parameters. Format as YAML.</description>
</hudson.model.TextParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY | default(true) }}</defaultValue>
<description>Create or update the InsertToMysqlCourseEnrollByCountryWorkflow analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.TextParameterDefinition>
<name>INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS | default('{}') | to_nice_json }}</defaultValue>
<description>Set default values for the InsertToMysqlCourseEnrollByCountryWorkflowJob job parameters. Format as YAML.</description>
</hudson.model.TextParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
</properties>
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.4.0">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>{{ jenkins_seed_job.scm.url}}</url>
{% if jenkins_seed_job.scm.credential_id is defined and jenkins_seed_job.scm.credential_id %}
<credentialsId>{{ jenkins_seed_job.scm.credential_id }}</credentialsId>
<scm class='org.jenkinsci.plugins.multiplescms.MultiSCM'>
<scms>
{% for scm in jenkins_seed_job.multiscm %}
{% if scm.url %}
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.4.0">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>{{ scm.url }}</url>
{% if scm.credential_id %}
<credentialsId>{{ scm.credential_id }}</credentialsId>
{% endif %}
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>{{ scm.branch }}</name>
</hudson.plugins.git.BranchSpec>
</branches>
{% if scm.dest %}
<relativeTargetDir>{{ scm.dest }}</relativeTargetDir>
{% endif %}
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>master</name>
</hudson.plugins.git.BranchSpec>
</branches>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<browser class="hudson.plugins.git.browser.AssemblaWeb">
<url></url>
</browser>
<submoduleCfg class="list"/>
<extensions/>
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<browser class="hudson.plugins.git.browser.AssemblaWeb">
<url></url>
</browser>
<submoduleCfg class="list"/>
<extensions/>
</scm>
{% endif %}
{% endfor %}
</scms>
</scm>
<canRoam>true</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers>
<hudson.triggers.TimerTrigger>
<spec>{{ jenkins_seed_job.time_trigger }}</spec>
</hudson.triggers.TimerTrigger>
</triggers>
<concurrentBuild>false</concurrentBuild>
<builders>
<hudson.plugins.gradle.Gradle plugin="gradle@1.24">
<description></description>
<switches></switches>
<tasks>clean test</tasks>
<tasks>{{ jenkins_seed_job.dsl.gradle_tasks }}</tasks>
<rootBuildScriptDir></rootBuildScriptDir>
<buildFile></buildFile>
<gradleName>(x)</gradleName>
......@@ -57,13 +127,13 @@
<useWorkspaceAsHome>false</useWorkspaceAsHome>
</hudson.plugins.gradle.Gradle>
<javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.43">
<targets>{{ jenkins_seed_job.scm.target_jobs }}</targets>
<targets>{{ jenkins_seed_job.dsl.target_jobs }}</targets>
<usingScriptText>false</usingScriptText>
<ignoreExisting>false</ignoreExisting>
<removedJobAction>{{ jenkins_seed_job.removed_job_action }}</removedJobAction>
<removedViewAction>{{ jenkins_seed_job.removed_view_action }}</removedViewAction>
<lookupStrategy>JENKINS_ROOT</lookupStrategy>
<additionalClasspath>{{ jenkins_seed_job.scm.additional_classpath }}</additionalClasspath>
<removedJobAction>{{ jenkins_seed_job.dsl.removed_job_action }}</removedJobAction>
<removedViewAction>{{ jenkins_seed_job.dsl.removed_view_action }}</removedViewAction>
<lookupStrategy>SEED_JOB</lookupStrategy>
<additionalClasspath>{{ jenkins_seed_job.dsl.additional_classpath }}</additionalClasspath>
</javaposse.jobdsl.plugin.ExecuteDslScripts>
</builders>
<publishers/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment