Skip to content

C
configuration
Commit 34ef9de3 by Jillian Vogel
Options

Converts the seed job xml to groovy DSL, and incorporates changes from PR #2962 

* Builds the seed job xml from groovy DSL using the job-dsl-plugin's gradle run task.
  Ref https://github.com/jenkinsci/job-dsl-plugin/wiki/User-Power-Moves
* Adds COMMON_VARS text parameter
* Uses DSL_BRANCH and SECURE_BRANCH parameter values for seed job git repos
* Removes unnecessary jinja formatting from seedJob.groovy template
parent fd988d98
Showing with 147 additions and 164 deletions
playbooks/roles/jenkins_analytics/defaults/main.yml
......@@ -102,7 +102,8 @@ ANALYTICS_SCHEDULE_MASTER_SSH_CREDENTIAL_FILE: "{{ jenkins_home }}/workspace/{{
ANALYTICS_SCHEDULE_JOBS_DSL_CLASSPATH: |
src/main/groovy
lib/*.jar
ANALYTICS_SCHEDULE_JOBS_DSL_TARGET_JOBS: "jobs/analytics-edx-jenkins.edx.org/*Jobs.groovy"
ANALYTICS_SCHEDULE_JOBS_DSL_TARGET_JOBS:
- jobs/analytics-edx-jenkins.edx.org/*Jobs.groovy
JENKINS_ANALYTICS_CREDENTIALS:
- id: "{{ JENKINS_ANALYTICS_GITHUB_CREDENTIAL_ID }}"
......@@ -120,6 +121,7 @@ JENKINS_ANALYTICS_CREDENTIALS:
description: ssh access key, generated by ansible
privatekey: "{{ ANALYTICS_SCHEDULE_MASTER_SSH_CREDENTIAL_FILE }}"
ANALYTICS_SCHEDULE_COMMON_VARS: ''
ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION: true
ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS: ''
ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY: true
......@@ -134,6 +136,9 @@ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS: ''
jenkins_credentials_root: '/tmp/credentials'
jenkins_credentials_file_dest: "{{ jenkins_credentials_root }}/credentials.json"
jenkins_credentials_script: "{{ jenkins_credentials_root }}/addCredentials.groovy"
jenkins_seed_job_root: '{{ jenkins_home }}/workspace/_seedjob'
jenkins_seed_job_script: "{{ jenkins_seed_job_root }}/seedJob.dsl"
jenkins_seed_job_xmlfile: "{{ jenkins_seed_job_root }}/job-dsl-core/{{ jenkins_seed_job.name }}.xml"
jenkins_connection_retries: 240
jenkins_connection_delay: 1
......@@ -179,16 +184,35 @@ jenkins_seed_job:
type: git
url: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_URL }}"
dest: ""
branch: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_VERSION | default('master') }}"
branch: '\$DSL_BRANCH'
credential_id: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_CREDENTIAL_ID | default('') }}"
- scm:
type: git
url: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_URL | default('') }}"
dest: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST | default('') }}"
branch: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION | default('master') }}"
branch: '\$SECURE_BRANCH'
credential_id: "{{ ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID | default('') }}"
analytics_tasks:
- id: ANSWER_DISTRIBUTION
enable: "{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION | default(true) }}"
extra_vars: "{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS | default('') }}"
- id: COURSE_ACTIVITY_WEEKLY
enable: "{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY | default(true) }}"
extra_vars: "{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY_EXTRA_VARS | default('') }}"
- id: IMPORT_ENROLLMENTS_INTO_MYSQL
enable: "{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL | default(true) }}"
extra_vars: "{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS | default('') }}"
- id: INSERT_TO_MYSQL_ALL_VIDEO
enable: "{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO | default(true) }}"
extra_vars: "{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS | default('') }}"
- id: INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY
enable: "{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY | default(true) }}"
extra_vars: "{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS | default('') }}"
dsl:
gradle_tasks: "clean libs test"
gradle_tasks:
- clean
- libs
- test
removed_view_action: IGNORE
removed_job_action: IGNORE
additional_classpath: "{{ ANALYTICS_SCHEDULE_JOBS_DSL_CLASSPATH }}"
......
playbooks/roles/jenkins_analytics/tasks/main.yml
......@@ -118,7 +118,7 @@
tags:
- jenkins-credentials
- name: upload groovy script
- name: upload groovy credentials script
template:
src: addCredentials.groovy
dest: "{{ jenkins_credentials_script }}"
......@@ -149,10 +149,39 @@
# Upload seed job
- name: upload job file
template: src=seed_job_template.xml dest=/tmp/{{ jenkins_seed_job.name }} mode="600"
- name: create seed job dir
file: name={{ jenkins_seed_job_root }} state=directory owner={{ jenkins_user }} group={{ jenkins_group }}
tags:
- create-seed-job
- jenkins-seed-job
- name: clone job-dsl-plugin repo
git:
repo: https://github.com/jenkinsci/job-dsl-plugin.git
depth: 1
dest: "{{ jenkins_seed_job_root }}"
sudo: yes
sudo_user: "{{ jenkins_user }}"
tags:
- jenkins-seed-job
- name: upload groovy seed job script
template:
src: seedJob.groovy
dest: "{{ jenkins_seed_job_script }}"
mode: "600"
owner: "{{ jenkins_user }}"
group: "{{ jenkins_group }}"
tags:
- jenkins-seed-job
- name: generate seed job xml
shell: >
cd {{ jenkins_seed_job_root }} &&
GRADLE_OPTS="-Dorg.gradle.daemon=true" ./gradlew run -Pargs={{ jenkins_seed_job_script }}
sudo: yes
sudo_user: "{{ jenkins_user }}"
tags:
- jenkins-seed-job
- name: check if job is present
include: execute_jenkins_cli.yaml
......@@ -160,12 +189,12 @@
jenkins_command_string: "get-job {{ jenkins_seed_job.name }}"
jenkins_ignore_cli_errors: yes
tags:
- create-seed-job
- jenkins-seed-job
- set_fact:
get_job_output: "{{ jenkins_command_output }}"
tags:
- create-seed-job
- jenkins-seed-job
# Upload seed job to Jenkins
......@@ -173,19 +202,19 @@
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "create-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
jenkins_command_prefix: "cat {{ jenkins_seed_job_xmlfile }} | "
when: get_job_output.rc != 0
tags:
- create-seed-job
- jenkins-seed-job
- name: update seed job
include: execute_jenkins_cli.yaml
vars:
jenkins_command_string: "update-job {{ jenkins_seed_job.name }}"
jenkins_command_prefix: "cat /tmp/{{ jenkins_seed_job.name }} | "
jenkins_command_prefix: "cat {{ jenkins_seed_job_xmlfile }} | "
when: get_job_output.rc == 0
tags:
- create-seed-job
- jenkins-seed-job
# Build the seed job
......@@ -194,4 +223,4 @@
vars:
jenkins_command_string: "build {{ jenkins_seed_job.name }} -s"
tags:
- create-seed-job
- jenkins-seed-job
playbooks/roles/jenkins_analytics/templates/seedJob.groovy 0 → 100644
/*
Jenkins Analytics Seed Job DSL template
*/
job('{{ jenkins_seed_job.name }}') {
description('Creates and configures the analytics task jobs.')
multiscm {
{% for scm in jenkins_seed_job.multiscm %}
{% if scm.url %}
git {
remote {
url('{{ scm.url }}')
branch("{{ scm.branch | default('master') }}")
{% if scm.dest %}
relativeTargetDir('{{ scm.dest }}')
{% endif %}
{% if scm.credential_id %}
credentials('{{ scm.credential_id }}')
{% endif %}
}
clean(true)
pruneBranches(true)
}
{% endif %}
{% endfor %}
}
parameters {
credentialsParam('MASTER_SSH_CREDENTIAL_ID', {
defaultValue('{{ ANALYTICS_SCHEDULE_MASTER_SSH_CREDENTIAL_ID | default("") }}')
description('Jenkins Credential with ssh access to EMR resources.')
})
credentialsParam('GIT_CREDENTIAL_ID', {
defaultValue('{{ ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID | default("") }}')
description('Jenkins Credential with read access to the secure git repos.')
})
stringParam('DSL_BRANCH', "{{ ANALYTICS_SCHEDULE_JOBS_DSL_REPO_VERSION | default('master') }}",
'Branch or version of the DSL repo to checkout and use to generate the jobs.')
stringParam('SECURE_BRANCH', "{{ ANALYTICS_SCHEDULE_SECURE_REPO_VERSION | default('master') }}",
'Branch or version of the secure repo to checkout and use to generate the jobs.')
textParam('COMMON_VARS', "{{ ANALYTICS_SCHEDULE_COMMON_VARS | default('') }}",
'Set default values for the common job parameters. Format as YAML or provide YAML file as @path/to/file.yml, ' +
' absolute or relative to seed job workpace.')
{% for task in jenkins_seed_job.analytics_tasks %}
booleanParam('{{ task.id }}',
{{ task.enable | ternary('true', 'false') }},
'Create or update this analytics task job.'
)
textParam('{{ task.id }}_EXTRA_VARS',
"{{ task.extra_vars }}",
'Default values for the analytics task job parameters. Format as YAML, or provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.'
)
{% endfor %}
}
steps {
gradle {
useWrapper(true)
makeExecutable(false)
{% for task in jenkins_seed_job.dsl.gradle_tasks %}
tasks('{{ task }}')
{% endfor %}
}
dsl {
removeAction('{{ jenkins_seed_job.dsl.removed_job_action }}')
removeViewAction('{{ jenkins_seed_job.dsl.removed_view_action }}')
additionalClasspath($/{{ jenkins_seed_job.dsl.additional_classpath }}/$)
lookupStrategy('SEED_JOB')
{% for job in jenkins_seed_job.dsl.target_jobs %}
external('{{ job }}')
{% endfor %}
}
}
keepDependencies(false)
disabled(false)
configure { project ->
canRoam(true)
}
}
playbooks/roles/jenkins_analytics/templates/seed_job_template.xml deleted 100644 → 0
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description>
Creates and configures the analytics task jobs.
</description>
<keepDependencies>false</keepDependencies>
<properties>
<jenkins.advancedqueue.AdvancedQueueSorterJobProperty plugin="PrioritySorter@2.9">
<useJobPriority>false</useJobPriority>
<priority>-1</priority>
</jenkins.advancedqueue.AdvancedQueueSorterJobProperty>
<hudson.model.ParametersDefinitionProperty>
<parameterDefinitions>
<com.cloudbees.plugins.credentials.CredentialsParameterDefinition plugin="credentials@1.24">
<name>MASTER_SSH_CREDENTIAL_ID</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_MASTER_SSH_CREDENTIAL_ID }}</defaultValue>
<credentialType>com.cloudbees.plugins.credentials.common.StandardCredentials</credentialType>
<required>true</required>
<description>Jenkins Credential ID used for ssh access to EMR resources.</description>
</com.cloudbees.plugins.credentials.CredentialsParameterDefinition>
<com.cloudbees.plugins.credentials.CredentialsParameterDefinition plugin="credentials@1.24">
<name>GIT_CREDENTIAL_ID</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_SECURE_REPO_CREDENTIAL_ID | default('') }}</defaultValue>
<credentialType>com.cloudbees.plugins.credentials.common.StandardCredentials</credentialType>
<required>false</required>
<description>Jenkins Credential ID used for cloning secure git repos.</description>
</com.cloudbees.plugins.credentials.CredentialsParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>ANSWER_DISTRIBUTION</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION | default(true) }}</defaultValue>
<description>Create or update the AnswerDistributionWorkflow analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>ANSWER_DISTRIBUTION_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS | default('') }}</defaultValue>
<description>Default values for the AnswerDistributionWorkflow job parameters. Provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.</description>
</hudson.model.StringParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>COURSE_ACTIVITY_WEEKLY</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY | default(true) }}</defaultValue>
<description>Create or update the CourseActivityWeeklyTask analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>COURSE_ACTIVITY_WEEKLY_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY_EXTRA_VARS | default('') }}</defaultValue>
<description>Default values for the CourseActivityWeeklyTask job parameters. Provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.</description>
</hudson.model.StringParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>IMPORT_ENROLLMENTS_INTO_MYSQL</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL | default(true) }}</defaultValue>
<description>Create or update the ImportEnrollmentsIntoMysql analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS | default('') }}</defaultValue>
<description>Default values for the ImportEnrollmentsIntoMysql job parameters. Provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.</description>
</hudson.model.StringParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>INSERT_TO_MYSQL_ALL_VIDEO</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO | default(true) }}</defaultValue>
<description>Create or update the InsertToMysqlAllVideoTask analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS | default('') }}</defaultValue>
<description>Default values for the InsertToMysqlAllVideoTask job parameters. Provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.</description>
</hudson.model.StringParameterDefinition>
<hudson.model.BooleanParameterDefinition>
<name>INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY | default(true) }}</defaultValue>
<description>Create or update the InsertToMysqlCourseEnrollByCountryWorkflow analytics task.</description>
</hudson.model.BooleanParameterDefinition>
<hudson.model.StringParameterDefinition>
<name>INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS</name>
<defaultValue>{{ ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS | default('') }}</defaultValue>
<description>Default values for the InsertToMysqlCourseEnrollByCountryWorkflowJob job parameters. Provide YAML file as @path/to/file.yml, absolute or relative to seed job workpace.</description>
</hudson.model.StringParameterDefinition>
</parameterDefinitions>
</hudson.model.ParametersDefinitionProperty>
</properties>
<scm class='org.jenkinsci.plugins.multiplescms.MultiSCM'>
<scms>
{% for scm in jenkins_seed_job.multiscm %}
{% if scm.url %}
<scm class="hudson.plugins.git.GitSCM" plugin="git@2.4.0">
<configVersion>2</configVersion>
<userRemoteConfigs>
<hudson.plugins.git.UserRemoteConfig>
<url>{{ scm.url }}</url>
{% if scm.credential_id %}
<credentialsId>{{ scm.credential_id }}</credentialsId>
{% endif %}
</hudson.plugins.git.UserRemoteConfig>
</userRemoteConfigs>
<branches>
<hudson.plugins.git.BranchSpec>
<name>{{ scm.branch }}</name>
</hudson.plugins.git.BranchSpec>
</branches>
{% if scm.dest %}
<relativeTargetDir>{{ scm.dest }}</relativeTargetDir>
{% endif %}
<doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
<browser class="hudson.plugins.git.browser.AssemblaWeb">
<url></url>
</browser>
<submoduleCfg class="list"/>
<extensions/>
</scm>
{% endif %}
{% endfor %}
</scms>
</scm>
<canRoam>true</canRoam>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<concurrentBuild>false</concurrentBuild>
<builders>
<hudson.plugins.gradle.Gradle plugin="gradle@1.24">
<description></description>
<switches></switches>
<tasks>{{ jenkins_seed_job.dsl.gradle_tasks }}</tasks>
<rootBuildScriptDir></rootBuildScriptDir>
<buildFile></buildFile>
<gradleName>(x)</gradleName>
<useWrapper>true</useWrapper>
<makeExecutable>false</makeExecutable>
<fromRootBuildScriptDir>true</fromRootBuildScriptDir>
<useWorkspaceAsHome>false</useWorkspaceAsHome>
</hudson.plugins.gradle.Gradle>
<javaposse.jobdsl.plugin.ExecuteDslScripts plugin="job-dsl@1.43">
<targets>{{ jenkins_seed_job.dsl.target_jobs }}</targets>
<usingScriptText>false</usingScriptText>
<ignoreExisting>false</ignoreExisting>
<removedJobAction>{{ jenkins_seed_job.dsl.removed_job_action }}</removedJobAction>
<removedViewAction>{{ jenkins_seed_job.dsl.removed_view_action }}</removedViewAction>
<lookupStrategy>SEED_JOB</lookupStrategy>
<additionalClasspath>{{ jenkins_seed_job.dsl.additional_classpath }}</additionalClasspath>
</javaposse.jobdsl.plugin.ExecuteDslScripts>
</builders>
<publishers/>
<buildWrappers/>
</project>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment