/* https://build.libelektra.org/jenkins/job/elektra-jenkinsfile/
 * This file describes how the elektra-jenkinsfile buildjob should be
 * executed.
 *
 * 1. imports and global variables are set
 * 2. define the main stages of the pipeline
 * 3. describe sub stages. This is where you will want to add new builds
 * 4. helper section to help write build scripts
 *
 * General Information about Jenkinsfiles can be found at
 * https://jenkins.io/doc/book/pipeline/jenkinsfile/.
 *
 * A Snippet generator is available to the public at
 * https://qa.nuxeo.org/jenkins/pipeline-syntax/.
 * A list of available commands on the build server can be found after a login at
 * https://build.libelektra.org/jenkins/job/elektra-jenkinsfile/pipeline-syntax/.
 */

// FIXME sloc publish errors
// XXX add missing jobs
// TODO have a per plugin/binding deps in Dockerfile for easier maintanence
// TODO add warnings plugins to scan for compiler warnings
// TODO maybe intend with 2 lines to not waste so much ws

// Imports
import java.text.SimpleDateFormat

// Buildjob properties
properties([
    buildDiscarder(logRotator(numToKeepStr: '60', artifactNumToKeepStr: '60'))
])

// Globals
docker_node_label = 'docker'

DOCKER_IMAGES = [
    sid: [
        id: prefixDockerId('debian-sid'),
        context: "./scripts/docker/debian/sid",
        file: "./scripts/docker/debian/sid/Dockerfile"
    ],
    stretch: [
        id: prefixDockerId('debian-stretch'),
        context: "./scripts/docker/debian/stretch",
        file: "./scripts/docker/debian/stretch/Dockerfile",
    ],
    stretch_minimal: [
        id: prefixDockerId('debian-stretch-minimal'),
        context: "./scripts/docker/debian/stretch",
        file: "./scripts/docker/debian/stretch/Dockerfile.minimal",
    ],
    stretch_doc: [
        id: prefixDockerId('debian-stretch-doc'),
        context: "./scripts/docker/debian/stretch",
        file: "./scripts/docker/debian/stretch/Dockerfile.doc",
    ],
    jessie: [
        id: prefixDockerId('debian-jessie'),
        context: "./scripts/docker/debian/jessie",
        file: "./scripts/docker/debian/jessie/Dockerfile",
    ],
    xenial: [
        id: prefixDockerId('ubuntu-xenial'),
        context: "./scripts/docker/ubuntu/xenial",
        file: "./scripts/docker/ubuntu/xenial/Dockerfile",
    ]
]

/* Define reusable cmake Flag globals
 *
 * They can be passed to many of the test helper functions and the cmake
 * function and represent flags usually passed to cmake.
 */
CMAKE_FLAGS_BASE = [
    'SITE': '${STAGE_NAME}',
    'KDB_DB_SYSTEM': '${HOME}/.config/kdb/system',
    'KDB_DB_SPEC': '${HOME}/.config/kdb/spec',
    'CMAKE_INSTALL_PREFIX': '${WORKSPACE}/system',
    'INSTALL_SYSTEM_FILES': 'OFF',
    'BUILD_DOCUMENTATION': 'OFF'
]

// TODO Remove -DEPRECATED after #1954 is resolved
CMAKE_FLAGS_BUILD_ALL = [
    'BINDINGS': 'ALL;-DEPRECATED',
    'PLUGINS': 'ALL;-DEPRECATED',
    'TOOLS': 'ALL'
]

CMAKE_FLAGS_COVERAGE = ['ENABLE_COVERAGE': 'ON']

CMAKE_FLAGS_CLANG = [
    'CMAKE_C_COMPILER': 'clang',
    'CMAKE_CXX_COMPILER': 'clang++'
]

CMAKE_FLAGS_ASAN = ['ENABLE_ASAN': 'ON']
CMAKE_FLAGS_DEBUG = ['ENABLE_DEBUG': 'ON']
CMAKE_FLAGS_LOGGER = ['ENABLE_LOGGER': 'ON']

// Define Test Flag globals
TEST_MEM = 'mem'
TEST_NOKDB = 'nokdb'
TEST_ALL = 'all'
TEST_INSTALL = 'install'

NOW = new Date()

/* Determine what needs to be build
 *
 * Execution of this script is triggered by SCM changes or manually by
 * commenting 'jenkins build Jenkinsfile[REGEX] please. The [REGEX] is
 * optional but when it exists it allows for specification of which
 * stages should be build.
 */
BUILD_TARGET = determineBuildTarget()

/*****************************************************************************
 * Main Stages
 *
 * Serial stages that contain parallized logic. Only proceeds to the next
 * if previous stage did not fail.
 *****************************************************************************/
lock('docker-images') {
    stage("Build docker images") {
        // makes sure docker images corresponding to the Dockerfiles are
        //   available to run the tests in
        parallel generate_docker_build_stages()
    }
}


// quick builds to give fast initial feedback to programmers
stage("Fast builds") {
    parallel generate_fast_build_stages()
}


// tests to reach full coverage
stage("Full builds") {
    parallel generate_full_build_stages()
}

// build debian packages + upload if on master
stage("Build Debian Packages") {
    parallel generate_package_build_stages()
}

/*****************************************************************************
 * Stage Generators
 *****************************************************************************/
def generate_docker_build_stages() {
    def tasks = [:]

    DOCKER_IMAGES.each { id, image ->
        tasks << maybe_build_image(image)
    }
    return tasks
}

/* Build image if not available in docker repository
 * @param image Map identifying which image to build
 */
def maybe_build_image(image) {
    def taskname = "build/${image.id}/"
    return [(taskname): {
        stage(taskname) {
            node(docker_node_label) {
                echo "Starting ${env.STAGE_NAME} on ${env.NODE_NAME}"
                checkout scm
                def id = imageFullName(image)
                docker.withRegistry('https://hub.libelektra.org',
                                    'docker-hub-elektra-jenkins') {
                    try {
                        // check if Docker file with tag exists
                        docker.image(id).pull()
                    } catch(e) {
                        // build and push if it does not
                        def uid = getUid()
                        def gid = getGid()
                        def i = docker.build(
                            id,"""\
--pull \
--build-arg JENKINS_GROUPID=${gid} \
--build-arg JENKINS_USERID=${uid} \
-f ${image.file} ${image.context}"""
                        )
                        i.push()
                    }
                }
            }
        }
    }]
}

/* Generate Test stages that return quickly
 *
 * Should be used to give quick feedback to developer and check for obvious
 * errors before the intensive tasks start
 */
def generate_fast_build_stages() {
    def tasks = [:]
    tasks << build_and_test(
        "debian-stable-fast",
        DOCKER_IMAGES.stretch,
        CMAKE_FLAGS_BUILD_ALL,
        [TEST_NOKDB]
    )
    tasks << build_and_test(
        "debian-unstable-fast",
        DOCKER_IMAGES.sid,
        CMAKE_FLAGS_DEBUG,
        [TEST_NOKDB]
    )
    return tasks
}

/* Generate Test stages for full tets coverage
 */
def generate_full_build_stages() {
    def tasks = [:]
    tasks << build_todo()
    tasks << build_doc()
    /* TODO Reenable for next release
    tasks << build_and_test_asan(
        "debian-stable-asan",
        DOCKER_IMAGES.stretch,
        CMAKE_FLAGS_BUILD_ALL
    )
    */
    tasks << build_and_test(
        "debian-stable-full",
        DOCKER_IMAGES.stretch,
        CMAKE_FLAGS_BUILD_ALL +
            CMAKE_FLAGS_COVERAGE,
        [TEST_ALL, TEST_MEM, TEST_INSTALL]
    )
    tasks << build_and_test(
        "debian-oldstable-full",
        DOCKER_IMAGES.jessie,
        CMAKE_FLAGS_BUILD_ALL +
            CMAKE_FLAGS_COVERAGE,
        [TEST_ALL, TEST_MEM, TEST_INSTALL]
    )
    tasks << build_and_test(
        "debian-unstable-full",
        DOCKER_IMAGES.sid,
        CMAKE_FLAGS_BUILD_ALL +
            CMAKE_FLAGS_COVERAGE,
        [TEST_ALL, TEST_MEM, TEST_INSTALL]
    )
    /* TODO Reenable for next release
    tasks << build_and_test(
        "debian-stable-full-clang",
        DOCKER_IMAGES.stretch,
        CMAKE_FLAGS_BUILD_ALL +
            CMAKE_FLAGS_COVERAGE +
            CMAKE_FLAGS_CLANG,
        [TEST_ALL, TEST_MEM, TEST_INSTALL]
    )
    */
    tasks << build_and_test(
        "ubuntu-xenial",
        DOCKER_IMAGES.xenial,
        CMAKE_FLAGS_BUILD_ALL,
        [TEST_ALL]
    )
    tasks << build_and_test(
        "debian-stable-minimal",
        DOCKER_IMAGES.stretch_minimal,
        [:],
        [TEST_ALL]
    )
    return tasks
}

def build_todo() {
    def test_name = "todo"
    def open_task_patterns = '''\
**/*.c, **/*.h, **/*.hpp, **/*.cpp,\
**/CMakeLists.txt, **/Dockerfile*, Jenkinsfile*
'''
    return withDockerEnv(test_name, DOCKER_IMAGES.stretch_doc) {
        sh "sloccount --duplicates --wide --details ${WORKSPACE} > sloccount.sc"
        step([$class: 'SloccountPublisher', ignoreBuildFailure: true])
        openTasks pattern: open_task_patterns,
                  high: 'XXX',
                  normal: 'FIXME',
                  low: 'TODO'
        archive(["sloccount.sc"])
        deleteDir()
    }
}

def build_doc() {
    def test_name = "doc"
    cmake_flags = [
        'BUILD_PDF': 'ON',
        'BUILD_FULL': 'OFF',
        'BUILD_SHARED': 'OFF',
        'BUILD_STATIC': 'OFF',
        'BUILD_TESTING': 'OFF'
    ]
    return withDockerEnv(test_name, DOCKER_IMAGES.stretch_doc) {
        dir('build') {
            deleteDir()
            cmake(env.WORKSPACE, cmake_flags)
            sh "make html latex man pdf"
        }
        warnings parserConfigurations: [
            [parserName: 'Doxygen', pattern: 'build/doc/doxygen.log']
        ]
        // TODO don't write to latest on PR's
        sshPublisher(publishers: [
          sshPublisherDesc(
            configName: 'doc.libelektra.org',
            transfers: [
              sshTransfer(
                sourceFiles: 'build/doc/latex/*',
                removePrefix: 'build/doc/',
                remoteDirectory: 'api/latest'
              ),
              sshTransfer(
                sourceFiles: 'build/doc/man/*',
                removePrefix: 'build/doc/',
                remoteDirectory: 'api/latest'
              ),
              sshTransfer(
                sourceFiles: 'doc/help/*.html',
                removePrefix: 'doc/help/',
                remoteDirectory: 'help'
              )
          ])
        ])
        deleteDir()
    }
}

/* Helper to generate an asan enabled test */
def build_and_test_asan(test_name, image, extra_cmake_flags = [:]) {
    def cmake_flags = CMAKE_FLAGS_BASE +
                      CMAKE_FLAGS_ASAN +
                      extra_cmake_flags
    return withDockerEnv(test_name, image) {
        dir('build') {
            deleteDir()
            cmake(env.WORKSPACE, cmake_flags)
            sh "make"
            def llvm_symbolizer = sh(returnStdout: true,
                                     script: 'which llvm-symbolizer').trim()
            withEnv(["ASAN_OPTIONS='symbolize=1'",
                     "ASAN_SYMBOLIZER_PATH=${llvm_symbolizer}"]){
                ctest()
            }
        }
    }
}

/* Helper to generate a typical elektra test environment
 *   Builds elektra, depending on the contents of 'tests' it runs the
 *   corresponding test suites.
 * test_name: used to identify the test and name the stage
 * image: which docker image should be used
 * cmake_flags: which flags should be passed to cmake
 * tests: list of tests which should be run
 * extra_artifacts: which files should be additionally saved from the build
 */
def build_and_test(test_name, image, extra_cmake_flags = [:],
                   tests = [], extra_artifacts = []) {
    def cmake_flags = CMAKE_FLAGS_BASE + extra_cmake_flags
    def artifacts = []
    if(tests) {
        artifacts.add("build/Testing/*/*.xml")
    }
    def test_coverage = cmake_flags.intersect(CMAKE_FLAGS_COVERAGE)
                                   .equals(CMAKE_FLAGS_COVERAGE)
    def test_mem = tests.contains(TEST_MEM)
    def test_nokdb = tests.contains(TEST_NOKDB)
    def test_all = tests.contains(TEST_ALL)
    def install = tests.contains(TEST_INSTALL)
    return withDockerEnv(test_name, image) {
        try {
            dir('build') {
                deleteDir()
                cmake(env.WORKSPACE, cmake_flags)
                sh "make"
                trackCoverage(test_coverage) {
                    if(test_all) {
                        ctest()
                    } else if(test_nokdb) {
                        cnokdbtest()
                    }
                    if(test_mem) {
                        cmemcheck(test_nokdb)
                    }
                }
                if(install) {
                    sh 'make install'
                }
            }
            if(install) {
                sh '''\
export LD_LIBRARY_PATH=${WORKSPACE}/system/lib:$LD_LIBRARY_PATH
export PATH=${WORKSPACE}/system/bin:$PATH
export DBUS_SESSION_BUS_ADDRESS=`dbus-daemon --session --fork --print-address`
export LUA_CPATH="${WORKSPACE}/system/lib/lua/5.2/?.so;"
kdb run_all
kill `pidof dbus-daemon`
'''
            }
        } catch(e) {
            // rethrow to mark as failed
            throw e
        } finally {
            /* Warnings plugin overwrites each other, disable for now
            warnings canRunOnFailed: true, consoleParsers: [
                [parserName: 'GNU Make + GNU C Compiler (gcc)']
            ]
            */
            archive(artifacts)
            if(test_coverage) {
                publishCoverage(test_name)
            }
            if(test_mem || test_nokdb || test_all) {
                xunitUpload()
            }
            deleteDir()
        }
    }
}

/* Generate Stages that build Debian packages
 */
def generate_package_build_stages() {
    def tasks = [:]
    tasks << build_package_debian_stretch()
    return tasks
}

def build_package_debian_stretch() {
    return withDockerEnv("package/stretch", DOCKER_IMAGES.stretch) {
        withCredentials([file(credentialsId: 'jenkins-key', variable: 'KEY'),
                         file(credentialsId: 'jenkins-secret-key', variable: 'SKEY')]) {
            sh "gpg --import $KEY"
            sh "gpg --import $SKEY"
        }
        withEnv(["DEBSIGN_PROGRAM=gpg",
                 "DEBFULLNAME=Jenkins (User for Elektra automated build system)",
                 "DEBEMAIL=autobuilder@libelektra.org"]) {
            sh "rm -R ./*"
            def target_dir="./libelektra"
            checkout scm: [
                $class: 'GitSCM',
                branches: scm.branches,
                extensions: scm.extensions + [
                    [$class: 'PerBuildTag'],
                    [$class: 'RelativeTargetDirectory',
                     relativeTargetDir: target_dir]
                ],
                userRemoteConfigs: scm.userRemoteConfigs
            ]
            dir(target_dir) {
                sh "git checkout -B temp"
                sh "git tag -f $VERSION"

                sh "git checkout -B debian origin/debian"
                sh "git merge --no-ff -m 'merge $VERSION' temp"

                sh "dch -l '.$BUILD_NUMBER' 'auto build'"
                sh "git commit -am 'auto build $VERSION'"

                sh "gbp buildpackage -sa"
            }
        }
        publishDebianPackages()
    }
}

/*****************************************************************************
 * Define helper functions
 *****************************************************************************/

/* Archives files located in paths
 *
 * Automatically prefixes with the current STAGE_NAME to identify where the
 * file was created.
 * @param paths List of paths to be archived
 */
def archive(paths) {
    echo "Start archivation"
    if (paths) {
        def prefix = "artifacts/"
        def dest = "${prefix}${env.STAGE_NAME}/"
        sh "mkdir -p ${dest}"
        paths.each { path ->
            sh "cp -v ${path} ${dest} || true"
        }
        archiveArtifacts artifacts: "${prefix}**", fingerprint: true
    } else {
        echo "No Artifacts to archive"
    }
    echo "Finish archivation"
}


/* Run cmake
 * @param directory Basedir for cmake
 * @param args_map Map of arguments for cmake
 */
def cmake(String directory, Map args_map) {
    def args_str = ""
    args_map.each { key, value ->
        args_str += "-D$key=\"$value\" "
    }
    sh("cmake $args_str $directory")
}

/* Publishes coverage reports
 * @param test_name Name of the test for identification
 */
def publishCoverage(String test_name) {
    echo "Start publication of coverage data"
    sh "mkdir -p ssh/coverage/ || true"
    sh "mv build/coverage ssh/coverage/${test_name} || true"
    sshPublisher(publishers: [
      sshPublisherDesc(
        configName: 'doc.libelektra.org',
        transfers: [
          sshTransfer(
            sourceFiles: 'ssh/**',
            removePrefix: 'ssh'
          )
      ])
    ])
    echo "Finish publication of coverage data"
}

/* Get the current users uid
 */
def getUid() {
    return sh(returnStdout: true, script: 'id -u').trim()
}


/* Get the current users gid
 */
def getGid() {
    return sh(returnStdout: true, script: 'id -g').trim()
}

/* Track coverage
 *
 * Tracks coverage of commands executed in the passed closure if do_track
 * evaluates to true.
 * @param do_track If true track coverage
 * @param cl A closure that this function wraps around
 */
def trackCoverage(do_track, cl) {
    if(do_track) {
        sh 'make coverage-start'
    }
    cl()
    if(do_track) {
        sh 'make coverage-stop'
        sh 'make coverage-genhtml'
    }
}

/* Run the passed closure in a docker environment
 *
 * Automatically takes care of docker registry authentification,
 * selecting a docker capable node,
 * checkout of scm and
 * setting of useful Environment variables
 * @param stage_name Name of the stage
 * @param image Docker image that should be used
 * @param cl A closure that should be run inside the docker image
 */
def withDockerEnv(stage_name, image, cl) {
    def run_stage = (stage_name =~ BUILD_TARGET) as boolean
    return [(stage_name): {
        maybeStage(stage_name, run_stage) {
            node(docker_node_label) {
                docker.withRegistry('https://hub.libelektra.org',
                                    'docker-hub-elektra-jenkins') {
                    timeout(activity: true, time: 5, unit: 'MINUTES') {
                        def cpu_count = cpuCount()
                        withEnv(["MAKEFLAGS='-j${cpu_count+2} -l${cpu_count*2}'",
                                 "CTEST_PARALLEL_LEVEL='${cpu_count+2}'"]) {
                            echo "Starting ${env.STAGE_NAME} on ${env.NODE_NAME}"
                            checkout scm
                            docker.image(imageFullName(image))
                                  .inside("-v ${env.HOME}/git_mirrors:/home/jenkins/git_mirrors") { cl() }
                        }
                    }
                }
            }
        }
    }]
}

/* Get cpu count
 */
def cpuCount() {
    return sh(returnStdout: true,
              script: 'grep -c ^processor /proc/cpuinfo').trim() as Integer
}

/* Run ctest with appropriate env variables
 * @param target What target to pass to ctest
 */
def ctest(target = "Test") {
    sh """ctest -j ${env.CTEST_PARALLEL_LEVEL} --force-new-ctest-process \
            --output-on-failure --no-compress-output -T ${target}"""
}

/* Helper for ctest to run MemCheck without memleak tagged tests
 * @param kdbtests If true run tests tagged as kdbtests
 */
def cmemcheck(kdbtests) {
    if(kdbtests) {
        ctest("MemCheck -LE memleak")
    } else {
        ctest("MemCheck -LE memleak||kdbtests")
    }
}

/* Helper for ctest to run tests without tests tagged as kdbtests
 */
def cnokdbtest() {
    ctest("Test -LE kdbtests")
}

/* Uploads ctest restults
 */
def xunitUpload() {
    step([$class: 'XUnitBuilder',
          thresholds: [
            [$class: 'SkippedThreshold', failureThreshold: '0'],
            [$class: 'FailedThreshold', failureThreshold: '0']
          ],
          tools: [
            [$class: 'CTestType',
                pattern: 'build/Testing/**/*.xml']
        ]
    ])
}

/* Build full name of docker image
 *
 * We use identifiers in the form of name:yyyyMM-hash
 * The hash is build from reading the Dockerfile
 * @param image_map Map identifying an docker image (see DOCKER_IMAGES)
 */
def imageFullName(image_map) {
    def cs = checksum(image_map.file)
    def dateString = dateFormatter(NOW)
    return "${image_map.id}:${dateString}-${cs}"
}

/* Generate the checksum of a file
 * @param file File to generate a checksum for
 */
def checksum(file) {
    // Used to identify if a Dockerfile changed
    // TODO expand to use more than one file if Dockerfile ever depends on
    //      external files
    return sh(returnStdout: true,
              script: "cat $file | sha256sum | dd bs=1 count=64 status=none")
           .trim()
}

/* Generate a Stage
 *
 * If `expression` evaluates to TRUE, a stage(`name`) with `body` is run
 * @param name Name of the stage
 * @param expression If True, run body
 * @param body Closure representing stage body
 */
def maybeStage(String name, boolean expression, Closure body) {
    if(expression) {
        stage(name, body)
    } else {
        stage(name) {
            echo "Stage skipped: ${name}"
        }
    }
}

/* Format the date input
 * @param date Date to format
 */
def dateFormatter(date) {
    df = new SimpleDateFormat("yyyyMM")
    return df.format(date)
}

/* Determines which stages should be build
 *
 * Usually build is triggered by scm changes, but it can also manually be
 * started for a PR by commenting "jenkins build jenkinsfile[REGEX] please"
 * The optional REGEX determines if all stages should be build or only a
 * selection of them.
 * To build only fast builds one would commenet "jenkins build
 * jenkinsfile[.*-fast] please".
 * NOTE: none complete builds are automatically set to UNSTABLE which reports
 * as a test failure on github.
 * @param default_target What should be build if no REGEX was provided
 */
def determineBuildTarget(default_target = ".*") {
    def target = default_target

    matcher = /jenkins build jenkinsfile(?:\[(.*)\])? please/
    try {
        new_target = (env.ghprbCommentBody =~ matcher)[0][1]
    } catch(e) {
        new_target = null
    }
    if(new_target) {
        // if we specify a new target we degrade the build to unstable
        // this will report the test as failed on github
        currentBuild.result = 'UNSTABLE'
        target = new_target
    }
    return target
}

/* Provide a common prefix for an image name
 * @param name Name of the docker image
 */
def prefixDockerId(name) {
    def base = 'build-elektra-'
    return base + name
}

/* Returns True if we are on the master branch
 */
def isMaster() {
    return env.sha1=="master"
}

/* Publishes all files necessary for hosting a debian package
 * @param remote where the repository is located
 */
def publishDebianPackages(remote="a7") {
    if(isMaster()) {
        def remotedir = 'compose/frontend/volumes/incoming'
        sshPublisher(
          publishers: [
            sshPublisherDesc(
              configName: remote,
              transfers: [
                sshTransfer(
                  sourceFiles: '*.deb',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.build',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.buildinfo',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.dsc',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.tar.xz',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.tar.gz',
                  remoteDirectory: remotedir
                ),
                sshTransfer(
                  sourceFiles: '*.changes',
                  remoteDirectory: remotedir
                )
              ]
            )
          ],
          verbose: true,
          failOnError: true
        )
    } else {
        echo "Skipping package publish because we are not on master"
    }
}
