提交测试

ci/gitlab_jenkins_templates/core_ci.jenkins

@@ -0,0 +1,492 @@
+#!/usr/bin/env groovy
+
+import groovy.transform.Field
+
+if (gitlabActionType == "MERGE" || gitlabSourceBranch == "master") {
+gitlabCommitStatus("launch all builds") {
+
+
+// Configs for build from pytorch docker images
+// (See: https://hub.docker.com/r/pytorch/pytorch/tags)
+def ubuntu_from_pytorch_configs = [
+    [
+        // python: 3.7
+        'cudaVer': '11.3', 'cudnnVer': '8', 'torchVer': '1.12.1',
+        'archsToTest': 'MULTI'
+    ],
+    [
+        // python: 3.7
+        'cudaVer': '11.6', 'cudnnVer': '8', 'torchVer': '1.13.1',
+        'archsToTest': 'MULTI'
+    ],
+    [
+        // python: 3.7
+        'cudaVer': '12.1', 'cudnnVer': '8', 'torchVer': '2.1.0',
+        'archsToTest': 'MULTI'
+    ],
+    [
+        // python: 3.10
+        'cudaVer': '11.8', 'cudnnVer': '8', 'torchVer': '2.1.0',
+        'archsToTest': 'MULTI'
+    ]
+]
+
+// Configs for build from NGC pytorch docker images
+// (See: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags)
+def ubuntu_from_nvcr_configs = [
+    [
+        'baseImageTag': '23.10-py3',
+        'archsToTest': 'MULTI'
+    ],
+]
+
+// Configs for build from cuda container
+// with custom installation of all the dependencies like PyTorch
+// (See: https://hub.docker.com/r/nvidia/cuda/tags)
+def ubuntu_from_cuda_configs = [
+    [
+        'cudaVer': '11.3.1', 'cudnnVer': '8',
+        'pythonVer': '3.8', 'torchVer': '1.11.0',
+        'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '12.1.0', 'cudnnVer': '8',
+        'pythonVer': '3.10', 'torchVer': '2.1.0',
+        'archsToTest': 'MULTI'
+    ],
+]
+
+// Configs for build for cpu only
+// (Use docker image ubuntu:18.04 as a base)
+def ubuntu_cpuonly_configs = [
+    [
+        'pythonVer': '3.8', 'torchVer': '1.11.0',
+    ],
+    [
+        'pythonVer': '3.10', 'torchVer': '2.0.1',
+    ]
+]
+
+// Configs for building python wheels
+def ubuntu_for_wheels_configs = [
+/*
+    [
+        'cudaVer': '11.3.1', 'cudnnVer': '8',
+        'torchVer': '1.12.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.6.2', 'cudnnVer': '8',
+        'torchVer': '1.12.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.3.1', 'cudnnVer': '8',
+        'torchVer': '1.12.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.6.2', 'cudnnVer': '8',
+        'torchVer': '1.12.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.6.2', 'cudnnVer': '8',
+        'torchVer': '1.13.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.7.1', 'cudnnVer': '8',
+        'torchVer': '1.13.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.6.2', 'cudnnVer': '8',
+        'torchVer': '1.13.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.7.1', 'cudnnVer': '8',
+        'torchVer': '1.13.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.7.1', 'cudnnVer': '8',
+        'torchVer': '2.0.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.8.0', 'cudnnVer': '8',
+        'torchVer': '2.0.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.7.1', 'cudnnVer': '8',
+        'torchVer': '2.0.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.8.0', 'cudnnVer': '8',
+        'torchVer': '2.0.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.8.0', 'cudnnVer': '8',
+        'torchVer': '2.1.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '12.1.0', 'cudnnVer': '8',
+        'torchVer': '2.1.0', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '11.8.0', 'cudnnVer': '8',
+        'torchVer': '2.1.1', 'archsToTest': 'MULTI'
+    ],
+    [
+        'cudaVer': '12.1.0', 'cudnnVer': '8',
+        'torchVer': '2.1.1', 'archsToTest': 'MULTI'
+    ],
+*/
+]
+
+def windows_for_wheels_configs = [
+/*
+    [
+        'cudaVer': '11.3', 'cudnnVer': '8',
+        'torchVer': '1.12.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.6', 'cudnnVer': '8',
+        'torchVer': '1.12.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.3', 'cudnnVer': '8',
+        'torchVer': '1.12.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.6', 'cudnnVer': '8',
+        'torchVer': '1.12.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.6', 'cudnnVer': '8',
+        'torchVer': '1.13.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.7', 'cudnnVer': '8',
+        'torchVer': '1.13.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.6', 'cudnnVer': '8',
+        'torchVer': '1.13.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.7', 'cudnnVer': '8',
+        'torchVer': '1.13.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.7', 'cudnnVer': '8',
+        'torchVer': '2.0.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.8', 'cudnnVer': '8',
+        'torchVer': '2.0.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.7', 'cudnnVer': '8',
+        'torchVer': '2.0.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.8', 'cudnnVer': '8',
+        'torchVer': '2.0.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.8', 'cudnnVer': '8',
+        'torchVer': '2.1.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '12.1', 'cudnnVer': '8',
+        'torchVer': '2.1.0', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '11.8', 'cudnnVer': '8',
+        'torchVer': '2.1.1', 'archsToTest': ''
+    ],
+    [
+        'cudaVer': '12.1', 'cudnnVer': '8',
+        'torchVer': '2.1.1', 'archsToTest': ''
+    ]
+*/
+]
+
+// Configs for build from Windows server docker images
+// (See: https://hub.docker.com/_/microsoft-dotnet-framework-sdk)
+def windows_from_server_configs = [
+    // CUDA drivers on test machines only available for CUDA version >= 11.0
+    //  see: https://gitlab-master.nvidia.com/ipp/cloud-infra/blossom/dev/windows-gpu-pods/-/tree/master/ContainerDriverSetup
+    // test machines currently are the only option, named 'gpu_tester'
+    //      two machines exist, only the TITAN RTX will pass tests
+    [
+        'cudaVer': '11.3',
+        'pythonVer': '3.8', 'torchVer': '1.11.0',
+        'archsToTest': 'gpu_tester' //'Tesla_V100_PCIE_32GB'
+    ],
+    [
+        'cudaVer': '11.8',
+        'pythonVer': '3.10', 'torchVer': '2.0.1',
+        'archsToTest': 'gpu_tester' //'Tesla_V100_PCIE_32GB'
+    ]
+]
+
+dockerRegistryServer = 'gitlab-master.nvidia.com:5005'
+dockerRegistryName = 'toronto_dl_lab/kaolin'
+imageBaseTag = "${dockerRegistryServer}/${dockerRegistryName}/kaolin"
+// Used for target docker image tag, as it doesn't support all characters (such as /)
+branchRef = gitlabSourceBranch.replaceAll("[^a-zA-Z0-9]", "-")
+
+node {
+    checkout scm
+    // Sanity check, in case this script fail to launch all builds and tests
+    // Right now we only apply CI on MR and master branch.
+    // To enable master branch we have to accept all the push requests
+    // and prune them here.
+    sh "echo ${gitlabActionType}"
+    jobMap = [:]
+    // Jenkins doesn't parse the commit hash from the webhook.
+    // So we need to get the commit hash from the last commit in the branch,
+    // So we unsure that all the build and run are on the same commit
+    //
+    // Note:
+    //   If two commits on the same branch are pushed before the first
+    //   run this line then they will both run on the second commit
+    commitHash = sh(script: "git log -1 --pretty=format:%h",
+                    returnStdout: true).trim()
+    sh "echo ${commitHash}"
+    if (gitlabActionType == "MERGE" &&
+        gitlabMergeRequestTitle.contains("[for wheels]")) {
+            for (config in ubuntu_for_wheels_configs) {
+                for (pythonVer in ['3.8', '3.9', '3.10']) {
+                    def configName = "custom-wheels-torch${config['torchVer']}-" + \
+                                     "cuda${config['cudaVer']}-" +
+                                     "cudnn${config['cudnnVer']}-" +
+                                     "py${pythonVer}"
+                    jobMap["${configName}"] = prepareUbuntuFromCUDAJob(
+                        configName,
+                        config['cudaVer'],
+                        config['cudnnVer'],
+                        pythonVer,
+                        config['torchVer'],
+                        config['archsToTest'],
+                        true
+                    )
+                }
+            }
+            for (config in windows_for_wheels_configs) {
+                for (pythonVer in ['3.8', '3.9', '3.10']) {
+
+                    def cudaVerLabel = config['cudaVer'].split('\\.').join('')
+                    def torchVerLabel = config['torchVer'].split('\\.').join('')
+                    def pythonVerLabel = pythonVer.split('\\.').join('')
+                    def configName = "windows-wheels-cuda${cudaVerLabel}-py${pythonVerLabel}-torch${torchVerLabel}"
+                    jobMap["${configName}"] = prepareWindowsJob(
+                        configName,
+                        config['cudaVer'],
+                        pythonVer,
+                        config['torchVer'],
+                        config['archsToTest'],
+                        true
+                    )
+                }
+            }
+    } else {
+        // Check if the last commit message has a [with custom] tag
+        def hasNoCustomInMess = sh(script: "git log -1 | grep '.*\\[with custom\\].*'",
+                                   returnStatus: true)
+        if (gitlabActionType == "MERGE") {
+            sh "echo ${gitlabMergeRequestTitle}"
+        }
+        // We try to build from cuda docker image if the commit has such tag
+        // or CI is applied on master
+        if (hasNoCustomInMess == 0 || gitlabSourceBranch == "master" ||
+            gitlabMergeRequestTitle.contains("[with custom]")) {
+            for (config in ubuntu_from_cuda_configs) {
+                def configName = "custom-torch${config['torchVer']}-" + \
+                                 "cuda${config['cudaVer']}-" +
+                                 "cudnn${config['cudnnVer']}-" +
+                                 "py${config['pythonVer']}"
+                jobMap["${configName}"] = prepareUbuntuFromCUDAJob(
+                    configName,
+                    config['cudaVer'],
+                    config['cudnnVer'],
+                    config['pythonVer'],
+                    config['torchVer'],
+                    config['archsToTest'],
+                    false
+                )
+            }
+        }
+
+        for (config in ubuntu_from_pytorch_configs) {
+            def configName = "pytorch-torch${config['torchVer']}-" + \
+                             "cuda${config['cudaVer']}-cudnn${config['cudnnVer']}"
+            def baseImageTag = "pytorch/pytorch:${config['torchVer']}-" + \
+                               "cuda${config['cudaVer']}-" + \
+                               "cudnn${config['cudnnVer']}-devel"
+            jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
+                configName,
+                baseImageTag,
+                config['archsToTest']
+            )
+        }
+
+        for (config in ubuntu_from_nvcr_configs) {
+            def configName = "nvcr-${config['baseImageTag']}"
+            def baseImageTag = "nvcr.io/nvidia/pytorch:${config['baseImageTag']}"
+            jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
+                configName,
+                baseImageTag,
+                config['archsToTest']
+            )
+        }
+
+        for (config in ubuntu_cpuonly_configs) {
+            def torchVerLabel = config['torchVer'].split('\\.').join('')
+            def pythonVerLabel = config['pythonVer'].split('\\.').join('')
+            def configName = "cpuonly-py${config['pythonVer']}-torch${config['torchVer']}"
+            jobMap["${configName}"] = prepareUbuntuCPUOnlyJob(
+                configName,
+                config['pythonVer'],
+                config['torchVer']
+            )
+        }
+
+        for (config in windows_from_server_configs) {
+            def cudaVerLabel = config['cudaVer'].split('\\.').join('')
+            def torchVerLabel = config['torchVer'].split('\\.').join('')
+            def pythonVerLabel = config['pythonVer'].split('\\.').join('')
+            def configName = "windows-cuda${cudaVerLabel}-py${pythonVerLabel}-torch${torchVerLabel}"
+            jobMap["${configName}"] = prepareWindowsJob(
+                configName,
+                config['cudaVer'],
+                config['pythonVer'],
+                config['torchVer'],
+                config['archsToTest'],
+                false
+            )
+        }
+    }
+    stage('Launch builds') {
+        parallel jobMap
+    }
+}
+
+}  // gitlabCommitStatus
+}  // if (gitlabActionType == "MERGE" || gitlabSourceBranch == "master")
+
+
+def prepareUbuntuFromBaseImageJob(configName, baseImageTag, archsToTest) {
+  return {
+    stage("${configName}") {
+      // Notify Gitlab about the build and tests it will be running
+      // so it doesn't the build successful before it start running them
+      // and we can also see issue if the build / test is never run.
+      updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
+      for (arch in archsToTest.split(';')) {
+        updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
+      }
+      build job: "ubuntu_build_template_CI",
+      parameters: [
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'baseImageTag', value: "${baseImageTag}"),
+        string(name: 'targetImageTag',
+               value: "${imageBaseTag}:${branchRef}-${BUILD_ID}-${configName}"),
+        string(name: 'archsToTest', value: "${archsToTest}"),
+        string(name: 'sourceBranch', value: "${env.gitlabSourceBranch}"),
+        string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}"),
+        string(name: 'commitHash', value: "${commitHash}")
+      ],
+      // This node doesn't need to be held while builds and tests run.
+      wait: false,
+      // Success of this script depend only on successful launch,
+      // Not successful builds and tests.
+      propagate: false
+    }
+  }
+}
+
+def prepareUbuntuFromCUDAJob(configName, cudaVer, cudnnVer, pythonVer, torchVer, archsToTest,
+                             buildWheel) {
+  return {
+    stage("${configName}") {
+      // Notify Gitlab about the build and tests it will be running
+      // so it doesn't the build successful before it start running them
+      // and we can also see issue if the build / test is never run.
+      updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
+      for (arch in archsToTest.split(';')) {
+        updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
+      }
+      build job: "ubuntu_custom_build_template_CI",
+      parameters: [
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'cudaVer', value: "${cudaVer}"),
+        string(name: 'cudnnVer', value: "${cudnnVer}"),
+        string(name: 'pythonVer', value: "${pythonVer}"),
+        string(name: 'torchVer', value: "${torchVer}"),
+        string(name: 'targetImageTag',
+               value: "${imageBaseTag}:${branchRef}-${BUILD_ID}-${configName}"),
+        string(name: 'sourceBranch', value: "${env.gitlabSourceBranch}"),
+        string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}" ),
+        string(name: 'archsToTest', value: "${archsToTest}"),
+        string(name: 'commitHash', value: "${commitHash}"),
+        booleanParam(name: 'buildWheel', value: "${buildWheel}")
+      ],
+      // This node doesn't need to be held while builds and tests run.
+      wait: false,
+      // Success of this script depend only on successful launch,
+      // Not successful builds and tests.
+      propagate: false
+    }
+  }
+}
+
+def prepareUbuntuCPUOnlyJob(configName, pythonVer, torchVer) {
+  return {
+    stage("${configName}") {
+      updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
+      updateGitlabCommitStatus(name: "test-${configName}", state: "pending")
+      build job: "ubuntu_cpuonly_template_CI",
+      parameters: [
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'pythonVer', value: "${pythonVer}"),
+        string(name: 'torchVer', value: "${torchVer}"),
+        string(name: 'targetImageTag',
+               value: "${imageBaseTag}:${branchRef}-${BUILD_ID}-${configName}"),
+        string(name: 'sourceBranch', value: "${env.gitlabSourceBranch}"),
+        string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}" ),
+        string(name: 'commitHash', value: "${commitHash}")
+      ],
+      wait: false,
+      propagate: false
+    }
+  }
+}
+
+def prepareWindowsJob(configName, cudaVer, pythonVer, torchVer, archsToTest,
+                      buildWheel) {
+  return {
+    stage("${configName}") {
+      updateGitlabCommitStatus(name: "build-${configName}", state: "pending")
+      if (buildWheel.toBoolean()) {
+        updateGitlabCommitStatus(name: "test-${configName}", state: "pending")
+      } //else {
+      //  for (arch in archsToTest.split(';')) {
+      //    updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: "pending")
+      //  }
+      //}
+
+      build job: "windows_build_template_CI",
+      parameters: [
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'cudaVer', value: "${cudaVer}"),
+        string(name: 'pythonVer', value: "${pythonVer}"),
+        string(name: 'torchVer', value: "${torchVer}"),
+        string(name: 'targetImageTag',
+               value: "${imageBaseTag}:${branchRef}-${BUILD_ID}-${configName}"),
+        string(name: 'sourceBranch', value: "${gitlabSourceBranch}"),
+        string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}" ),
+        string(name: 'commitHash', value: "${commitHash}"),
+        string(name: 'archsToTest', value: "${archsToTest}"),
+        booleanParam(name: 'buildWheel', value: "${buildWheel}")
+      ],
+      wait: false,
+      propagate: false
+    }
+  }
+}

ci/gitlab_jenkins_templates/ubuntu_build_CI.jenkins

@@ -0,0 +1,117 @@
+#!/usr/bin/env groovy
+
+docker_registry_server = targetImageTag.split(':')[0..1].join(':')
+// This will be the "RUN" displayed on Blue Ocean
+currentBuild.displayName = targetImageTag.split(':')[2]
+// This will be the "MESSAGE" displayed on Blue Ocean
+currentBuild.description = sourceBranch + ": " + commitHash
+
+gitlabCommitStatus("build-${configName}") {
+
+podTemplate(
+cloud:'sc-ipp-blossom-prod',
+yaml:'''
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+  - name: docker
+    image: docker:20.10.23
+    command:
+    - sleep
+    args:
+    - 1d
+    env:
+      - name: DOCKER_HOST
+        value: tcp://localhost:2375
+  - name: docker-daemon
+    image: docker:20.10.23-dind
+    securityContext:
+      privileged: true
+    env:
+      - name: DOCKER_TLS_CERTDIR
+        value: ""
+    resources:
+        requests:
+          memory: 32Gi
+          cpu: 12
+        limits:
+          memory: 32Gi
+          cpu: 12
+''') {
+  node(POD_LABEL) {
+    container("docker") {
+      // This is to let the time for the docker-daemon to get initialized.
+      sleep 10
+      try {
+        stage("Checkout") {
+          checkout([
+              $class: 'GitSCM',
+              branches: [[name: "${commitHash}"]],
+              // We need submodules
+              extensions: [[
+                  $class: 'SubmoduleOption',
+                  disableSubmodules: false,
+                  parentCredentials: false,
+                  recursiveSubmodules: true,
+                  reference: '',
+                  trackingSubmodules: false
+              ]],
+              userRemoteConfigs: [[
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
+                  url: "${repoUrl}"
+              ]]
+          ])
+        }
+        docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
+          stage("Build") {
+            targetImage = docker.build(
+                "${targetImageTag}",
+                """--no-cache --network host -f ./tools/linux/Dockerfile.install \
+                   --build-arg BASE_IMAGE=${baseImageTag} \
+                   .
+                """)
+          }
+          stage("Push") {
+            targetImage.push()
+          }
+        }
+      } catch (e) {
+        // In case of build failure, we need to update the following tests as we won't run them.
+        for (arch in archsToTest.split(';')) {
+          updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: 'canceled')
+        }
+        throw e
+      }
+      stage("Launch tests") {
+        jobMap = [:]
+        for (arch in archsToTest.split(';')) {
+          jobMap["${arch}"] = prepareUbuntuTestJob(arch)
+        }
+        parallel jobMap
+      }
+    }
+  }
+}
+
+}  // gitlabCommitStatus
+
+def prepareUbuntuTestJob(arch) {
+  return {
+    stage("Test ${arch}") {
+      build job: "ubuntu_test_template_CI",
+      parameters: [
+        string(name: 'sourceBranch', value: "${sourceBranch}"),
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'imageTag', value: "${targetImageTag}"),
+        string(name: 'arch', value: "${arch}"),
+        string(name: 'commitHash', value: "${commitHash}")
+      ],
+      // This node doesn't need to be held while tests run.
+      wait: false,
+      // Success of this script depend only on successful build
+      // and launch of tests, not successful tests.
+      propagate: false
+    }
+  }
+}

ci/gitlab_jenkins_templates/ubuntu_cpuonly_CI.jenkins

@@ -0,0 +1,97 @@
+#!/usr/bin/env groovy
+
+gitlabCommitStatus("build-${configName}") {
+
+docker_registry_server = targetImageTag.split(':')[0..1].join(':')
+currentBuild.displayName = targetImageTag.split(':')[2]
+currentBuild.description = sourceBranch + ": " + commitHash
+
+podTemplate(
+cloud:'sc-ipp-blossom-prod',
+yaml:'''
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+  - name: docker
+    image: docker:20.10.23
+    command:
+    - sleep
+    args:
+    - 1d
+    env:
+      - name: DOCKER_HOST
+        value: tcp://localhost:2375
+  - name: docker-daemon
+    image: docker:20.10.23-dind
+    securityContext:
+      privileged: true
+    env:
+      - name: DOCKER_TLS_CERTDIR
+        value: ""
+    resources:
+        requests:
+          memory: 32Gi
+          cpu: 12  
+        limits:
+          memory: 32Gi
+          cpu: 12
+''') {
+  node(POD_LABEL) {
+    container("docker") {
+      // This is to let the time for the docker-daemon to get initialized.
+      sleep 10
+      try {
+        stage("Checkout") {
+          checkout([
+              $class: 'GitSCM',
+              branches: [[name: "${commitHash}"]],
+              extensions: [[
+                  $class: 'SubmoduleOption',
+                  disableSubmodules: false,
+                  parentCredentials: false,
+                  recursiveSubmodules: true,
+                  reference: '',
+                  trackingSubmodules: false
+              ]],
+              userRemoteConfigs: [[
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
+                  url: "${repoUrl}"
+              ]]
+          ])
+        }
+
+        stage("Build") {
+          def baseImage = docker.build(
+              "${targetImageTag}-base",
+              """--no-cache --network host -f ./tools/linux/Dockerfile.base_cpuonly \
+                 --build-arg PYTHON_VERSION=${pythonVer} \
+                 --build-arg PYTORCH_VERSION=${torchVer} \
+                 .
+              """)
+          targetImage = docker.build(
+              "${targetImageTag}",
+              """--no-cache --network host -f ./tools/linux/Dockerfile.install \
+                 --build-arg BASE_IMAGE=${targetImageTag}-base \
+                 --build-arg FORCE_CUDA=0 \
+                 .
+              """)
+        }
+        
+      } catch (e) {
+        updateGitlabCommitStatus(name: "test-${configName}", state: 'canceled')
+        throw e
+      }
+      gitlabCommitStatus("test-${configName}") {
+        stage("Test") {
+          targetImage.inside {
+            // Don't know why but it doesn't work from /kaolin with docker plugin
+            sh 'cd /tmp && python -c "import kaolin"'
+          }
+        }
+      }
+    }
+  }
+}
+
+}  // gitlabCommitStatus

ci/gitlab_jenkins_templates/ubuntu_custom_build_CI.jenkins

@@ -0,0 +1,167 @@
+#!/usr/bin/env groovy
+
+docker_registry_server = targetImageTag.split(':')[0..1].join(':')
+// This will be the "RUN" displayed on Blue Ocean
+currentBuild.displayName = targetImageTag.split(':')[2]
+// This will be the "MESSAGE" displayed on Blue Ocean
+currentBuild.description = sourceBranch + ": " + commitHash
+
+podTemplate(
+cloud:'sc-ipp-blossom-prod',
+//volumes: [persistentVolumeClaim(mountPath: '/mount_binaries', claimName: 'kaolin-pvc', readOnly: false)],
+yaml:'''
+apiVersion: v1
+kind: Pod
+spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
+  containers:
+  - name: docker
+    image: docker:19.03.1
+    command:
+    - sleep
+    args:
+    - 1d
+    env:
+      - name: DOCKER_HOST
+        value: tcp://localhost:2375
+    volumeMounts:
+      - mountPath: /mount_binaries
+        name: pvc-mount
+  - name: docker-daemon
+    image: docker:19.03.1-dind
+    securityContext:
+      privileged: true
+    env:
+      - name: DOCKER_TLS_CERTDIR
+        value: ""
+    resources:
+        requests:
+          memory: 32Gi
+          cpu: 12
+        limits:
+          memory: 32Gi
+          cpu: 12
+    volumeMounts:
+      - mountPath: /mount_binaries
+        name: pvc-mount
+''') {
+  node(POD_LABEL) {
+    container("docker") {
+      try {
+        gitlabCommitStatus("build-${configName}") {
+          stage("Checkout") {
+            checkout([
+                $class: 'GitSCM',
+                branches: [[name: "${commitHash}"]],
+                extensions: [[
+                    $class: 'SubmoduleOption',
+                    disableSubmodules: false,
+                    parentCredentials: false,
+                    recursiveSubmodules: true,
+                    reference: '',
+                    trackingSubmodules: false
+                ]],
+                userRemoteConfigs: [[
+                    credentialsId: 'kaolin-gitlab-access-token-as-password',
+                    url: "${repoUrl}"
+                ]]
+            ])
+          }
+          docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
+            stage("Build") {
+              baseImage = docker.build(
+                  "${targetImageTag}-base",
+                  """--no-cache --network host -f ./tools/linux/Dockerfile.base \
+                     --build-arg CUDA_VERSION=${cudaVer} \
+                     --build-arg CUDNN_VERSION=${cudnnVer} \
+                     --build-arg PYTHON_VERSION=${pythonVer} \
+                     --build-arg PYTORCH_VERSION=${torchVer} \
+                     .
+                  """)
+              targetImage = docker.build(
+                  "${targetImageTag}",
+                  """--no-cache --network host -f ./tools/linux/Dockerfile.install \
+                     --build-arg BASE_IMAGE=${targetImageTag}-base \
+                     .
+                  """)
+            }
+            if (buildWheel.toBoolean()) {
+              stage("Build wheel") {
+                cudaTag = cudaVer.split('\\.')[0..<2].join('')
+                targetImage.inside() {
+                  sh """
+                  ls .
+                  python setup.py bdist_wheel --dist-dir .
+                  """
+                }
+                pythonVerTag = pythonVer.split('\\.').join('')
+                Integer MinorVal = pythonVer.split('\\.')[1].toInteger()
+                pythonVerAbiTag = (MinorVal < 8) ? pythonVerTag + 'm' : pythonVerTag
+                kaolinVer = sh(script: "cat ./version.txt", returnStdout: true).trim()
+                baseWheelName = "kaolin-${kaolinVer}-cp${pythonVerTag}-cp${pythonVerAbiTag}"
+                wheelName = "${baseWheelName}-linux_x86_64.whl"
+              }
+              stage("Reinstall from wheel") {
+                targetImage = docker.build(
+                    "${targetImageTag}",
+                    """--no-cache --network host -f ./tools/linux/Dockerfile.install_wheel \
+                       --build-arg BASE_IMAGE=${targetImageTag}-base \
+                       --build-arg WHEEL_NAME=${wheelName} \
+                       .
+                    """)
+              }
+              stage("Push wheel to volume") {
+                sh """
+                mkdir -p /mount_binaries/whl/torch-${torchVer}_cu${cudaTag}
+                cp ./${wheelName} /mount_binaries/whl/torch-${torchVer}_cu${cudaTag}/${wheelName}
+                """
+              }
+              stage("Push wheel to artifact") {
+                archiveArtifacts artifacts: "${wheelName}"
+              }
+            }
+            stage("Push") {
+              targetImage.push()
+            }
+          }
+        }
+      } catch (e) {
+        // In case of build failure, we need to update the following tests as we won't run them.
+        for (arch in archsToTest.split(';')) {
+          updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: 'canceled')
+        }
+        throw e
+      }
+      stage("Launch tests") {
+        jobMap = [:]
+        for (arch in archsToTest.split(';')) {
+          jobMap["${arch}"] = prepareUbuntuTestJob(arch)
+        }
+        parallel jobMap
+      }
+    }
+  }
+}
+
+def prepareUbuntuTestJob(arch) {
+  return {
+    stage("Test ${arch}") {
+      build job: "ubuntu_test_template_CI",
+      parameters: [
+        string(name: 'sourceBranch', value: "${sourceBranch}"),
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'imageTag', value: "${targetImageTag}"),
+        string(name: 'arch', value: "${arch}"),
+        string(name: 'commitHash', value: "${commitHash}")
+      ],
+      // This node doesn't need to be held while tests run.
+      wait: false,
+      // Success of this script depend only on successful build
+      // and launch of tests, not successful tests.
+      propagate: false
+    }
+  }
+}

ci/gitlab_jenkins_templates/ubuntu_test_CI.jenkins

@@ -0,0 +1,293 @@
+#!/usr/bin/env groovy
+
+docker_registry_server = ImageTag.split(':')[0..1].join(':')
+currentBuild.displayName = ImageTag.split(':')[2] + "-${arch}"
+currentBuild.description = sourceBranch + ": " + commitHash
+
+if (arch == "MULTI") {
+    gpu_list = """
+            - "A100_PCIE_40GB"
+            - "A100_PCIE_80GB"
+            - "A100_80GB_PCIE"
+            - "A100_PCIE_100GB"
+            - "A10"
+            - "A30"
+            - "A40"
+            - "GA100-E4720-HBM2"
+            - "GA100-E4720-DVT"
+            - "GA104-300-PG142-SKU10-TS3"
+            - "GeForce_RTX_4090"
+            - "H100_80GB_HBM3"
+            - "GH100_P1010_PCIE_CR"
+            - "Tesla_V100_PCIE_32GB"
+    """
+} else {
+    gpu_list = """
+            - "${arch}"
+    """
+}
+
+node_blacklist = """
+            - "a4u8g-0031.ipp1u1.colossus"
+"""
+
+gitlabCommitStatus("test-${configName}-${arch}") {
+
+podTemplate(
+cloud:'sc-ipp-blossom-prod',
+yaml:"""
+apiVersion: v1
+kind: Pod
+spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
+  containers:
+  - name: docker
+    image: ${imageTag}
+    command:
+    - cat
+    resources: 
+        requests:
+          nvidia.com/gpu: 1
+        limits:
+          nvidia.com/gpu: 1
+    tty: true
+    volumeMounts:
+      - mountPath: /mnt
+        name: pvc-mount
+  imagePullSecrets:
+  - name: gitlabcred
+  nodeSelector:
+    kubernetes.io/os: linux
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: "nvidia.com/gpu_type"
+            operator: "In"
+            values:${gpu_list}
+          - key: "kubernetes.io/hostname"
+            operator: "NotIn"
+            values:${node_blacklist}
+          - key: "nvidia.com/driver_version"
+            operator: "NotIn"
+            values:
+            - "545.23"
+""") {
+  node(POD_LABEL) {
+    container("docker") {
+      timeout(time: 60, unit: 'MINUTES') {
+        stage("Install deps") {
+          sh 'pip install -r /kaolin/tools/ci_requirements.txt'
+          sh 'apt update && apt install -y unzip && unzip /kaolin/examples/samples/rendered_clock.zip -d /kaolin/examples/samples/'
+        }
+        def build_passed = true
+        try {
+          stage('Disp info') {
+            sh 'nvidia-smi'
+            sh 'python --version'
+            sh 'lscpu'
+            sh 'free -h --si'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Pytest") {
+            sh '''
+            export KAOLIN_TEST_NVDIFFRAST=1
+            export KAOLIN_TEST_SHAPENETV1_PATH=/mnt/data/ci_shapenetv1
+            export KAOLIN_TEST_SHAPENETV2_PATH=/mnt/data/ci_shapenetv2
+            export KAOLIN_TEST_MODELNET_PATH=/mnt/data/ModelNet
+            export KAOLIN_TEST_SHREC16_PATH=/mnt/data/ci_shrec16
+            pytest --durations=50 --import-mode=importlib -rs --cov=/kaolin/kaolin \
+                --log-disable=PIL.PngImagePlugin \
+                --log-disable=PIL.TiffImagePlugin \
+                --log-disable=kaolin.rep.surface_mesh \
+                /kaolin/tests/python/kaolin
+            '''
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Dash3D") {
+            sh '''
+            pytest -s --cov=/kaolin/kaolin /kaolin/tests/integration/
+            '''
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          if (arch == "TITAN_RTX") {
+            stage("Doc examples") {
+              // using wheel you don't have /kaolin/kaolin
+              sh '''
+              if [ -f "/kaolin/kaolin" ]; then
+                  pytest --doctest-modules --ignore=/kaolin/kaolin/experimental /kaolin/kaolin
+              fi
+              '''
+            }
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        // TUTORIALS
+        try {
+          stage("BBox Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython bbox_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Camera and Rasterization Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython camera_and_rasterization.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("DIB-R Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython dibr_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Diffuse lighting Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython diffuse_lighting.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("DMTet Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython dmtet_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("GLTF Visualizer") {
+            sh 'cd /kaolin/examples/tutorial && ipython gltf_viz.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Interactive Visualizer") {
+            sh 'cd /kaolin/examples/tutorial && ipython interactive_visualizer.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Spherical Gaussian lighting Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython sg_specular_lighting.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Understanding SPCs Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython understanding_spcs_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Working with meshes Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython working_with_meshes.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+
+        // RECIPES
+        try {
+          stage("SPC from Pointcloud Recipe") {
+            sh 'cd /kaolin/examples/recipes/dataload/ && python spc_from_pointcloud.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("SPC Basics Recipe") {
+            sh 'cd /kaolin/examples/recipes/spc/ && python spc_basics.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Occupancy Sampling Recipe") {
+            sh 'cd /kaolin/examples/recipes/preprocess/ && python occupancy_sampling.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Fast Mesh Sampling Recipe") {
+            sh 'cd /kaolin/examples/recipes/preprocess/ && python fast_mesh_sampling.py --shapenet-dir=/mnt/data/ci_shapenetv2/'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("SPC Dual Octree Recipe") {
+            sh 'cd /kaolin/examples/recipes/spc/ && python spc_dual_octree.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("SPC Trilinear Interpolation Recipe") {
+            sh 'cd /kaolin/examples/recipes/spc/ && python spc_trilinear_interp.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("SPC Convolution 3D Recipe") {
+            sh 'cd /kaolin/examples/recipes/spc/ && python spc_conv3d_example.py'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        if (build_passed) {
+          currentBuild.result = "SUCCESS"
+        } else {
+          currentBuild.result = "FAILURE"
+          error "Build failed. See logs..."
+        }
+      }
+    }
+  }
+}
+
+}  // gitlabCommitStatus

ci/gitlab_jenkins_templates/windows_build_CI.jenkins

@@ -0,0 +1,191 @@
+#!/usr/bin/env groovy
+
+// Map from CUDA version to URL to obtain windows installer
+def cuda_version_url = [
+    // CUDA drivers on test machines only available for CUDA version >= 11.0
+    //  see: https://gitlab-master.nvidia.com/ipp/cloud-infra/blossom/dev/windows-gpu-pods/-/tree/master/ContainerDriverSetup
+    // test machines currently are the only option, named 'gpu_tester'
+    //      two machines exist, only the TITAN RTX will pass tests
+    '11.1': 'http://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_456.81_win10.exe',
+    '11.3': 'https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.89_win10.exe',
+    '11.5': 'https://developer.download.nvidia.com/compute/cuda/11.5.2/local_installers/cuda_11.5.2_496.13_windows.exe',
+    '11.6': 'https://developer.download.nvidia.com/compute/cuda/11.6.2/local_installers/cuda_11.6.2_511.65_windows.exe',
+    '11.7': 'https://developer.download.nvidia.com/compute/cuda/11.7.1/local_installers/cuda_11.7.1_516.94_windows.exe',
+    '11.8': 'https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_522.06_windows.exe',
+    '12.1': 'https://developer.download.nvidia.com/compute/cuda/12.1.0/local_installers/cuda_12.1.0_531.14_windows.exe'
+]
+
+
+docker_registry_server = targetImageTag.split(':')[0..1].join(':')
+// This will be the "RUN" displayed on Blue Ocean
+currentBuild.displayName = targetImageTag.split(':')[2]
+// This will be the "MESSAGE" displayed on Blue Ocean
+currentBuild.description = sourceBranch + ": " + commitHash
+
+gitlabCommitStatus("build-${configName}") {
+
+podTemplate(
+cloud:'sc-ipp-blossom-116',
+envVars:[envVar(key:"JENKINS_URL", value:"${env.JENKINS_URL}")],
+yaml:'''
+apiVersion: v1
+kind: Pod
+spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
+  containers:
+  - name: jnlp
+    image: urm.nvidia.com/sw-ipp-blossom-sre-docker-local/jnlp-agent:jdk11-windows
+    env: 
+    - name: JENKINS_AGENT_WORKDIR
+      value: C:/Jenkins/agent
+    - name: DOCKER_HOST
+      value: "win-docker-proxy.blossom-system.svc.cluster.local"
+    - name: DOCKER_TLS_CERTDIR
+      value: ""
+    volumeMounts:
+      - mountPath: c:/mnt
+        name: pvc-mount
+    resources:
+        requests:
+          memory: 32Gi
+        limits:
+          memory: 32Gi
+  imagePullSecrets:
+  - name: gitlabcred
+  nodeSelector:
+    kubernetes.io/os: windows
+''') 
+{
+  node(POD_LABEL) {
+    try {
+      timeout(time: 300, unit: 'MINUTES') {
+        stage("Checkout") {
+          checkout([
+              $class: 'GitSCM',
+              branches: [[name: "${commitHash}"]],
+              // We need submodules
+              extensions: [[
+                  $class: 'SubmoduleOption',
+                  disableSubmodules: false,
+                  parentCredentials: false,
+                  recursiveSubmodules: true,
+                  reference: '',
+                  trackingSubmodules: false
+              ]],
+              userRemoteConfigs: [[
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
+                  url: "${repoUrl}"
+              ]]
+          ])
+        }
+        docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
+          stage("Build base") {
+            cudaUrl = cuda_version_url[cudaVer]
+            baseImage = docker.build(
+                "${targetImageTag}-base",
+                """-m 32g --no-cache -f ./tools/windows/Dockerfile.base \
+                    --build-arg CUDA_VERSION=${cudaVer} \
+                    --build-arg CUDA_URL=${cudaUrl} \
+                    --build-arg PYTHON_VERSION=${pythonVer} \
+                    --build-arg PYTORCH_VERSION=${torchVer} \
+                    .
+                """)
+          }
+
+          if (buildWheel.toBoolean()) {
+            stage("Build with wheel") {
+              targetImage = docker.build(
+                  "${targetImageTag}",
+                  """-m 32g --no-cache -f ./tools/windows/Dockerfile.install_wheel \
+                     --build-arg BASE_IMAGE=${targetImageTag}-base \
+                     .
+                  """
+              )
+            }
+          } else {
+            stage("Build") {
+              targetImage = docker.build(
+                  "${targetImageTag}",
+                  """-m 32g --no-cache -f ./tools/windows/Dockerfile.install \
+                      --build-arg BASE_IMAGE=${targetImageTag}-base \
+                      .
+                  """)
+            }
+          }
+          
+          stage("Push") {
+            targetImage.push()
+          }
+        }
+      }
+    } catch (e) {
+        // In case of build failure, we need to update the following tests as we won't run them.
+        if (buildWheel.toBoolean()) {
+          updateGitlabCommitStatus(name: "test-${configName}", state: 'canceled')
+        } else {
+          for (arch in archsToTest.split(';')) {
+            updateGitlabCommitStatus(name: "test-${configName}-${arch}", state: 'canceled')
+          }
+        }
+        throw e
+    }
+    stage("Launch tests") {
+      jobMap = [:]
+      if (buildWheel.toBoolean()) {
+        jobMap["test"] = prepareWindowsWheelTestJob()
+      } //else { 
+      //  for (arch in archsToTest.split(';')) {
+      //    jobMap["${arch}"] = prepareWindowsTestJob(arch)
+      //  }
+      parallel jobMap
+    }
+  }
+}
+
+}  // gitlabCommitStatus
+
+/*
+def prepareWindowsTestJob(arch) {
+  return {
+    stage("Test ${arch}") {
+      build job: "windows_test_template_CI",
+      parameters: [
+        string(name: 'sourceBranch', value: "${sourceBranch}"),
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'imageTag', value: "${targetImageTag}"),
+        string(name: 'arch', value: "${arch}"),
+        string(name: 'commitHash', value: "${commitHash}"),
+      ],
+      // This node doesn't need to be held while tests run.
+      wait: false,
+      // Success of this script depends only on successful build
+      // and launch of tests, not successful tests.
+      propagate: false
+    }
+  }
+}
+*/
+
+def prepareWindowsWheelTestJob() {
+  return {
+    stage("Test") {
+      build job: "windows_wheels_template_CI",
+      parameters: [
+        string(name: 'sourceBranch', value: "${sourceBranch}"),
+        string(name: 'configName', value: "${configName}"),
+        string(name: 'imageTag', value: "${targetImageTag}"),
+        string(name: 'commitHash', value: "${commitHash}"),
+        string(name: 'torchVer', value: "${torchVer}"),
+        string(name: 'cudaVer', value: "${cudaVer}"),
+      ],
+      // This node doesn't need to be held while tests run.
+      wait: false,
+      // Success of this script depends only on successful build
+      // and launch of tests, not successful tests.
+      propagate: false
+    }
+  }
+}

ci/gitlab_jenkins_templates/windows_test_CI.jenkins

@@ -0,0 +1,297 @@
+#!/usr/bin/env groovy
+
+docker_registry_server = ImageTag.split(':')[0..1].join(':')
+currentBuild.displayName = ImageTag.split(':')[2] + "-${arch}"
+currentBuild.description = sourceBranch + ": " + commitHash
+
+// to manage image secrets:
+// 1) log into docker
+//  docker login gitlab-master.nvidia.com:5005
+// 2) create secret
+//  kubectl create secret docker-registry test-secret -n kaolin --docker-server=gitlab-master.nvidia.com:5005 --docker-username azook --docker-password XXX
+// 3) add to service account
+//  https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account
+//  kubectl patch kaolin-sa default -p '{"imagePullSecrets": [{"name": "myregistrykey"}]}'
+// 4) add to pod template
+
+gitlabCommitStatus("test-${configName}-${arch}") {
+
+podTemplate(cloud:'sc-ipp-blossom-prod', 
+slaveConnectTimeout: 4000,
+yaml: """
+apiVersion: v1
+kind: Pod
+spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
+  containers:
+  - name: jnlp
+    image: jenkins/jnlp-agent:latest-windows
+    env:
+    - name: JENKINS_AGENT_WORKDIR
+      value: C:/Jenkins/agent
+  - name: windows
+    image: ${imageTag}
+    resources:
+      limits:
+        nvidia.com/gpu: 1
+    restartPolicy: Never
+    backoffLimit: 4
+    tty: true
+    volumeMounts:
+      - mountPath: c:/mnt
+        name: pvc-mount
+  imagePullSecrets:
+  - name: gitlabcred
+  nodeSelector:
+    kubernetes.io/os: windows
+    nvidia.com/node_type: ${arch}
+"""
+) 
+{
+  node(POD_LABEL) {
+    container("windows") {
+      if (testWheel.toBoolean()) {
+        stage('Test') {
+          powershell '''
+          python -c "import kaolin; print(kaolin.__version__)"
+          python -c "import torch; print(torch.__version__)"
+          '''
+        }
+        stage('Move wheels') {
+          powersheel '''
+          cudaTag = cudaVer.split('\\.')[0..<2].join('')
+           /tmp/kaolin-*.whl
+          '''
+        }
+        //mv /tmp/mount_binaries/tmp/torch-${torchVer}+cu${cudaTag}
+      } else {
+        stage('Enable cuda') {
+          powershell '''
+          $Env:driver_store=$(ls $($($(Get-WmiObject Win32_VideoController).InstalledDisplayDrivers | sort -Unique).ToString().Split(',')| sort -Unique).ToString().Replace("\\DriverStore\\", "\\HostDriverStore\\")).Directory.FullName
+          cp "$Env:driver_store\\nvcuda64.dll" C:\\Windows\\System32\\nvcuda.dll
+          cp "$Env:driver_store\\nvapi64.dll" C:\\Windows\\System32\\nvapi64.dll
+        '''
+        }
+        stage("Check cuda") {
+          powershell '''
+          dir c:\\
+          dir c:\\kaolin
+          dir c:\\data
+          '''
+          powershell '''
+          c:\\data\\deviceQuery.exe
+          c:\\data\\bandwidthTest.exe
+          '''
+        }
+        stage("Check mount") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              dir c:\\
+              dir c:\\mnt
+            '''
+          }
+        }
+        stage("Fix paging memory") {
+          // addresses this error on Windows with pytorch consuming too much paging memory: https://stackoverflow.com/a/69489193
+          powershell '''
+          python c:\\data\\fixNvPe.py --input=C:\\Users\\Administrator\\miniconda3\\Lib\\site-packages\\torch\\lib\\*.dll
+          '''
+        }
+        stage("Prepare data") {
+          powershell '''
+            python --version
+            Expand-Archive c:\\kaolin\\examples\\samples\\rendered_clock.zip c:\\kaolin\\examples\\samples
+          '''
+        }
+        stage("DIB-R Tutorial") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\tutorial
+              ipython dibr_tutorial.ipynb
+            '''          
+          }
+        }
+        stage("DMTet Tutorial") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\tutorial
+              ipython dmtet_tutorial.ipynb
+            '''
+          }
+        }
+        stage("Understanding SPCs Tutorial") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\tutorial
+              ipython understanding_spcs_tutorial.ipynb  --matplotlib
+            '''
+          }
+        }
+        stage("Diffuse lighting Tutorial") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\tutorial
+              ipython diffuse_lighting.ipynb
+            '''
+          }
+        }
+        stage("Spherical Gaussian lighting Tutorial") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\tutorial
+              ipython sg_specular_lighting.ipynb
+            '''
+          }
+        }
+        // requires nvdiffrast. not currently supported on Windows
+        // stage("Camera and Rasterization Tutorial") {
+        //   catchError(stageResult: "failure") {
+        //     powershell '''
+        //       cd c:\\kaolin\\examples\\tutorial
+        //       ipython camera_and_rasterization.ipynb
+        //     '''
+        //   }
+        // }
+        stage("SPC from Pointcloud Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\dataload
+              python spc_from_pointcloud.py
+            '''
+          }
+        }
+        stage("SPC Basics Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\spc
+              python spc_basics.py
+            '''
+          }
+        }
+        stage("Occupancy Sampling Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\preprocess
+              python occupancy_sampling.py
+            '''
+          }
+        }
+        stage("Fast Mesh Sampling Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\preprocess
+              python fast_mesh_sampling.py --shapenet-dir=c:/mnt/data/ci_shapenetv2/
+            '''
+          }
+        }
+        stage("SPC Dual Octree Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\spc
+              python spc_dual_octree.py
+            '''
+          }
+        }
+        stage("SPC Trilinear Interpolation Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\spc
+              python spc_trilinear_interp.py
+            '''
+          }
+        }
+        stage("SPC Convolution 3D Recipe") {
+          catchError(stageResult: "failure") {
+            powershell '''
+              cd c:\\kaolin\\examples\\recipes\\spc
+              python spc_conv3d_example.py
+            '''
+          }
+        }
+        stage("Run pytest - io") {
+          catchError(stageResult: "failure") {
+            timeout(time: 5, unit: "MINUTES") {
+              powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/io
+              '''
+            }
+          }
+        }
+        stage("Run pytest - metrics") {
+          catchError(stageResult: "failure") {
+            timeout(time: 5, unit: "MINUTES") {
+              powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/metrics
+              '''
+            }
+          }
+        }
+        stage("Run pytest - ops") {
+          catchError(stageResult: "failure") {
+            timeout(time: 50, unit: "MINUTES") {
+              powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/ops
+              '''
+            }
+          }
+        }
+        stage("Run pytest - render") {
+          catchError(stageResult: "failure") {
+            timeout(time: 50, unit: "MINUTES") {
+                powershell '''
+                set CI=true
+                set KAOLIN_TEST_NVDIFFRAST=0
+                pytest -s /kaolin/tests/python/kaolin/render
+              '''
+            }
+          }
+        }
+        stage("Run pytest - rep") {
+          catchError(stageResult: "failure") {
+            timeout(time: 5, unit: "MINUTES") {
+                powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/rep
+              '''
+            }
+          }
+        }
+        stage("Run pytest - utils") {
+          catchError(stageResult: "failure") {
+            timeout(time: 5, unit: "MINUTES") {
+                powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/utils
+              '''
+            }
+          }
+        }
+        stage("Run pytest - visualize") {
+          catchError(stageResult: "failure") {
+            timeout(time: 5, unit: "MINUTES") {
+                powershell '''
+                set CI=true
+                pytest -s /kaolin/tests/python/kaolin/visualize
+              '''
+            }
+          }
+        }
+        stage("Update build status") {
+          // update build result gitlab status
+          //  catchError only updates the pipeline
+          if (currentBuild.getCurrentResult() == "FAILURE") {
+            error "Build failed. See logs..."
+          }
+        }
+      }
+    }
+  }
+}
+
+}  // gitlabCommitStatus

ci/gitlab_jenkins_templates/windows_wheels_CI.jenkins

@@ -0,0 +1,84 @@
+#!/usr/bin/env groovy
+
+docker_registry_server = ImageTag.split(':')[0..1].join(':')
+currentBuild.displayName = ImageTag.split(':')[2]
+currentBuild.description = sourceBranch + ": " + commitHash
+
+// to manage image secrets:
+// 1) log into docker
+//  docker login gitlab-master.nvidia.com:5005
+// 2) create secret
+//  kubectl create secret docker-registry test-secret -n kaolin --docker-server=gitlab-master.nvidia.com:5005 --docker-username azook --docker-password XXX
+// 3) add to service account
+//  https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account
+//  kubectl patch kaolin-sa default -p '{"imagePullSecrets": [{"name": "myregistrykey"}]}'
+// 4) add to pod template
+
+gitlabCommitStatus("test-${configName}") {
+
+podTemplate(cloud:'sc-ipp-blossom-116',
+slaveConnectTimeout: 4000,
+yaml: """
+apiVersion: v1
+kind: Pod
+spec:
+  volumes:
+  - name: pvc-mount
+    persistentVolumeClaim:
+      claimName: 'kaolin-pvc'
+  containers:
+  - name: jnlp
+    image: urm.nvidia.com/sw-ipp-blossom-sre-docker-local/jnlp-agent:jdk11-windows
+    env:
+    - name: JENKINS_AGENT_WORKDIR
+      value: C:/Jenkins/agent
+  - name: windows
+    image: ${imageTag}
+    restartPolicy: Never
+    backoffLimit: 4
+    tty: true
+    volumeMounts:
+      - mountPath: c:/mnt
+        name: pvc-mount
+  imagePullSecrets:
+  - name: gitlabcred
+  nodeSelector:
+    kubernetes.io/os: windows
+"""
+) 
+{
+  node(POD_LABEL) {
+    container("windows") {
+      stage("Basic test") {
+        powershell '''
+          python --version
+          python -c "import kaolin; print(kaolin.__version__)"
+          python -c "import torch; print(torch.__version__)"
+        '''
+      }
+      if (currentBuild.getCurrentResult() != "FAILURE") {
+        stage("Push wheels on volume") {
+          def cudaTag = cudaVer.split('\\.')[0..<2].join('')
+          withEnv(["cudaTag=$cudaTag"]) {
+            powershell '''
+            New-Item -Path /mnt/whl/torch-"$env:torchVer"_cu"$env:cudaTag" -ItemType "directory" -Force
+            '''
+            powershell '''
+            cp /kaolin/kaolin-*.whl /mnt/whl/torch-"$env:torchVer"_cu"$env:cudaTag"/
+            '''
+          }
+        }
+        stage("Push wheels on artifacts") {
+          // archiveArtifacts only take relative path, and the working directory doesn't work in jenkins
+          // So we copy from /kaolin to current dir
+          powershell '''
+          cp /kaolin/kaolin-*.whl .
+          '''
+          archiveArtifacts artifacts: "kaolin-*.whl"
+        }
+      }
+    }
+  }
+}
+
+}  // gitlabCommitStatus