Feature: Add VSTS CI (#561)

* fix job submission bug and integration tests

* merge

* update tests, add vsts-ci.yml

* add python step

* debug statements

* update build

* update build add print

* build update

* fix bug

* debug

* undo

* parallelize build

* add trigger

* typo

* remove env

* whitespace

* whitespace

* whitespace

* remove debug branch

Author: jafreck

.vscode/launch.json

@@ -35,7 +35,7 @@
             "spark", "cluster", "create", "--id", "spark-debug"
           ],
           "env": {},
-          "envFile": "${workspaceFolder}/.env",
+          "envFile": "${workspaceFolder}/.venv",
           "debugOptions": [
               "RedirectOutput"
           ]

.vsts-ci.yml

@@ -0,0 +1,20 @@
+trigger:
+  - master
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '>= 3.5' 
+    addToPath: true 
+    architecture: 'x64'
+
+- script: |
+    pip install -r requirements.txt
+    pip install -e .
+  condition: and(succeeded(), eq(variables['agent.os'], 'linux'))
+  displayName: install aztk
+
+- script: |
+    pytest -n 50
+  condition: and(succeeded(), in(variables['agent.os'], 'linux'))
+  displayName: pytest

aztk/spark/helpers/job_submission.py

@@ -22,7 +22,8 @@ def __app_cmd():
     docker_exec.add_argument("spark /bin/bash >> output.log 2>&1 -c \"" \
                              "source ~/.bashrc; " \
                              "export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; " \
-                             "$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
+                             "cd \$AZ_BATCH_TASK_WORKING_DIR; " \
+                             "\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
     return docker_exec.to_str()
 
 

aztk/spark/helpers/submit.py

@@ -84,7 +84,7 @@ def generate_task(spark_client, container_id, application):
     task_cmd.add_argument('spark /bin/bash >> output.log 2>&1')
     task_cmd.add_argument('-c "source ~/.bashrc; ' \
                           'export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; ' \
-                          'cd $AZ_BATCH_TASK_WORKING_DIR; ' \
+                          'cd \$AZ_BATCH_TASK_WORKING_DIR; ' \
                           '\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/submit.py"')
 
     # Create task

aztk/spark/models/models.py

@@ -8,6 +8,7 @@
 class SparkToolkit(aztk.models.Toolkit):
     def __init__(self, version: str, environment: str = None, environment_version: str = None):
         super().__init__(
+            software="spark",
             version=version,
             environment=environment,
             environment_version=environment_version,

aztk/spark/utils/util.py

@@ -45,4 +45,3 @@ def wait_for_master_to_be_ready(client, cluster_id: str):
                     "Master didn't become ready before timeout.")
 
             time.sleep(10)
-    time.sleep(5)