Feature: Add VSTS CI by jafreck · Pull Request #561 · Azure/aztk

.vscode/launch.json

-Original file line number
+Diff line change
@@ Expand Up / @@ -35,7 +35,7 @@ @@
                 "spark", "cluster", "create", "--id", "spark-debug"
               ],
               "env": {},
-              "envFile": "${workspaceFolder}/.env",
+              "envFile": "${workspaceFolder}/.venv",
               "debugOptions": [
                   "RedirectOutput"
               ]
@@ Expand Down @@

.vsts-ci.yml

-Original file line number
+Diff line change
@@ -0,0 +1,20 @@
+    trigger:
+      - master
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '>= 3.5'
+        addToPath: true
+        architecture: 'x64'
+    - script: |
+        pip install -r requirements.txt
+        pip install -e .
+      condition: and(succeeded(), eq(variables['agent.os'], 'linux'))
+      displayName: install aztk
+    - script: |
+        pytest -n 50
+      condition: and(succeeded(), in(variables['agent.os'], 'linux'))
+      displayName: pytest

aztk/spark/helpers/job_submission.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -22,7 +22,8 @@ def __app_cmd(): @@
         docker_exec.add_argument("spark /bin/bash >> output.log 2>&1 -c \"" \
                                  "source ~/.bashrc; " \
                                  "export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; " \
-                                 "$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
+                                 "cd \$AZ_BATCH_TASK_WORKING_DIR; " \
+                                 "\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
         return docker_exec.to_str()
@@ Expand Down @@

aztk/spark/helpers/submit.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -84,7 +84,7 @@ def generate_task(spark_client, container_id, application): @@
         task_cmd.add_argument('spark /bin/bash >> output.log 2>&1')
         task_cmd.add_argument('-c "source ~/.bashrc; ' \
                               'export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; ' \
-                              'cd $AZ_BATCH_TASK_WORKING_DIR; ' \
+                              'cd \$AZ_BATCH_TASK_WORKING_DIR; ' \
                               '\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/submit.py"')
         # Create task
@@ Expand Down @@

aztk/spark/models/models.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -8,6 +8,7 @@ @@
     class SparkToolkit(aztk.models.Toolkit):
         def __init__(self, version: str, environment: str = None, environment_version: str = None):
             super().__init__(
+                software="spark",
                 version=version,
                 environment=environment,
                 environment_version=environment_version,
@@ Expand Down @@

aztk/spark/utils/util.py

Original file line number	Diff line number	Diff line change
Expand Up		@@ -45,4 +45,3 @@ def wait_for_master_to_be_ready(client, cluster_id: str):
		"Master didn't become ready before timeout.")

		time.sleep(10)
		time.sleep(5)

Provide feedback