Feature: Readthedocs support (#497)

Author: timotheeguerin

.editorconfig

@@ -3,6 +3,7 @@ indent_style = space
 indent_size = 4
 insert_final_newline = true
 trim_trailing_whitespace = true
+end_of_line = lf
 
 [*.{json,yml,yaml}]
 indent_size = 2

.gitignore

@@ -39,3 +39,7 @@ tmp/
 
 # PyTest
 .cache/
+
+
+# Built docs
+docs/_build/

.vscode/settings.json

@@ -15,6 +15,5 @@
   "python.formatting.yapfArgs": [
     "--style=.style.yapf"
   ],
-  "python.venvPath": "${workspaceFolder}/ENV",
   "python.pythonPath": "${workspaceFolder}\\ENV\\Scripts\\python.exe"
 }

aztk/client.py

@@ -61,7 +61,7 @@ def __delete_pool_and_job(self, pool_id: str, keep_logs: bool = False):
 
         if pool_exists:
             self.batch_client.pool.delete(pool_id)
-        
+
         if not keep_logs:
             cluster_data = self._get_cluster_data(pool_id)
             cluster_data.delete_container(pool_id)

aztk/error.py

@@ -4,10 +4,8 @@
 """
 
 
-
 class AztkError(Exception):
-    def __init__(self, message: str=None):
-        super().__init__(message)
+    pass
 
 class ClusterNotReadyError(AztkError):
     pass

aztk/internal/cluster_data/__init__.py

@@ -1,3 +1,3 @@
-from .blob_data import *
-from .node_data import *
-from .cluster_data import *
+from .blob_data import BlobData
+from .node_data import NodeData
+from .cluster_data import ClusterData

aztk/internal/cluster_data/cluster_data.py

@@ -1,7 +1,6 @@
-import yaml
 import logging
+import yaml
 import azure.common
-from azure.storage.blob import BlockBlobService
 from .node_data import NodeData
 from .blob_data import BlobData
 
@@ -15,7 +14,7 @@ class ClusterData:
     APPLICATIONS_DIR = "applications"
     CLUSTER_CONFIG_FILE = "config.yaml"
 
-    def __init__(self, blob_client: BlockBlobService, cluster_id: str):
+    def __init__(self, blob_client, cluster_id: str):
         self.blob_client = blob_client
         self.cluster_id = cluster_id
         self._ensure_container()

aztk/internal/cluster_data/node_data.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import List
 import yaml
-from aztk.spark import models
+from aztk import models
 from aztk.utils import constants, file_utils, secure_utils
 from aztk.error import InvalidCustomScriptError
 
@@ -61,10 +61,12 @@ def add_files(self, file_paths: List[str], zip_dir, binary: bool = True):
         for file in file_paths:
             self.add_file(file, zip_dir, binary)
 
-    def add_dir(self, path: str, dest: str = None, exclude: List[str] = []):
+    def add_dir(self, path: str, dest: str = None, exclude: List[str] = None):
         """
             Zip all the files in the given directory into the zip file handler
         """
+        exclude = exclude or []
+
         for base, _, files in os.walk(path):
             relative_folder = os.path.relpath(base, path)
             for file in files:
@@ -156,7 +158,8 @@ def _add_plugins(self):
     def _add_node_scripts(self):
         self.add_dir(os.path.join(ROOT_PATH, NODE_SCRIPT_FOLDER), NODE_SCRIPT_FOLDER, exclude=['*.pyc*'])
 
-    def _includeFile(self, filename: str, exclude: List[str] = []) -> bool:
+    def _includeFile(self, filename: str, exclude: List[str]) -> bool:
+        exclude = exclude or []
         for pattern in exclude:
             if fnmatch.fnmatch(filename, pattern):
                 return False

aztk/spark/__init__.py

@@ -1,2 +1 @@
-from .models import *
 from .client import Client

aztk/spark/client.py

@@ -15,13 +15,27 @@
 
 
 class Client(BaseClient):
+    """
+    Aztk Spark Client
+    This is the main entry point for using aztk for spark
+
+    Args:
+        secrets_config(aztk.spark.models.models.SecretsConfiguration): Configuration with all the needed credentials
+    """
     def __init__(self, secrets_config):
         super().__init__(secrets_config)
 
-    '''
-    Spark client public interface
-    '''
     def create_cluster(self, cluster_conf: models.ClusterConfiguration, wait: bool = False):
+        """
+        Create a new aztk spark cluster
+
+        Args:
+            cluster_conf(aztk.spark.models.models.ClusterConfiguration): Configuration for the the cluster to be created
+            wait(bool): If you should wait for the cluster to be ready before returning
+
+        Returns:
+            aztk.spark.models.Cluster
+        """
         cluster_conf.validate()
         cluster_data = self._get_cluster_data(cluster_conf.cluster_id)
         try: