Bug: set explicit file open encoding (#448)

* explicit file encoding

* crlf->lf

Author: jafreck

aztk/internal/cluster_data/node_data.py

@@ -47,7 +47,7 @@ def add_file(self, file: str, zip_dir: str, binary: bool = True):
             return
         if isinstance(file, (str, bytes)):
             full_file_path = Path(file)
-            with io.open(file, 'r') as f:
+            with io.open(file, 'r', encoding='UTF-8') as f:
                 if binary:
                     self.zipf.write(file, os.path.join(zip_dir, full_file_path.name))
                 else:
@@ -70,7 +70,7 @@ def add_dir(self, path: str, dest: str = None, exclude: List[str] = []):
             relative_folder = os.path.relpath(base, path)
             for file in files:
                 if self._includeFile(file, exclude):
-                    with io.open(os.path.join(base, file), 'r') as f:
+                    with io.open(os.path.join(base, file), 'r', encoding='UTF-8') as f:
                         self.zipf.writestr(os.path.join(dest, relative_folder, file), f.read().replace('\r\n', '\n'))
 
     def _add_custom_scripts(self):
@@ -83,7 +83,7 @@ def _add_custom_scripts(self):
                 new_file_name = str(index) + '_' + os.path.basename(custom_script.script)
                 data.append(dict(script=new_file_name, runOn=str(custom_script.run_on)))
                 try:
-                    with io.open(custom_script.script, 'r') as f:
+                    with io.open(custom_script.script, 'r', encoding='UTF-8') as f:
                         self.zipf.writestr(
                             os.path.join(CUSTOM_SCRIPT_FOLDER, new_file_name),
                             f.read().replace('\r\n', '\n'))

aztk/models/plugins/plugin_file.py

@@ -13,7 +13,7 @@ def __init__(self, target: str, local_path: str):
         # TODO handle folders?
 
     def content(self):
-        with open(self.local_path, "r") as f:
+        with open(self.local_path, "r", encoding='UTF-8') as f:
             return  f.read()
 
 

aztk/node_scripts/install/create_user.py

@@ -16,7 +16,7 @@ def create_user(batch_client):
         print("No user to create.")
         return
 
-    with open(path) as file:
+    with open(path, 'r', encoding='UTF-8') as file:
         user_conf = yaml.load(file.read())
 
     try:
@@ -43,7 +43,8 @@ def decrypt_password(user_conf):
     tag = user_conf['tag']
 
     # Read private key
-    private_key = RSA.import_key(open(os.path.join(os.environ['DOCKER_WORKING_DIR'], 'id_rsa')).read())
+    with open(os.path.join(os.environ['DOCKER_WORKING_DIR'], 'id_rsa'), encoding='UTF-8') as f:
+        private_key = RSA.import_key(f.read())
     # Decrypt the session key with the public RSA key
     cipher_rsa = PKCS1_OAEP.new(private_key)
     session_key = cipher_rsa.decrypt(encrypted_aes_session_key)

aztk/node_scripts/install/plugins.py

@@ -12,7 +12,7 @@ def _read_manifest_file(path=None):
     if not os.path.isfile(path):
         print("Plugins manifest file doesn't exist at {0}".format(path))
     else:
-        with open(path, 'r') as stream:
+        with open(path, 'r', encoding='UTF-8') as stream:
             try:
                 custom_scripts = yaml.load(stream)
             except json.JSONDecodeError as err:
@@ -86,7 +86,7 @@ def _run_script(name: str, script_path: str = None, args: dict = None, env: dict
     if args is None:
         args = []
 
-    out_file = open(os.path.join(log_folder, '{0}.txt'.format(name)), 'w')
+    out_file = open(os.path.join(log_folder, '{0}.txt'.format(name)), 'w', encoding='UTF-8')
     try:
         subprocess.call(
             [script_path] + args,

aztk/node_scripts/install/scripts.py

@@ -9,7 +9,7 @@ def _read_yaml_file(path=None):
     if not os.path.isfile(path):
         print("Configuration file doesn't exist at {0}".format(path))
     else:
-        with open(path, 'r') as stream:
+        with open(path, 'r', encoding='UTF-8') as stream:
             try:
                 custom_scripts = yaml.load(stream)
             except yaml.YAMLError as err:

aztk/node_scripts/install/spark.py

@@ -43,7 +43,7 @@ def setup_connection():
     master_node = get_node(master_node_id)
 
     master_config_file = os.path.join(spark_conf_folder, "master")
-    master_file = open(master_config_file, 'w')
+    master_file = open(master_config_file, 'w', encoding='UTF-8')
 
     print("Adding master node ip {0} to config file '{1}'".format(
         master_node.ip_address, master_config_file))
@@ -195,7 +195,7 @@ def copy_jars():
 
 def parse_configuration_file(path_to_file: str):
     try:
-        file = open(path_to_file, 'r')
+        file = open(path_to_file, 'r', encoding='UTF-8')
         properties = {}
         for line in file:
             if (not line.startswith('#') and len(line) > 1):

aztk/node_scripts/job_submission.py

@@ -27,7 +27,7 @@ def schedule_tasks(tasks_path):
     blob_client = config.blob_client
 
     for task_definition in tasks_path:
-        with open(task_definition, 'r') as stream:
+        with open(task_definition, 'r', encoding='UTF-8') as stream:
             try:
                 task = yaml.load(stream)
             except yaml.YAMLError as exc:

aztk/node_scripts/submit.py

@@ -124,7 +124,7 @@ def load_application(application_file_path):
     '''
         Read and parse the application from file
     '''
-    with open(application_file_path) as f:
+    with open(application_file_path, encoding='UTF-8') as f:
         application = yaml.load(f)
     return application
 
@@ -176,7 +176,8 @@ def upload_error_log(error, application_file_path):
     application = load_application(application_file_path)
     blob_client = config.blob_client
 
-    with open(os.path.join(os.environ["AZ_BATCH_TASK_WORKING_DIR"], "error.log"), "w") as error_log:
+    error_log_path = os.path.join(os.environ["AZ_BATCH_TASK_WORKING_DIR"], "error.log")
+    with open(error_log_path, "w", encoding='UTF-8') as error_log:
         error_log.write(error)
 
     upload_file_to_container(

aztk/utils/get_ssh_key.py

@@ -28,6 +28,6 @@ def __read_ssh_key_from_file(path: str) -> str:
     """
         Read the content of the given file
     """
-    with open(os.path.expanduser(path), 'r') as content_file:
+    with open(os.path.expanduser(path), 'r', encoding='UTF-8') as content_file:
         content = content_file.read()
         return content

aztk_cli/config.py

@@ -46,7 +46,7 @@ def _load_secrets_config(
     if not os.path.isfile(path):
         return None
 
-    with open(path, 'r') as stream:
+    with open(path, 'r', encoding='UTF-8') as stream:
         try:
             return yaml.load(stream)
         except yaml.YAMLError as err:
@@ -129,7 +129,7 @@ def read_cluster_config(
     if not os.path.isfile(path):
         return
 
-    with open(path, 'r') as stream:
+    with open(path, 'r', encoding='UTF-8') as stream:
         try:
             config_dict = yaml.load(stream)
         except yaml.YAMLError as err:
@@ -224,7 +224,7 @@ def _read_config_file(
         if not os.path.isfile(path):
             return
 
-        with open(path, 'r') as stream:
+        with open(path, 'r', encoding='UTF-8') as stream:
             try:
                 config = yaml.load(stream)
             except yaml.YAMLError as err:
@@ -359,7 +359,7 @@ def _read_config_file(
         if not path or not os.path.isfile(path):
             return
 
-        with open(path, 'r') as stream:
+        with open(path, 'r', encoding='UTF-8') as stream:
             try:
                 config = yaml.load(stream)
             except yaml.YAMLError as err: