Allow ignoring big files (by extension)

config.ini.sample

@@ -39,4 +39,10 @@ UseAutomaticConflictResolution = False
 
 [Miscellaneous]
 # Set to true if you want to see which commands are sent to command line
-LogShellCommands = False
+LogShellCommands = False
+
+# Ignore big (binary) files
+# Define a semicolon-separated list of extensions to be generally ignored
+# Example:
+# IgnoreFileExtensions = .zip; .jar; .exe; .dll
+IgnoreFileExtensions =

configuration.py

@@ -14,12 +14,13 @@ def read(configname="config.ini"):
     parsedconfig.read(configname)
     generalsection = parsedconfig['General']
     migrationsection = parsedconfig['Migration']
+    miscellaneoussection = parsedconfig['Miscellaneous']
 
     user = generalsection['User']
     password = generalsection['Password']
     repositoryurl = generalsection['Repo']
     scmcommand = generalsection['ScmCommand']
-    shell.logcommands = parsedconfig['Miscellaneous']['LogShellCommands'] == "True"
+    shell.logcommands = miscellaneoussection['LogShellCommands'] == "True"
     shell.setencoding(generalsection['encoding'])
 
     workspace = shlex.quote(generalsection['WorkspaceName'])
@@ -33,13 +34,15 @@ def read(configname="config.ini"):
     streamname = shlex.quote(migrationsection['StreamToMigrate'].strip())
     previousstreamname = migrationsection['PreviousStream'].strip()
     baselines = getinitialcomponentbaselines(migrationsection['InitialBaseLines'])
+    ignorefileextensions = miscellaneoussection['IgnoreFileExtensions']
 
     configbuilder = Builder().setuser(user).setpassword(password).setrepourl(repositoryurl).setscmcommand(scmcommand)
     configbuilder.setworkspace(workspace).setgitreponame(gitreponame).setrootfolder(os.getcwd())
     configbuilder.setuseexistingworkspace(useexistingworkspace).setuseprovidedhistory(useprovidedhistory)
     configbuilder.setuseautomaticconflictresolution(useautomaticconflictresolution)
     configbuilder.setworkdirectory(workdirectory).setstreamname(streamname).setinitialcomponentbaselines(baselines)
     configbuilder.setpreviousstreamname(previousstreamname)
+    configbuilder.setignorefileextensions(ignorefileextensions)
     global config
     config = configbuilder.build()
     return config
@@ -88,6 +91,7 @@ def __init__(self):
         self.gitreponame = ""
         self.clonedgitreponame = ""
         self.previousstreamname = ""
+        self.ignorefileextensions = ""
 
     def setuser(self, user):
         self.user = user
@@ -150,6 +154,10 @@ def setpreviousstreamname(self, previousstreamname):
         self.previousstreamname = previousstreamname
         return self
 
+    def setignorefileextensions(self, ignorefileextensions):
+        self.ignorefileextensions = ignorefileextensions
+        return self
+
     @staticmethod
     def isenabled(stringwithbooleanexpression):
         return stringwithbooleanexpression == "True"
@@ -159,13 +167,14 @@ def build(self):
                             self.useexistingworkspace, self.workdirectory, self.initialcomponentbaselines,
                             self.streamname, self.gitreponame, self.useprovidedhistory,
                             self.useautomaticconflictresolution, self.clonedgitreponame, self.rootFolder,
-                            self.previousstreamname)
+                            self.previousstreamname, self.ignorefileextensions)
 
 
 class ConfigObject:
     def __init__(self, user, password, repourl, scmcommand, workspace, useexistingworkspace, workdirectory,
                  initialcomponentbaselines, streamname, gitreponame, useprovidedhistory,
-                 useautomaticconflictresolution, clonedgitreponame, rootfolder, previousstreamname):
+                 useautomaticconflictresolution, clonedgitreponame, rootfolder, previousstreamname,
+                 ignorefileextensionsproperty):
         self.user = user
         self.password = password
         self.repo = repourl
@@ -185,6 +194,7 @@ def __init__(self, user, password, repourl, scmcommand, workspace, useexistingwo
         self.streamuuid = ""
         self.previousstreamname = previousstreamname
         self.previousstreamuuid = ""
+        self.ignorefileextensions = ConfigObject.parseignorefileextensionsproperty(ignorefileextensionsproperty)
 
     def getlogpath(self, filename):
         if not self.hasCreatedLogFolder:
@@ -216,6 +226,20 @@ def collectstreamuuids(self):
         self.streamuuid = self.collectstreamuuid(self.streamname)
         self.previousstreamuuid = self.collectstreamuuid(self.previousstreamname)
 
+    @staticmethod
+    def parseignorefileextensionsproperty(ignorefileextensionsproperty):
+        """
+        :param ignorefileextensionsproperty
+        :return: a list of file extensions to be ignored, possibly empty
+        """
+        splittedextensions = []
+        if ignorefileextensionsproperty and len(ignorefileextensionsproperty) > 0:
+            splittedextensions = ignorefileextensionsproperty.split(';')
+        ignorefileextensions = []
+        for splittedextension in splittedextensions:
+            ignorefileextensions.append(splittedextension.strip())
+        return ignorefileextensions
+
 
 class ComponentBaseLineEntry:
     def __init__(self, component, baseline, componentname, baselinename):

gitFunctions.py

@@ -61,6 +61,7 @@ class Commiter:
 
     @staticmethod
     def addandcommit(changeentry):
+        Commiter.filterignore()
         Commiter.replaceauthor(changeentry.author, changeentry.email)
         shell.execute("git add -A")
 
@@ -78,23 +79,19 @@ def addandcommit(changeentry):
     def handle_captitalization_filename_changes():
         sandbox = os.path.join(configuration.get().workDirectory, configuration.get().clonedGitRepoName)
         lines = shell.getoutput("git status -z")
-        for line in lines:
-            for entry in line.split(sep='\x00'):  # ascii 0 is the delimiter
-                entry = entry.strip()
-                if entry.startswith("A "):
-                    newfilerelativepath = entry[3:]  # cut A and following space and NUL at the end
-                    directoryofnewfile = os.path.dirname(os.path.join(sandbox, newfilerelativepath))
-                    newfilename = os.path.basename(newfilerelativepath)
-                    cwd = os.getcwd()
-                    os.chdir(directoryofnewfile)
-                    files = shell.getoutput("git ls-files")
-                    for previousFileName in files:
-                        was_same_file_name = newfilename.lower() == previousFileName.lower()
-                        file_was_renamed = newfilename != previousFileName
-
-                        if was_same_file_name and file_was_renamed:
-                            shell.execute("git rm --cached %s" % previousFileName)
-                    os.chdir(cwd)
+        for newfilerelativepath in Commiter.splitoutputofgitstatusz(lines, "A  "):
+            directoryofnewfile = os.path.dirname(os.path.join(sandbox, newfilerelativepath))
+            newfilename = os.path.basename(newfilerelativepath)
+            cwd = os.getcwd()
+            os.chdir(directoryofnewfile)
+            files = shell.getoutput("git ls-files")
+            for previousFileName in files:
+                was_same_file_name = newfilename.lower() == previousFileName.lower()
+                file_was_renamed = newfilename != previousFileName
+
+                if was_same_file_name and file_was_renamed:
+                    shell.execute("git rm --cached %s" % previousFileName)
+            os.chdir(cwd)
 
     @staticmethod
     def getcommitcommand(changeentry):
@@ -163,8 +160,79 @@ def promotebranchtomaster(branchname):
             shouter.shout("Branch %s couldnt get renamed to master, please do that on your own" % branchname)
             return 1  # branch couldnt get renamed
 
+    @staticmethod
+    def filterignore():
+        """
+        add files with extensions to be ignored to .gitignore
+        """
+        # there is only work to to if there are extensions configured at all
+        ignorefileextensions = configuration.get().ignorefileextensions
+        if len(ignorefileextensions) > 0:
+            # make sure we see all untracked files:
+            strippedlines = shell.getoutput('git status --untracked-files=all --porcelain -z')
+            repositoryfiles = Commiter.splitoutputofgitstatusz(strippedlines)
+            Commiter.ignore(ExtensionFilter.match(repositoryfiles, ignorefileextensions))
+
+    @staticmethod
+    def ignore(filelines):
+        """
+        append the file lines to the toplevel .gitignore
+        :param filelines: a list of newline terminated file names to be ignored
+        """
+        if len(filelines) > 0:
+            with open(".gitignore", "a") as ignore:
+                ignore.writelines(filelines)
+
+    @staticmethod
+    def splitoutputofgitstatusz(lines, filterprefix=None):
+        """
+        Split the output of  'git status -z' into single files
+
+        :param lines: the output line(s) from the command
+        :param filterprefix: if given, only the files of those entries matching the prefix will be returned
+        :return: a list of repository files with status changes
+        """
+        repositoryfiles = []
+        for line in lines:                           # expect exactly one line
+            entries = line.split(sep='\x00')         # ascii 0 is the delimiter
+            for entry in entries:
+                entry = entry.strip()
+                if len(entry) > 0:
+                    if not filterprefix or entry.startswith(filterprefix):
+                        start = entry.find(' ')
+                        if 1 <= start <= 2:
+                            repositoryfile = entry[3:]   # output is formatted
+                        else:
+                            repositoryfile = entry       # file on a single line (e.g. rename continuation)
+                        repositoryfiles.append(repositoryfile)
+        return repositoryfiles
+
 
 class Differ:
     @staticmethod
     def has_diff():
         return shell.execute("git diff --quiet") is 1
+
+
+class ExtensionFilter:
+
+    @staticmethod
+    def match(repositoryfiles, extensions):
+        """
+        Determine the repository files to ignore.
+        These filenames are returned as a list of newline terminated lines,
+        ready to be added to .gitignore with writelines()
+
+        :param repositoryfiles: a list of (changed) files
+        :param extensions the extensions to be ignored
+        :return: a list of newline terminated file names, possibly empty
+        """
+        repositoryfilestoignore = []
+        for extension in extensions:
+          for repositoryfile in repositoryfiles:
+                extlen = len(extension)
+                if len(repositoryfile) >= extlen:
+                    if repositoryfile[-extlen:] == extension:
+                        # escape a backslash with a backslash, and append a newline
+                        repositoryfilestoignore.append(repositoryfile.replace('\\', '\\\\') + '\n')
+        return repositoryfilestoignore

tests/resources/test_config.ini

@@ -0,0 +1,21 @@
+[General]
+Repo=https://rtc.supercompany.com/ccm/
+User=superuser
+Password=supersecret
+GIT-Reponame = super.git
+WorkspaceName=Superworkspace
+Directory = /tmp/migration
+useExistingWorkspace = True
+ScmCommand = lscm
+encoding = UTF-8
+
+[Migration]
+StreamToMigrate = Superstream
+PreviousStream = Previousstream
+InitialBaseLines = Component1=Baseline1, Component2=Baseline2
+UseProvidedHistory = True
+UseAutomaticConflictResolution = True
+
+[Miscellaneous]
+LogShellCommands = True
+IgnoreFileExtensions = .zip; .jar

tests/test_configuration.py

@@ -3,6 +3,7 @@
 
 from configuration import Builder
 import configuration
+import shell
 
 
 class ConfigurationTestCase(unittest.TestCase):
@@ -38,3 +39,55 @@ def test_getSampleConfig_ExpectInitializedConfigWithDefaultValues(self):
         config = configuration.read("../config.ini.sample")
         self.assertEqual("lscm", config.scmcommand)
         self.assertEqual(config, configuration.get())
+
+    def test_fileExtensionsToBeIgnored_ShouldBeEmpty_FromNone(self):
+        config = Builder().setignorefileextensions(None).build()
+        self.assertTrue(len(config.ignorefileextensions) == 0)
+
+    def test_fileExtensionsToBeIgnored_ShouldBeEmpty_FromEmpty(self):
+        config = Builder().setignorefileextensions("").build()
+        self.assertTrue(len(config.ignorefileextensions) == 0)
+
+    def test_fileExtensionsToBeIgnored_SingleExtensions(self):
+        config = Builder().setignorefileextensions(" .zip  ").build()
+        self.assertTrue(len(config.ignorefileextensions) == 1)
+        self.assertEqual(['.zip'], config.ignorefileextensions)
+
+    def test_fileExtensionsToBeIgnored_MultipleExtensions(self):
+        config = Builder().setignorefileextensions(".zip; .jar;  .exe").build()
+        self.assertTrue(len(config.ignorefileextensions) == 3)
+        self.assertEqual(['.zip', '.jar', '.exe'], config.ignorefileextensions)
+
+    def test_read(self):
+        # [General]
+        config = configuration.read('resources/test_config.ini')
+        self.assertEqual('https://rtc.supercompany.com/ccm/', config.repo)
+        self.assertEqual('superuser', config.user)
+        self.assertEqual('supersecret', config.password)
+        self.assertEqual('super.git', config.gitRepoName)
+        self.assertEqual('Superworkspace', config.workspace)
+        self.assertEqual('/tmp/migration', config.workDirectory)
+        self.assertTrue(config.useexistingworkspace)
+        self.assertEqual('lscm', config.scmcommand)
+        self.assertEqual('UTF-8', shell.encoding)  # directly deviated to shell
+        # [Migration]
+        self.assertEqual('Superstream', config.streamname)
+        self.assertEqual('Previousstream', config.previousstreamname)
+        initialcomponentbaselines = config.initialcomponentbaselines
+        self.assertEqual(2, len(initialcomponentbaselines))
+        initialcomponentbaseline = initialcomponentbaselines[0]
+        self.assertEqual('Component1', initialcomponentbaseline.componentname)
+        self.assertEqual('Baseline1', initialcomponentbaseline.baselinename)
+        initialcomponentbaseline = initialcomponentbaselines[1]
+        self.assertEqual('Component2', initialcomponentbaseline.componentname)
+        self.assertEqual('Baseline2', initialcomponentbaseline.baselinename)
+        self.assertTrue(config.useprovidedhistory)
+        self.assertTrue(config.useautomaticconflictresolution)
+        # [Miscellaneous]
+        self.assertTrue(shell.logcommands)  # directly deviated to shell
+        ignorefileextensions = config.ignorefileextensions
+        self.assertEqual(2, len(ignorefileextensions))
+        self.assertEqual('.zip', ignorefileextensions[0])
+        self.assertEqual('.jar', ignorefileextensions[1])
+
+

tests/test_gitFunctions.py

@@ -116,6 +116,64 @@ def test_CopyBranch_TargetAlreadyExist_ShouldFail(self):
             Commiter.branch(branchname)
             self.assertFalse(Commiter.copybranch("master", branchname) is 0)
 
+    def test_splitoutputofgitstatusz(self):
+        with open('./resources/test_ignore_git_status_z.txt', 'r') as file:
+            repositoryfiles = Commiter.splitoutputofgitstatusz(file.readlines())
+            self.assertEqual(12, len(repositoryfiles))
+            self.assertEqual('project1/src/tobedeleted.txt', repositoryfiles[0])
+            self.assertEqual('project2/src/taka.txt', repositoryfiles[1])
+            self.assertEqual('project1/src/taka.txt', repositoryfiles[2]) # rename continuation would bite here
+            self.assertEqual('project2/src/takatuka.txt', repositoryfiles[3])
+            self.assertEqual('project2/src/tuka.txt', repositoryfiles[4])
+            self.assertEqual('project1/src/sub/kling -- klong.zip', repositoryfiles[5])
+            self.assertEqual('project1/src/sub/kling :and: klong.zip', repositoryfiles[6])
+            self.assertEqual('project1/src/sub/kling ;and; klong.zip', repositoryfiles[7])
+            self.assertEqual('project1/src/sub/kling >and< klong.zip', repositoryfiles[8])
+            self.assertEqual('project1/src/sub/kling \\and\\ klong.zip', repositoryfiles[9])
+            self.assertEqual('project1/src/sub/kling |and| klong.zip', repositoryfiles[10])
+            self.assertEqual('project1/src/sub/klingklong.zip', repositoryfiles[11])
+
+    def test_splitoutputofgitstatusz_filterprefix_A(self):
+        with open('./resources/test_ignore_git_status_z.txt', 'r') as file:
+            repositoryfiles = Commiter.splitoutputofgitstatusz(file.readlines(), 'A  ')
+            self.assertEqual(1, len(repositoryfiles))
+            self.assertEqual('project1/src/tobedeleted.txt', repositoryfiles[0])
+
+    def test_splitoutputofgitstatusz_filterprefix_double_question(self):
+        with open('./resources/test_ignore_git_status_z.txt', 'r') as file:
+            repositoryfiles = Commiter.splitoutputofgitstatusz(file.readlines(), '?? ')
+            self.assertEqual(7, len(repositoryfiles))
+            self.assertEqual('project1/src/sub/kling -- klong.zip', repositoryfiles[0])
+            self.assertEqual('project1/src/sub/kling :and: klong.zip', repositoryfiles[1])
+            self.assertEqual('project1/src/sub/kling ;and; klong.zip', repositoryfiles[2])
+            self.assertEqual('project1/src/sub/kling >and< klong.zip', repositoryfiles[3])
+            self.assertEqual('project1/src/sub/kling \\and\\ klong.zip', repositoryfiles[4])
+            self.assertEqual('project1/src/sub/kling |and| klong.zip', repositoryfiles[5])
+            self.assertEqual('project1/src/sub/klingklong.zip', repositoryfiles[6])
+
+    def test_filterignore(self):
+        with testhelper.mkchdir("aFolder") as folder:
+            # create test repo
+            configuration.config = Builder().setworkdirectory(folder).setgitreponame("test.git").setignorefileextensions('.zip; .jar').build()
+            ignore = '.gitignore'
+            Initializer().createrepo()
+            # simulate addition of .zip and .jar files
+            zip = 'test.zip'
+            with open(zip, 'w') as testzip:
+                testzip.write('test zip content')
+            jar = 'test.jar'
+            with open(jar, 'w') as testjar:
+                testjar.write('test jar content')
+            # do the filtering
+            Commiter.filterignore()
+            # check output of .gitignore
+            with open(ignore, 'r') as gitIgnore:
+                lines = gitIgnore.readlines()
+                self.assertEqual(2, len(lines))
+                lines.sort()
+                self.assertEqual(jar, lines[0].strip())
+                self.assertEqual(zip, lines[1].strip())
+
     def simulateCreationAndRenameInGitRepo(self, originalfilename, newfilename):
         open(originalfilename, 'a').close()  # create file
         Initializer.initialcommit()