Track which attachments are actually patches

This weill eventually get exposed as some sort of API for a buildbot,
but we might as well track it and show it on the website already now.

diff --git a/pgcommitfest/commitfest/ajax.py b/pgcommitfest/commitfest/ajax.py
index 7c73771f6f1f9f05b9c7ddc13df93dbc8a63cbf4..469d630f7aceed4eb8308723d2cf1bbe98869162 100644 (file)
--- a/pgcommitfest/commitfest/ajax.py
+++ b/pgcommitfest/commitfest/ajax.py
@@ -58,12 +58,16 @@ def getThreads(request):
 
 def parse_and_add_attachments(threadinfo, mailthread):
        for t in threadinfo:
-               if t['att']:
+               if len(t['atts']):
+                       # One or more attachments. For now, we're only actually going
+                       # to store and process the first one, even though the API gets
+                       # us all of them.
                        MailThreadAttachment.objects.get_or_create(mailthread=mailthread,
                                                                                                           messageid=t['msgid'],
                                                                                                           defaults={
                                                                                                                   'date': t['date'],
-                                                                                                                  'author': t['from']
+                                                                                                                  'author': t['from'],
+                                                                                                                  'attachmentid': t['atts'][0],
                                                                                                           })
                # In theory we should remove objects if they don't have an
                # attachment, but how could that ever happen? Ignore for now.

diff --git a/pgcommitfest/commitfest/models.py b/pgcommitfest/commitfest/models.py
index e4326db8a8cc89ed8125c1d2a178e4739d0c05b5..e7ac9c706edcd2b2ff073e41b5f774e1ff3e2a6d 100644 (file)
--- a/pgcommitfest/commitfest/models.py
+++ b/pgcommitfest/commitfest/models.py
@@ -184,8 +184,10 @@ class MailThread(models.Model):
 class MailThreadAttachment(models.Model):
        mailthread = models.ForeignKey(MailThread, null=False, blank=False)
        messageid = models.CharField(max_length=1000, null=False, blank=False)
+       attachmentid = models.IntegerField(null=False, blank=False)
        date = models.DateTimeField(null=False, blank=False)
        author = models.CharField(max_length=500, null=False, blank=False)
+       ispatch = models.NullBooleanField()
 
        class Meta:
                ordering = ('-date',)

diff --git a/pgcommitfest/commitfest/templates/patch.html b/pgcommitfest/commitfest/templates/patch.html
index e69dcf46bb8431f9096d1d4b0216124304beb082..09fe9a979ebc336213dd36d0effa51d182044d02 100644 (file)
--- a/pgcommitfest/commitfest/templates/patch.html
+++ b/pgcommitfest/commitfest/templates/patch.html
@@ -70,9 +70,8 @@
          {%if forloop.first%}
          Latest attachment at <a href="http://www.postgrsql.org/message-id/{{ta.messageid}}/">{{ta.date}}</a> from {{ta.author|hidemail}} <button type="button" class="btn btn-mini" data-toggle="collapse" data-target="#att{{t.pk}}"><i class="icon-plus"></i></button>
            <div id="att{{t.pk}}" class="collapse">
-         {%else%}
-           &nbsp;&nbsp;&nbsp;&nbsp;Attachment at <a href="http://www.postgresql.org/message-id/{{ta.messageid}}/">{{ta.date}}</a> from {{ta.author|hidemail}}<br/>
           {%endif%}
+           &nbsp;&nbsp;&nbsp;&nbsp;Attachment at <a href="http://www.postgresql.org/message-id/{{ta.messageid}}/">{{ta.date}}</a> from {{ta.author|hidemail}} (Patch: {{ta.ispatch|yesno:"Yes,No,Pending check"}})<br/>
           {%if forloop.last%}</div>{%endif%}
          {%endfor%}
         </dd>

diff --git a/tools/commitfest/check_patches_in_archives.py b/tools/commitfest/check_patches_in_archives.py
new file mode 100755 (executable)
index 0000000..7fcc4e7
--- /dev/null
+++ b/tools/commitfest/check_patches_in_archives.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+#
+# check_patches_in_archives.py
+#
+# Download and check attachments in the archives, to see if they are
+# actually patches. We do this asynchronously in a separate script
+# so we don't block the archives unnecessarily.
+#
+
+import os
+import sys
+import socket
+import httplib
+import magic
+
+# Set up for accessing django
+from django.core.management import setup_environ
+sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../../pgcommitfest'))
+import settings
+setup_environ(settings)
+
+from commitfest.models import MailThreadAttachment
+
+if __name__ == "__main__":
+       socket.setdefaulttimeout(settings.ARCHIVES_TIMEOUT)
+       mag = magic.open(magic.MIME)
+       mag.load()
+       
+       # Try to fetch/scan all attachments that haven't already been scanned.
+       # If they have already been scanned, we don't bother.
+       # We will hit the archives without delay when doing this, but that
+       # should generally not be a problem because it's not going to be
+       # downloading a lot...
+       for a in MailThreadAttachment.objects.filter(ispatch=None):
+               url = "/message-id/attachment/%s/attach" % a.attachmentid
+               h = httplib.HTTPConnection(settings.ARCHIVES_SERVER,
+                                                                  settings.ARCHIVES_PORT,
+                                                                  True,
+                                                                  settings.ARCHIVES_TIMEOUT)
+               h.request('GET', url, headers={
+                       'Host': settings.ARCHIVES_HOST,
+                       })
+               resp = h.getresponse()
+               if resp.status != 200:
+                       print "Failed to get %s: %s" % (url, resp.status)
+
+               contents = resp.read()
+               resp.close()
+               h.close()
+
+               # Attempt to identify the file using magic information
+               mtype = mag.buffer(contents)
+
+               # We don't support gzipped or tar:ed patches or anything like
+               # that at this point - just plain patches.
+               if mtype.startswith('text/x-diff'):
+                       a.ispatch = True
+               else:
+                       a.ispatch = False
+               a.save()