Turn rawtxt into a bytea, since w edon't know the encoding

diff --git a/django/archives/mailarchives/models.py b/django/archives/mailarchives/models.py
index a55370d14e5094193f0881e213c418bf3974ce1a..cf457411b0a063fccbc9d6956ccb03a1260f772b 100644 (file)
--- a/django/archives/mailarchives/models.py
+++ b/django/archives/mailarchives/models.py
@@ -9,7 +9,8 @@ class Message(models.Model):
        date = models.DateTimeField(null=False)
        messageid = models.TextField(null=False)
        bodytxt = models.TextField(null=False)
-       rawtxt = models.TextField(null=True)
+#      rawtxt = models.TextField(null=True)
+       # rawtxt is a bytea field, which django doesn't support
        parentid = models.IntegerField(null=False, blank=False)
        has_attachment = models.BooleanField(null=False, default=False)
 

diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py
index 7fe05b638025589e9c560787b5119963d5ed6ddc..4a6c8dee737a7ffeef41588e1dd8668fe1320f1b 100644 (file)
--- a/django/archives/mailarchives/views.py
+++ b/django/archives/mailarchives/views.py
@@ -31,7 +31,7 @@ def render_datelist_from(request, l, d, title, to=None):
        if to:
                datefilter.add(Q(date__lt=to), Q.AND)
 
-       mlist = Message.objects.defer('bodytxt', 'rawtxt', 'cc', 'to').select_related().filter(datefilter).extra(where=["threadid IN (SELECT threadid FROM list_threads WHERE listid=%s)" % l.listid]).order_by('date')[:200]
+       mlist = Message.objects.defer('bodytxt', 'cc', 'to').select_related().filter(datefilter).extra(where=["threadid IN (SELECT threadid FROM list_threads WHERE listid=%s)" % l.listid]).order_by('date')[:200]
 
        threads = set([m.threadid for m in mlist])
        r = render_to_response('datelist.html', {
@@ -46,7 +46,7 @@ def render_datelist_to(request, l, d, title):
        # Need to sort this backwards in the database to get the LIMIT applied
        # properly, and then manually resort it in the correct order. We can do
        # the second sort safely in python since it's not a lot of items..
-       mlist = sorted(Message.objects.defer('bodytxt', 'rawtxt', 'cc', 'to').select_related().filter(date__lte=d).extra(where=["threadid IN (SELECT threadid FROM list_threads WHERE listid=%s)" % l.listid]).order_by('-date')[:200], key=lambda m: m.date)
+       mlist = sorted(Message.objects.defer('bodytxt', 'cc', 'to').select_related().filter(date__lte=d).extra(where=["threadid IN (SELECT threadid FROM list_threads WHERE listid=%s)" % l.listid]).order_by('-date')[:200], key=lambda m: m.date)
 
        threads = set([m.threadid for m in mlist])
        r = render_to_response('datelist.html', {
@@ -113,7 +113,7 @@ SELECT id,_from,subject,date,messageid,has_attachment,parentid,datepath FROM t O
 
 def message(request, msgid):
        try:
-               m = Message.objects.defer('rawtxt').get(messageid=msgid)
+               m = Message.objects.get(messageid=msgid)
        except Message.DoesNotExist, e:
                raise Http404('Message does not exist')
 
@@ -139,10 +139,10 @@ def message(request, msgid):
 
 def message_flat(request, msgid):
        try:
-               msg = Message.objects.defer('rawtxt').get(messageid=msgid)
+               msg = Message.objects.get(messageid=msgid)
        except Message.DoesNotExist, e:
                raise Http404('Message does not exist')
-       allmsg = Message.objects.defer('rawtxt').filter(threadid=msg.threadid).order_by('date')
+       allmsg = Message.objects.filter(threadid=msg.threadid).order_by('date')
        # XXX: need to get the complete list of lists!
 
        r = render_to_response('message_flat.html', {
@@ -153,12 +153,16 @@ def message_flat(request, msgid):
        return r
 
 def message_raw(request, msgid):
-       try:
-               msg = Message.objects.defer('subject', 'mailfrom', 'to', 'cc', 'bodytxt').get(messageid=msgid)
-       except Message.DoesNotExist, e:
+       curs = connection.cursor()
+       curs.execute("SELECT threadid, rawtxt FROM messages WHERE messageid=%(messageid)s", {
+                       'messageid': msgid,
+                       })
+       row = curs.fetchall()
+       if len(row) != 1:
                raise Http404('Message does not exist')
-       r = HttpResponse(msg.rawtxt, content_type='text/plain')
-       r['X-pgthread'] = ":%s:" % msg.threadid
+
+       r = HttpResponse(row[0][1], content_type='text/plain')
+       r['X-pgthread'] = ":%s:" % row[0][0]
        return r
 
 def testview(request, seqid):

diff --git a/loader/lib/parser.py b/loader/lib/parser.py
index af691845d05ef33c5e40d08598cfe4897375d50d..9a7f9d4c2202a300df4e0ee3866b61c75d1f49a4 100644 (file)
--- a/loader/lib/parser.py
+++ b/loader/lib/parser.py
@@ -18,7 +18,7 @@ class ArchivesParser(object):
 
        def parse(self, stream):
                self.msg = self.parser.parse(stream)
-               self.rawtxt = unicode(self.msg)
+               self.rawtxt = str(self.msg)
 
        def is_msgid(self, msgid):
                # Look for a specific messageid. This means we might parse it twice,

diff --git a/loader/lib/storage.py b/loader/lib/storage.py
index 1a34b1b7f5bf3b3489e2f5e43cabadefb6b803c2..83c02af2c5efd933e894e1f198ce5c3fecea167b 100644 (file)
--- a/loader/lib/storage.py
+++ b/loader/lib/storage.py
@@ -142,7 +142,7 @@ class ArchivesParserStorage(ArchivesParser):
                                'has_attachment': len(self.attachments) > 0,
                                'messageid': self.msgid,
                                'bodytxt': self.bodytxt,
-                               'rawtxt': self.rawtxt,
+                               'rawtxt': bytearray(self.rawtxt),
                                })
                id = curs.fetchall()[0][0]
                if len(self.attachments):

diff --git a/loader/sql/schema.sql b/loader/sql/schema.sql
index 6177eb214fd00e3b8a87a9abf432bd96203b41b5..8797a382def234b40b1a6100f1c307070dae87e4 100644 (file)
--- a/loader/sql/schema.sql
+++ b/loader/sql/schema.sql
@@ -12,7 +12,7 @@ CREATE TABLE messages (
    has_attachment boolean NOT NULL,
    messageid text NOT NULL,
    bodytxt text NOT NULL,
-   rawtxt text NOT NULL
+   rawtxt bytea NOT NULL
 );
 CREATE INDEX idx_messages_threadid ON messages(threadid);
 CREATE UNIQUE INDEX idx_messages_msgid ON messages(messageid);