cipher = encryptor.encrypt(s + ' ' * (16-(len(s) % 16))) # pad to 16 bytes
return HttpResponseRedirect("%s?d=%s$%s" % (
- settings.PGAUTH_REDIRECT,
- base64.b64encode(iv, b"-_").decode('utf8'),
- base64.b64encode(cipher, b"-_").decode('utf8'),
- ))
+ settings.PGAUTH_REDIRECT,
+ base64.b64encode(iv, b"-_").decode('utf8'),
+ base64.b64encode(cipher, b"-_").decode('utf8'),
+ ))
else:
return HttpResponseRedirect(settings.PGAUTH_REDIRECT)
u = urllib.request.urlopen('%ssearch/?%s' % (
settings.PGAUTH_REDIRECT,
urlencode(q),
- ))
+ ))
(ivs, datas) = u.read().split('&')
u.close()
'description': l.description,
'active': l.active,
'group': l.group.groupname,
- } for l in List.objects.select_related('group').all()], resp)
+ } for l in List.objects.select_related('group').all()], resp)
return resp
resp = HttpResponse(content_type='application/json')
json.dump([
- {'msgid': m.messageid,
- 'date': m.date.isoformat(),
- 'from': m.mailfrom,
- 'subj': m.subject,
- 'atts': [{'id': a.id, 'name': a.filename} for a in m.attachment_set.all()],
- }
+ {
+ 'msgid': m.messageid,
+ 'date': m.date.isoformat(),
+ 'from': m.mailfrom,
+ 'subj': m.subject,
+ 'atts': [{'id': a.id, 'name': a.filename} for a in m.attachment_set.all()],
+ }
for m in mlist], resp)
resp['X-pgthread'] = m.threadid
return resp
'This message has been hidden because the message violated policies.', # 2
'This message has been hidden because for privacy reasons.', # 3
'This message was corrupt', # 4
- ]
+]
class Message(models.Model):
'sortkey': l.group.sortkey,
'lists': [l,],
'homelink': 'list/group/%s' % l.group.groupid,
- }
+ }
return (sorted(list(groups.values()), key=lambda g: g['sortkey']), listgroupid)
(groups, listgroupid) = get_all_groups_and_lists(request)
return render_nav(NavContext(request, all_groups=groups), 'index.html', {
- 'groups': [{'groupname': g['groupname'], 'lists': g['lists']} for g in groups],
- })
+ 'groups': [{'groupname': g['groupname'], 'lists': g['lists']} for g in groups],
+ })
@cache(hours=8)
raise Http404('List group does not exist')
return render_nav(NavContext(request, all_groups=groups, expand_groupid=groupid), 'index.html', {
- 'groups': mygroups,
- })
+ 'groups': mygroups,
+ })
@cache(hours=8)
def monthlist(request, listname):
months=[{'year':r[0],'month':r[1], 'date':datetime(r[0],r[1],1)} for r in curs.fetchall()]
return render_nav(NavContext(request, l.listid, l.listname), 'monthlist.html', {
- 'list': l,
- 'months': months,
- })
+ 'list': l,
+ 'months': months,
+ })
def get_monthday_info(mlist, l, d):
allmonths = set([m.date.month for m in mlist])
if monthdate:
curs = connection.cursor()
curs.execute("SELECT DISTINCT extract(day FROM date) FROM messages WHERE date >= %(startdate)s AND date < %(enddate)s AND threadid IN (SELECT threadid FROM list_threads WHERE listid=%(listid)s) ORDER BY 1", {
- 'startdate': datetime(year=monthdate.year, month=monthdate.month, day=1),
- 'enddate': monthdate + timedelta(days=calendar.monthrange(monthdate.year, monthdate.month)[1]),
- 'listid': l.listid,
- })
+ 'startdate': datetime(year=monthdate.year, month=monthdate.month, day=1),
+ 'enddate': monthdate + timedelta(days=calendar.monthrange(monthdate.year, monthdate.month)[1]),
+ 'listid': l.listid,
+ })
daysinmonth = [int(r[0]) for r in curs.fetchall()]
yearmonth = None
(yearmonth, daysinmonth) = get_monthday_info(mlist, l, d)
r = render_nav(NavContext(request, l.listid, l.listname), 'datelist.html', {
- 'list': l,
- 'messages': mlist,
- 'title': title,
- 'daysinmonth': daysinmonth,
- 'yearmonth': yearmonth,
- })
+ 'list': l,
+ 'messages': mlist,
+ 'title': title,
+ 'daysinmonth': daysinmonth,
+ 'yearmonth': yearmonth,
+ })
r['X-pglm'] = ':%s:' % (':'.join(['%s/%s/%s' % (l.listid, year, month) for year,month in allyearmonths]))
return r
def _get_nextprevious(listmap, dt):
curs = connection.cursor()
- curs.execute("""WITH l(listid) AS (
+ curs.execute("""
+WITH l(listid) AS (
SELECT unnest(%(lists)s)
)
SELECT l.listid,1,
INNER JOIN list_threads lt ON lt.threadid=m.threadid
WHERE m.date<%(time)s AND lt.listid=l.listid
ORDER BY m.date DESC LIMIT 1
- ) FROM l""", {
- 'lists': list(listmap.keys()),
- 'time': dt,
- })
+ ) FROM l""",
+ {
+ 'lists': list(listmap.keys()),
+ 'time': dt,
+ })
retval = {}
for listid, isnext, data in curs.fetchall():
if data:
'date': data[1],
'subject': data[2],
'from': data[3],
- }
+ }
if listname in retval:
retval[listname][isnext and 'next' or 'prev'] = d
else:
retval[listname] = {
isnext and 'next' or 'prev': d
- }
+ }
return retval
@cache(hours=4)
nextprev = _get_nextprevious(listmap, m.date)
r = render_nav(NavContext(request, lists[0].listid, lists[0].listname), 'message.html', {
- 'msg': m,
- 'threadstruct': threadstruct,
- 'responses': responses,
- 'parent': parent,
- 'lists': lists,
- 'nextprev': nextprev,
- })
+ 'msg': m,
+ 'threadstruct': threadstruct,
+ 'responses': responses,
+ 'parent': parent,
+ 'lists': lists,
+ 'nextprev': nextprev,
+ })
r['X-pgthread'] = ":%s:" % m.threadid
r['Last-Modified'] = http_date(newest)
return r
return HttpResponseNotModified()
r = render_nav(NavContext(request), 'message_flat.html', {
- 'msg': msg,
- 'allmsg': allmsg,
- 'lists': lists,
- 'isfirst': isfirst,
- })
+ 'msg': msg,
+ 'allmsg': allmsg,
+ 'lists': lists,
+ 'isfirst': isfirst,
+ })
r['X-pgthread'] = ":%s:" % msg.threadid
r['Last-Modified'] = http_date(newest)
return r
query = "SELECT messageid, rawtxt FROM messages m INNER JOIN list_threads t ON t.threadid=m.threadid WHERE listid=%(listid)s AND hiddenstatus IS NULL AND date >= %(startdate)s AND date <= %(enddate)s %%% ORDER BY date"
params = {
- 'listid': l.listid,
- 'startdate': date(mboxyear, mboxmonth, 1),
- 'enddate': datetime(mboxyear, mboxmonth, calendar.monthrange(mboxyear, mboxmonth)[1], 23, 59, 59),
+ 'listid': l.listid,
+ 'startdate': date(mboxyear, mboxmonth, 1),
+ 'enddate': datetime(mboxyear, mboxmonth, calendar.monthrange(mboxyear, mboxmonth)[1], 23, 59, 59),
}
if not settings.PUBLIC_ARCHIVES and not request.user.is_superuser:
# We don't do a more specific check if it's a messageid because doing
# a key lookup is cheap...
curs.execute("SELECT messageid FROM messages WHERE messageid=%(q)s", {
- 'q': query,
- })
+ 'q': query,
+ })
a = curs.fetchall()
if len(a) == 1:
# Yup, this was a messageid
resp = HttpResponse(content_type='application/json')
- json.dump([{
- 'm': messageid,
- 'd': date.isoformat(),
- 's': subject,
- 'f': mailfrom,
- 'r': rank,
- 'a': abstract.replace("[[[[[[", "<b>").replace("]]]]]]","</b>"),
-
- } for messageid, date, subject, mailfrom, rank, abstract in curs.fetchall()],
- resp)
+ json.dump([
+ {
+ 'm': messageid,
+ 'd': date.isoformat(),
+ 's': subject,
+ 'f': mailfrom,
+ 'r': rank,
+ 'a': abstract.replace("[[[[[[", "<b>").replace("]]]]]]","</b>"),
+ } for messageid, date, subject, mailfrom, rank, abstract in curs.fetchall()],
+ resp)
return resp
@cache(seconds=10)
def legacy(request, listname, year, month, msgnum):
curs = connection.cursor()
curs.execute("SELECT msgid FROM legacymap WHERE listid=(SELECT listid FROM lists WHERE listname=%(list)s) AND year=%(year)s AND month=%(month)s AND msgnum=%(msgnum)s", {
- 'list': listname,
- 'year': year,
- 'month': month,
- 'msgnum': msgnum,
- })
+ 'list': listname,
+ 'year': year,
+ 'month': month,
+ 'msgnum': msgnum,
+ })
r = curs.fetchall()
if len(r) != 1:
raise Http404('Message does not exist')
'media/css/table.css',
'media/css/text.css',
'media/css/docs.css'],
- }
+}
@cache(hours=8)
def dynamic_css(request, css):
MIDDLEWARE_CLASSES = [
'django.middleware.common.CommonMiddleware',
-# 'django.contrib.sessions.middleware.SessionMiddleware',
-# 'django.middleware.csrf.CsrfViewMiddleware',
-# 'django.contrib.auth.middleware.AuthenticationMiddleware',
-# 'django.contrib.messages.middleware.MessageMiddleware',
]
ROOT_URLCONF = 'archives.urls'
INSTALLED_APPS = [
-# 'django.contrib.auth',
-# 'django.contrib.contenttypes',
-# 'django.contrib.sessions',
-# 'django.contrib.sites',
-# 'django.contrib.messages',
-# 'django.contrib.staticfiles',
- # Uncomment the next line to enable the admin:
- # 'django.contrib.admin',
- # Uncomment the next line to enable admin documentation:
- # 'django.contrib.admindocs',
'archives.mailarchives',
]
url(r'^list/([\w-]+|\*)/latest.json$', archives.mailarchives.api.latest),
url(r'^message-id.json/(.+)$', archives.mailarchives.api.thread),
url(r'^listinfo/$', archives.mailarchives.api.listinfo),
-# url(r'^thread/(.+)/subscribe/$', archives.mailarchives.api.thread_subscribe),
+ # url(r'^thread/(.+)/subscribe/$', archives.mailarchives.api.thread_subscribe),
# Legacy forwarding from old archives site
url(r'^message-id/legacy/([\w-]+)/(\d+)-(\d+)/msg(\d+).php$', archives.mailarchives.views.legacy),
x = input("Parsed this as date %s. Update? " % d)
if x.upper() == 'Y':
curs.execute("UPDATE messages SET date=%(d)s WHERE messageid=%(m)s", {
- 'd': d,
- 'm': messageid,
- })
+ 'd': d,
+ 'm': messageid,
+ })
print("Updated.")
break
elif x.upper() == 'N':
break
-
+
if __name__ == "__main__":
cfg = ConfigParser()
cfg.read('%s/archives.ini' % os.path.realpath(os.path.dirname(sys.argv[0])))
'char-encoding': 'utf8',
'show-warnings': 0,
'show-info': 0,
- })
+ })
if errors:
print(("HTML tidy failed for %s!" % self.msgid))
print(errors)
# make the check anyway, and this keeps the code in one place..
if not overwrite:
curs.execute("INSERT INTO list_months (listid, year, month) SELECT %(listid)s, %(year)s, %(month)s WHERE NOT EXISTS (SELECT listid FROM list_months WHERE listid=%(listid)s AND year=%(year)s AND month=%(month)s)", {
- 'listid': listid,
- 'year': self.date.year,
- 'month': self.date.month,
- })
+ 'listid': listid,
+ 'year': self.date.year,
+ 'month': self.date.month,
+ })
curs.execute("SELECT threadid, EXISTS(SELECT threadid FROM list_threads lt WHERE lt.listid=%(listid)s AND lt.threadid=m.threadid), id FROM messages m WHERE m.messageid=%(messageid)s", {
- 'messageid': self.msgid,
- 'listid': listid,
- })
+ 'messageid': self.msgid,
+ 'listid': listid,
+ })
r = curs.fetchall()
if len(r) > 0:
# Has to be 1 row, since we have a unique index on id
if not r[0][1] and not overwrite:
log.status("Tagging message %s with list %s" % (self.msgid, listid))
curs.execute("INSERT INTO list_threads (threadid, listid) VALUES (%(threadid)s, %(listid)s)", {
- 'threadid': r[0][0],
- 'listid': listid,
- })
+ 'threadid': r[0][0],
+ 'listid': listid,
+ })
opstatus.tagged += 1
self.purge_list(listid, self.date.year, self.date.month)
self.purge_thread(r[0][0])
# We update bodytext as a separate step so as not to rewrite
# the TOAST table unnecessarily...
curs.execute("UPDATE messages SET bodytxt=%(bodytxt)s WHERE id=%(id)s AND NOT (bodytxt=%(bodytxt)s) RETURNING id", {
- 'id': pk,
- 'bodytxt': self.bodytxt,
- })
+ 'id': pk,
+ 'bodytxt': self.bodytxt,
+ })
rc = curs.rowcount
curs.execute("UPDATE messages SET _from=%(from)s, _to=%(to)s, cc=%(cc)s, subject=%(subject)s, date=%(date)s, has_attachment=%(has_attachment)s WHERE id=%(id)s AND NOT (_from=%(from)s AND _to=%(to)s AND cc=%(cc)s AND subject=%(subject)s AND date=%(date)s AND has_attachment=%(has_attachment)s) RETURNING id", {
- 'id': pk,
- 'from': self._from,
- 'to': self.to or '',
- 'cc': self.cc or '',
- 'subject': self.subject or '',
- 'date': self.date,
- 'has_attachment': len(self.attachments) > 0,
- })
+ 'id': pk,
+ 'from': self._from,
+ 'to': self.to or '',
+ 'cc': self.cc or '',
+ 'subject': self.subject or '',
+ 'date': self.date,
+ 'has_attachment': len(self.attachments) > 0,
+ })
rc += curs.rowcount
if rc == 0:
log.status("Message %s unchanged" % self.msgid)
return False
curs.execute("DELETE FROM attachments WHERE message=%(message)s", {
- 'message': pk,
- })
+ 'message': pk,
+ })
if len(self.attachments):
curs.executemany("INSERT INTO attachments (message, filename, contenttype, attachment) VALUES (%(message)s, %(filename)s, %(contenttype)s, %(attachment)s)",[ {
- 'message': pk,
- 'filename': a[0] or 'unknown_filename',
- 'contenttype': a[1],
- 'attachment': bytearray(a[2]),
- } for a in self.attachments])
+ 'message': pk,
+ 'filename': a[0] or 'unknown_filename',
+ 'contenttype': a[1],
+ 'attachment': bytearray(a[2]),
+ } for a in self.attachments])
opstatus.overwritten += 1
log.status("Message %s overwritten" % self.msgid)
else:
# Resolve own thread
curs.execute("SELECT id, messageid, threadid FROM messages WHERE messageid=ANY(%(parents)s)", {
- 'parents': self.parents,
- })
+ 'parents': self.parents,
+ })
all_parents = curs.fetchall()
if len(all_parents):
# At least one of the parents exist. Now try to figure out which one
# Now see if we are somebody elses *parent*...
curs.execute("SELECT message, priority, threadid FROM unresolved_messages INNER JOIN messages ON messages.id=unresolved_messages.message WHERE unresolved_messages.msgid=%(msgid)s ORDER BY threadid", {
- 'msgid': self.msgid,
- })
+ 'msgid': self.msgid,
+ })
childrows = curs.fetchall()
if len(childrows):
# We are some already existing message's parent (meaning the
# We have one or more merge threads
log.status("Merging threads %s into thread %s" % (",".join(str(s) for s in mergethreads), self.threadid))
curs.execute("UPDATE messages SET threadid=%(threadid)s WHERE threadid=ANY(%(oldthreadids)s)", {
- 'threadid': self.threadid,
- 'oldthreadids': list(mergethreads),
- })
+ 'threadid': self.threadid,
+ 'oldthreadids': list(mergethreads),
+ })
# Insert any lists that were tagged on the merged threads
curs.execute("INSERT INTO list_threads (threadid, listid) SELECT DISTINCT %(threadid)s,listid FROM list_threads lt2 WHERE lt2.threadid=ANY(%(oldthreadids)s) AND listid NOT IN (SELECT listid FROM list_threads lt3 WHERE lt3.threadid=%(threadid)s)", {
- 'threadid': self.threadid,
- 'oldthreadids': list(mergethreads),
- })
+ 'threadid': self.threadid,
+ 'oldthreadids': list(mergethreads),
+ })
# Remove all old leftovers
curs.execute("DELETE FROM list_threads WHERE threadid=ANY(%(oldthreadids)s)", {
- 'oldthreadids': list(mergethreads),
- })
+ 'oldthreadids': list(mergethreads),
+ })
# Purge varnish records for all the threads we just removed
for t in mergethreads:
self.purge_thread(t)
# repointing until later, since we don't know our own id yet.
self.children = [r[0] for r in childrows]
log.status("Children set to %s with mergethreads being %s (from childrows %s and threadid %s)" % (
- self.children, mergethreads, childrows, self.threadid))
+ self.children, mergethreads, childrows, self.threadid)
+ )
# Finally, remove all the pending messages that had a higher
# priority value (meaning less important) than us
curs.executemany("DELETE FROM unresolved_messages WHERE message=%(msg)s AND priority >= %(prio)s", [{
- 'msg': msg,
- 'prio': prio,
- } for msg, prio, tid in childrows])
+ 'msg': msg,
+ 'prio': prio,
+ } for msg, prio, tid in childrows])
else:
self.children = []
curs.execute("INSERT INTO list_threads (threadid, listid) SELECT %(threadid)s, %(listid)s WHERE NOT EXISTS (SELECT * FROM list_threads t2 WHERE t2.threadid=%(threadid)s AND t2.listid=%(listid)s) RETURNING threadid", {
'threadid': self.threadid,
'listid': listid,
- })
+ })
if len(curs.fetchall()):
log.status("Tagged thread %s with listid %s" % (self.threadid, listid))
curs.execute("INSERT INTO messages (parentid, threadid, _from, _to, cc, subject, date, has_attachment, messageid, bodytxt, rawtxt) VALUES (%(parentid)s, %(threadid)s, %(from)s, %(to)s, %(cc)s, %(subject)s, %(date)s, %(has_attachment)s, %(messageid)s, %(bodytxt)s, %(rawtxt)s) RETURNING id", {
- 'parentid': self.parentid,
- 'threadid': self.threadid,
- 'from': self._from,
- 'to': self.to or '',
- 'cc': self.cc or '',
- 'subject': self.subject or '',
- 'date': self.date,
- 'has_attachment': len(self.attachments) > 0,
- 'messageid': self.msgid,
- 'bodytxt': self.bodytxt,
- 'rawtxt': bytearray(self.rawtxt),
- })
+ 'parentid': self.parentid,
+ 'threadid': self.threadid,
+ 'from': self._from,
+ 'to': self.to or '',
+ 'cc': self.cc or '',
+ 'subject': self.subject or '',
+ 'date': self.date,
+ 'has_attachment': len(self.attachments) > 0,
+ 'messageid': self.msgid,
+ 'bodytxt': self.bodytxt,
+ 'rawtxt': bytearray(self.rawtxt),
+ })
id = curs.fetchall()[0][0]
log.status("Message %s, got id %s, set thread %s, parent %s" % (
- self.msgid, id, self.threadid, self.parentid))
+ self.msgid, id, self.threadid, self.parentid))
if len(self.attachments):
# Insert attachments
curs.executemany("INSERT INTO attachments (message, filename, contenttype, attachment) VALUES (%(message)s, %(filename)s, %(contenttype)s, %(attachment)s)",[ {
- 'message': id,
- 'filename': a[0] or 'unknown_filename',
- 'contenttype': a[1],
- 'attachment': bytearray(a[2]),
- } for a in self.attachments])
+ 'message': id,
+ 'filename': a[0] or 'unknown_filename',
+ 'contenttype': a[1],
+ 'attachment': bytearray(a[2]),
+ } for a in self.attachments])
if len(self.children):
log.status("Setting %s other messages to children of %s" % (len(self.children), self.msgid))
# Fetch the old one so we have something to diff against
curs.execute("SELECT id, _from, _to, cc, subject, date, has_attachment, bodytxt FROM messages WHERE messageid=%(msgid)s", {
'msgid': self.msgid,
- })
+ })
try:
id, _from, to, cc, subject, date, has_attachment, bodytxt = curs.fetchone()
except TypeError as e:
# We also put the data in the db. This happens in the main transaction
# so if the whole script dies, it goes away...
conn.cursor().execute("INSERT INTO loaderrors (listid, msgid, srctype, src, err) VALUES (%(listid)s, %(msgid)s, %(srctype)s, %(src)s, %(err)s)", {
- 'listid': listid,
- 'msgid': msgid,
- 'srctype': srctype,
- 'src': src,
- 'err': str(str(err), 'us-ascii', 'replace'),
- })
+ 'listid': listid,
+ 'msgid': msgid,
+ 'srctype': srctype,
+ 'src': src,
+ 'err': str(str(err), 'us-ascii', 'replace'),
+ })
if __name__ == "__main__":
# Get the listid we're working on
curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", {
- 'list': opt.list
- })
+ 'list': opt.list
+ })
r = curs.fetchall()
if len(r) != 1:
log.error("List %s not found" % opt.list)
cfg.get('pglister', 'root'),
cfg.get('pglister', 'myname'),
do_subscribers and 1 or 0,
- ), headers={
- 'X-Api-Key': cfg.get('pglister', 'apikey'),
- })
+ ), headers={
+ 'X-Api-Key': cfg.get('pglister', 'apikey'),
+ })
obj = r.json()
# For groups, just add them if they don't exist
if a == 'y' or a == 'yes':
curs.execute("INSERT INTO messages_edited SELECT * FROM messages WHERE id=%(id)s", {
'id': id,
- })
+ })
curs.execute("UPDATE messages SET rawtxt=%(raw)s WHERE id=%(id)s", {
'id': id,
'raw': bytearray(s2),
projects / pgarchives.git / commitdiff