Add support for varnish purging based on expressions

Previously we would only purge based on URLs, but some of the upcoming
new work requires arbitrary expression purging.

NOTE! Require the creation of the new SQL procecure in the database,
either from varnish.sql or varnish_local.sql depending on if it's prod
or dev.

diff --git a/pgweb/core/views.py b/pgweb/core/views.py
index 90edbe6ecdcd741c221f715f17210e3515b5675c..dcc1178314c0a577e70fa851f73034e0e5df0692 100644 (file)
--- a/pgweb/core/views.py
+++ b/pgweb/core/views.py
@@ -190,7 +190,7 @@ def admin_purge(request):
 
        # Fetch list of latest purges
        curs = connection.cursor()
-       curs.execute("SELECT ev_time, ev_data FROM pgq.event_1 WHERE ev_type='P' ORDER BY ev_time DESC LIMIT 20")
+       curs.execute("SELECT ev_time, ev_data FROM pgq.event_1 WHERE ev_type IN ('P', 'X') ORDER BY ev_time DESC LIMIT 20")
        latest = [{'t': r[0], 'u': r[1]} for r in curs.fetchall()]
 
        return render_to_response('core/admin_purge.html', {

diff --git a/sql/varnish.sql b/sql/varnish.sql
index 1b5f8d757c16ede07c04dbdeb32cd3df2c1e83e6..bfbdb7437bb5a3da4d6a81c653cb2f69b553e6b7 100644 (file)
--- a/sql/varnish.sql
+++ b/sql/varnish.sql
@@ -13,4 +13,9 @@ AS $$
    SELECT pgq.insert_event('varnish', 'P', $1);
 $$ LANGUAGE 'sql';
 
+CREATE OR REPLACE FUNCTION varnish_purge_expr(expr text)
+RETURNS bigint
+AS $$
+  SELECT pgq.insert_event('varnish', 'X', $1);
+$$ LANGUAGE 'sql';
 COMMIT;
\ No newline at end of file

diff --git a/sql/varnish_local.sql b/sql/varnish_local.sql
index cb3a9340273c26222c02bf600c9368ee25b6a0f6..b827017b37424c1dccbc892dcf2be05abf7363f8 100644 (file)
--- a/sql/varnish_local.sql
+++ b/sql/varnish_local.sql
@@ -12,4 +12,10 @@ AS $$
    SELECT 1::bigint;
 $$ LANGUAGE 'sql';
 
+CREATE OR REPLACE FUNCTION varnish_purge_expr(url text)
+RETURNS bigint
+AS $$
+   SELECT 1::bigint;
+$$ LANGUAGE 'sql';
+
 COMMIT;
\ No newline at end of file

diff --git a/tools/pgq/varnish_consumer.py b/tools/pgq/varnish_consumer.py
index 58648f25f0ea54bc9b8301b31c91d30502c8c04b..dcab39a85f77247e4d0e8a7b8d01e7e8db5fca0a 100755 (executable)
--- a/tools/pgq/varnish_consumer.py
+++ b/tools/pgq/varnish_consumer.py
@@ -45,25 +45,31 @@ class VarnishPurger(pgq.Consumer):
 
                for ev in ev_list:
                        if ev.type == 'P':
-                               # 'P' events means purge. Currently it's the only event
-                               # type we support.
+                               # 'P' events means purge.
                                print_t("Purging '%s' on %s" % (ev.data, self.frontend))
                                try:
                                        if self.do_purge(ev.data):
                                                ev.tag_done()
                                except Exception, e:
                                        print_t("Failed to purge '%s' on '%s': %s" % (ev.data, self.frontend, e))
+                       elif ev.type == 'X':
+                               # 'X' events means ban expression (rather than just urls)
+                               print_t("Purging expression '%s' on %s" % (ev.data, self.frontend))
+                               try:
+                                       if self.do_purge_expr(ev.data):
+                                               ev.tag_done()
+                               except Exception, e:
+                                       print_t("Failed to purge expression '%s' on '%s': %s" % (ev.data, self.frontend, e))
                        else:
                                print_t("Unknown event type '%s'" % ev.type)
 
 
-       def do_purge(self, url):
+       def internal_purge(self, headers):
                """
                Send the actual purge request, by contacting the frontend this
                purger is running for and sending a GET request to the special URL
                with our regexp in a special header.
                """
-               headers = {'X-Purge-URL': url}
                conn = httplib.HTTPConnection('%s.postgresql.org' % self.frontend)
                conn.request("GET", "/varnish-purge-url", '', headers)
                resp = conn.getresponse()
@@ -74,6 +80,11 @@ class VarnishPurger(pgq.Consumer):
                print_t("Varnish purge returned status %s (%s)" % (resp.status, resp.reason))
                return False
 
+       def do_purge(self, url):
+               return self.internal_purge({'X-Purge-URL': url})
+
+       def do_purge_expr(self, expr):
+               return self.internal_purge({'X-Purge-Expr': expr})
 
 class PurgerProcess(object):
        """