Add function to spider reporpms from the yum repo

The ftp server can then submit a list (and structure) of which platforms
are supported for yum downloads, which can then later (in a separate
commit) be used to generate a nicer download for yum repo rpms.

diff --git a/pgweb/downloads/views.py b/pgweb/downloads/views.py
index da29f3aabc3bfa612ec3035554e5181302e9936e..63d69981e252a9b7aa6208f61d4ed194d9a5d4fd 100644 (file)
--- a/pgweb/downloads/views.py
+++ b/pgweb/downloads/views.py
@@ -9,6 +9,7 @@ from django.conf import settings
 import os
 import urlparse
 import cPickle as pickle
+import json
 
 from pgweb.util.decorators import nocache
 from pgweb.util.contexts import NavContext
@@ -154,6 +155,35 @@ def uploadftp(request):
        # Finally, indicate to the client that we're happy
        return HttpResponse("OK", content_type="text/plain")
 
+@csrf_exempt
+def uploadyum(request):
+       if request.method != 'PUT':
+               return HttpServerError("Invalid method")
+       if not request.META['REMOTE_ADDR'] in settings.FTP_MASTERS:
+               return HttpServerError("Invalid client address")
+       # We have the data in request.body. Attempt to load it as
+       # json to ensure correct format.
+       json.loads(request.body)
+
+       # Next, check if it's the same as the current file
+       if os.path.isfile(settings.YUM_JSON):
+               with open(settings.YUM_JSON, "r") as f:
+                       if f.read() == request.body:
+                               # Don't rewrite the file or purge any data if nothing changed
+                               return HttpResponse("NOT CHANGED", content_type="text/plain")
+
+       # File has changed - let's write it!
+       with open("%s.new" % settings.YUM_JSON, "w") as f:
+               f.write(request.body)
+
+       os.rename("%s.new" % settings.YUM_JSON, settings.YUM_JSON)
+
+       # Purge it out of varnish so we start responding right away
+       varnish_purge("/download/js/yum.js")
+
+       # Finally, indicate to the client that we're happy
+       return HttpResponse("OK", content_type="text/plain")
+
 @nocache
 def mirrorselect(request, path):
        # Old access to mirrors will just redirect to the main ftp site.

diff --git a/pgweb/settings.py b/pgweb/settings.py
index c31cf40b84b66642d40e24579899210758682fd7..51d58cc9bd4a594baa2db112f00e979bebdd41ef 100644 (file)
--- a/pgweb/settings.py
+++ b/pgweb/settings.py
@@ -157,6 +157,7 @@ CSRF_COOKIE_HTTPONLY=SESSION_COOKIE_HTTPONLY
 
 SITE_ROOT="http://www.postgresql.org"                  # Root of working URLs
 FTP_PICKLE="/usr/local/pgweb/ftpsite.pickle"           # Location of file with current contents from ftp site
+YUM_JSON="/usr/local/pgweb/external/yum.json"
 STATIC_CHECKOUT="/usr/local/pgweb-static"              # Location of a checked out pgweb-static project
 NOTIFICATION_EMAIL="someone@example.com"               # Address to send notifications *to*
 NOTIFICATION_FROM="someone@example.com"                # Address to send notifications *from*

diff --git a/pgweb/urls.py b/pgweb/urls.py
index 7a94d87a60d2419ad3ccd3a02390ff9b039e6276..a8b18df1b3d7df423f56eb62ede4638a8e198881 100644 (file)
--- a/pgweb/urls.py
+++ b/pgweb/urls.py
@@ -35,6 +35,7 @@ urlpatterns = patterns('',
        (r'^download/products/(\d+)(-.*)?/$', 'pgweb.downloads.views.productlist'),
        (r'^applications-v2.xml$', 'pgweb.downloads.views.applications_v2_xml'),
        (r'^download/uploadftp/', 'pgweb.downloads.views.uploadftp'),
+       (r'^download/uploadyum/', 'pgweb.downloads.views.uploadyum'),
 
        (r'^docs/$', 'pgweb.docs.views.root'),
        (r'^docs/manuals/$', 'pgweb.docs.views.manuals'),

diff --git a/tools/ftp/spider_yum.py b/tools/ftp/spider_yum.py
new file mode 100755 (executable)
index 0000000..8fd588e
--- /dev/null
+++ b/tools/ftp/spider_yum.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+import argparse
+import sys
+import os
+import re
+import json
+import urllib2
+from tempfile import NamedTemporaryFile
+
+versions = ["9.6", "9.5", "9.4", "9.3", "9.2"]
+
+platform_names = {
+       'redhat': 'RedHat Enterprise Linux {0}',
+       'centos': 'CentOS {0}',
+       'sl': 'Scientific Linux {0}',
+       'fedora': 'Fedora {0}',
+       'oraclelinux': 'Oracle Enterprise Linux {0}',
+       'ami201503-': 'Amazon Linux AMI201503 {0}',
+}
+platform_sort = {
+       'redhat': 1,
+       'centos': 2,
+       'sl': 3,
+       'fedora': 4,
+       'oraclelinux': 5,
+       'ami201503-': 6,
+}
+archs = ['x86_64', 'i386', 'i686', 'ppc64le']
+
+def generate_platform(dirname, familyprefix, ver, installer, systemd):
+       for f in platform_names.keys():
+               yield ('%s-%s' % (f, ver), {
+                       't': platform_names[f].format(ver),
+                       'p': os.path.join(dirname, '{0}-{1}'.format(familyprefix, ver)),
+                       'f': f,
+                       'i': installer,
+                       'd': systemd,
+                       's': platform_sort[f]*1000-ver,
+                       'found': False,
+                       })
+
+def get_redhat_systemd(ver):
+       return (ver >= 7)
+
+platforms = {}
+for v in range(5, 7+1):
+       platforms.update(dict(generate_platform('redhat', 'rhel', v, 'yum', get_redhat_systemd(v))))
+for v in range(24, 25+1):
+       platforms.update(dict(generate_platform('fedora', 'fedora', v, 'dnf', True)))
+
+re_reporpm = re.compile('^pgdg-([a-z0-9-]+)([0-9]{2})-[^-]+-(\d+)\.noarch\.rpm$')
+if __name__ == "__main__":
+       parser = argparse.ArgumentParser(description="Spider repo RPMs")
+       parser.add_argument('yumroot', type=str, help='YUM root path')
+       parser.add_argument('target', type=str, help='Target URL or filename')
+
+       args = parser.parse_args()
+
+       reporpms = {}
+       for v in versions:
+               reporpms[v] = {}
+               vroot = os.path.join(args.yumroot, v)
+               for dirpath, dirnames, filenames in os.walk(vroot):
+                       rmatches = filter(None, (re_reporpm.match(f) for f in sorted(filenames, reverse=True)))
+
+                       if rmatches:
+                               familypath = os.path.join(*dirpath.split('/')[-2:])
+                               (familypath, arch) = familypath.rsplit('-', 1)
+
+                               for r in rmatches:
+                                       shortdist, shortver, ver = r.groups(1)
+
+                                       found = False
+                                       for p, pinfo in platforms.items():
+                                               if pinfo['p'] == familypath and pinfo['f'] == shortdist:
+                                                       if not reporpms[v].has_key(p):
+                                                               reporpms[v][p] = {}
+                                                       reporpms[v][p][arch] = max(ver, reporpms[v].get(p, 0))
+                                                       platforms[p]['found'] = True
+                                                       break
+                                       else:
+                                               # DEBUG
+#                                              print "%s (%s) not found in platform list" % (familypath, shortdist)
+                                               pass
+
+       # Filter all platforms that are not used
+       platforms = {k:v for k,v in platforms.iteritems() if v['found']}
+       for k,v in platforms.iteritems():
+               del v['found']
+
+       j = json.dumps({'platforms': platforms, 'reporpms': reporpms})
+
+       if args.target.startswith('http://') or args.target.startswith('https://'):
+               o = urllib2.build_opener(urllib2.HTTPHandler)
+               r = urllib2.Request(sys.argv[2], data=j)
+               r.add_header('Content-type', 'application/json')
+               r.add_header('Host', 'www.postgresql.org')
+               r.get_method = lambda: 'PUT'
+               u = o.open(r)
+               x = u.read()
+               if x != "NOT CHANGED" and x != "OK":
+                       print "Failed to upload: %s" % x
+                       sys.exit(1)
+       else:
+               with NamedTemporaryFile(dir=os.path.dirname(os.path.abspath(args.target))) as f:
+                       f.write(j)
+                       f.flush()
+                       if os.path.isfile(args.target):
+                               os.unlink(args.target)
+                       os.link(f.name, args.target)