Update yum spider and download page for new reporpms

The yum project has switched to single-reporpm-for-multiple-versions as
well as putting them in a dedicated directory. This makes a lot of the
cruft in the code to deal with the distribution of reporpms unnecessary,
so change it.

We still need to keep track of which platforms exist for which versions,
so we do still need some spidering, but it's  alot simpler.

Since all versions are also always availeble, the javascript that
populates the version list can now just use the list of supported
versions. This could be rendered directly in the HTML, but we might need
to restrict versions-per-platform at some point in the future, and
javascript is needed anyway for the rest of the page, so we keep doing
it the old way.

diff --git a/templates/downloads/js/yum.js b/templates/downloads/js/yum.js
index 2d571d1d154e161d6aa2ff0530951984bc8a242f..dee6b700f51e4d4b0f3ca970798c20615d543dbd 100644 (file)
--- a/templates/downloads/js/yum.js
+++ b/templates/downloads/js/yum.js
@@ -5,22 +5,64 @@ function sortNumeric(a,b) {
    return a-b;
 }
 
-window.onload = function() {
-   versions = Object.keys(repodata['reporpms']).sort(sortNumeric).reverse();
-   for (var p in versions) {
-      if (supported_versions.indexOf(Number(versions[p])) < 0)
-         continue;
+function get_platform_name(plat) {
+    if (plat == 'EL')
+       return "RedHat Enterprise, CentOS, Scientific or Oracle";
+    else if (plat == 'F')
+       return "Fedora";
+    return "Undefined distribution";
+}
+
+function get_rpm_prefix(plat) {
+   if (plat.startsWith('EL-'))
+       return 'redhat';
+    else if (plat.startsWith('F-'))
+       return 'fedora';
+    return 'unknown';
+}
+
+function get_installer(plat) {
+    if (plat.startsWith('F-'))
+       return 'dnf';
+    else if (plat.startsWith('EL-')) {
+       var a = plat.split('-');
+       if (a[1] >= 8)
+           return 'dnf';
+    }
+    return 'yum';
+}
+
+function uses_systemd(plat) {
+    if (plat.startsWith('EL-')) {
+       var a = plat.split('-');
+       if (a[1] < 7)
+           return false;
+    }
+    return true;
+}
+
+function get_platform_text(p) {
+    var a = p.split('-');
+    return get_platform_name(a[0]) + ' version ' + a[1];
+}
 
+window.onload = function() {
+   for (var p in supported_versions) {
       var opt = document.createElement('option');
-      opt.text = versions[p];
+      opt.text = supported_versions[p];
       document.getElementById('version').add(opt);
    }
 
-   verChanged();
+   loadPlatforms();
+   archChanged();
 }
 
 function verChanged() {
-   var newver = document.getElementById('version').value;
+    /* Just update like the architecture changed */
+    archChanged();
+}
+
+function loadPlatforms() {
    var platbox = document.getElementById('platform');
 
    while (platbox.options.length > 0) {
@@ -31,15 +73,11 @@ function verChanged() {
    opt.value = -1;
    platbox.add(opt);
 
-   plats = Object.keys(repodata['reporpms'][newver]).sort(
-       function(a,b) {
-          return repodata['platforms'][a].s - repodata['platforms'][b].s;
-       }
-   );
-   for (p in plats) {
+   platkeys = Object.keys(repodata['platforms']).sort();
+   for (var pp in platkeys) {
       var opt = document.createElement('option');
-      opt.text = repodata['platforms'][plats[p]].t;
-      opt.value = plats[p];
+      opt.text = get_platform_text(platkeys[pp]);
+      opt.value = platkeys[pp];
       platbox.add(opt);
    }
 
@@ -47,7 +85,6 @@ function verChanged() {
 }
 
 function platChanged() {
-   var ver = document.getElementById('version').value;
    var plat = document.getElementById('platform').value;
    var archbox = document.getElementById('arch');
 
@@ -60,13 +97,9 @@ function platChanged() {
       return;
    }
 
-   var platname = repodata['platforms'][plat].t;
-
-   archs = Object.keys(repodata['reporpms'][ver][plat]).sort().reverse();
-   for (a in archs) {
+   for (a in repodata['platforms'][plat].sort().reverse()) {
       var opt = document.createElement('option');
-      opt.text = archs[a];
-      opt.value = archs[a];
+      opt.text = opt.value = repodata['platforms'][plat][a];
       archbox.add(opt);
    }
 
@@ -78,7 +111,7 @@ function archChanged() {
    var plat = document.getElementById('platform').value;
    var arch = document.getElementById('arch').value;
 
-   if (plat == -1) {
+   if (!plat || plat == -1) {
       document.getElementById('reporpm').innerHTML = 'Select version and platform above';
       document.getElementById('clientpackage').innerHTML = 'Select version and platform above';
       document.getElementById('serverpackage').innerHTML = 'Select version and platform above';
@@ -89,12 +122,14 @@ function archChanged() {
    var pinfo = repodata['platforms'][plat];
    var shortver = ver.replace('.', '');
 
-   var url = 'https://download.postgresql.org/pub/repos/yum/' + ver + '/' + pinfo['p'] + '-' + arch + '/pgdg-' + pinfo['f'] + shortver + '-' + ver + '-' + repodata['reporpms'][ver][plat][arch] + '.noarch.rpm';
+   var url = 'https://download.postgresql.org/pub/repos/yum/reporpms/' + plat + '-' + arch + '/pgdg-' + get_rpm_prefix(plat) +'-repo-latest.noarch.rpm';
+
+   var installer = get_installer(plat);
+   document.getElementById('reporpm').innerHTML = installer + ' install ' + url;
+   document.getElementById('clientpackage').innerHTML = installer + ' install postgresql' + shortver;
+   document.getElementById('serverpackage').innerHTML = installer + ' install postgresql' + shortver + '-server';
 
-   document.getElementById('reporpm').innerHTML = pinfo['i'] + ' install ' + url;
-   document.getElementById('clientpackage').innerHTML = pinfo['i'] + ' install postgresql' + shortver;
-   document.getElementById('serverpackage').innerHTML = pinfo['i'] + ' install postgresql' + shortver + '-server';
-   if (pinfo.d) {
+   if (uses_systemd(plat)) {
        var setupcmd = 'postgresql-' + shortver + '-setup';
        if (ver < 10) {
           setupcmd = 'postgresql' + shortver + '-setup';

diff --git a/tools/ftp/spider_yum.py b/tools/ftp/spider_yum.py
index f2b61364e9a3e127b78375e54a9ca6b776400c06..4dfd6191991a83967d87361f5e5565185fdd7814 100755 (executable)
--- a/tools/ftp/spider_yum.py
+++ b/tools/ftp/spider_yum.py
@@ -5,53 +5,10 @@ import os
 import re
 import json
 import requests
-from decimal import Decimal
+from collections import defaultdict
 from tempfile import NamedTemporaryFile
 
-platform_names = {
-    'redhat': 'Red Hat Enterprise Linux {0}',
-    'centos': 'CentOS {0}',
-    'sl': 'Scientific Linux {0}',
-    'fedora': 'Fedora {0}',
-    'oraclelinux': 'Oracle Enterprise Linux {0}',
-    'ami201503-': 'Amazon Linux AMI201503 {0}',
-}
-platform_sort = {
-    'redhat': 1,
-    'centos': 2,
-    'sl': 3,
-    'fedora': 4,
-    'oraclelinux': 5,
-    'ami201503-': 6,
-}
-archs = ['x86_64', 'i386', 'i686', 'ppc64le']
-
-
-def generate_platform(dirname, familyprefix, ver, installer, systemd):
-    for f in list(platform_names.keys()):
-        yield ('%s-%s' % (f, ver), {
-            't': platform_names[f].format(ver),
-            'p': os.path.join(dirname, '{0}-{1}'.format(familyprefix, ver)),
-            'f': f,
-            'i': installer,
-            'd': systemd,
-            's': platform_sort[f] * 1000 - ver,
-            'found': False,
-        })
-
-
-def get_redhat_systemd(ver):
-    return (ver >= 7)
-
-
-platforms = {}
-for v in range(5, 7 + 1):
-    platforms.update(dict(generate_platform('redhat', 'rhel', v, 'yum', get_redhat_systemd(v))))
-for v in range(24, 30 + 1):
-    platforms.update(dict(generate_platform('fedora', 'fedora', v, 'dnf', True)))
-
-re_reporpm = re.compile('^pgdg-([a-z0-9-]+)([0-9]{2})-[^-]+-(\d+)\.noarch\.rpm$')
-re_versiondirs = re.compile(r'^\d+(\.\d+)?$')
+re_platformdir = re.compile('^(\w+)-(\d+)-([^-]+)$')
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Spider repo RPMs")
@@ -60,40 +17,16 @@ if __name__ == "__main__":
 
     args = parser.parse_args()
 
-    versions = sorted([v for v in os.listdir(args.yumroot) if re_versiondirs.match(v)], key=Decimal, reverse=True)
-    reporpms = {}
-    for v in versions:
-        reporpms[v] = {}
-        vroot = os.path.join(args.yumroot, v)
-        for dirpath, dirnames, filenames in os.walk(vroot):
-            rmatches = [_f for _f in (re_reporpm.match(f) for f in sorted(filenames, reverse=True)) if _f]
-
-            if rmatches:
-                familypath = os.path.join(*dirpath.split('/')[-2:])
-                (familypath, arch) = familypath.rsplit('-', 1)
-
-                for r in rmatches:
-                    shortdist, shortver, ver = r.groups(1)
-
-                    found = False
-                    for p, pinfo in list(platforms.items()):
-                        if pinfo['p'] == familypath and pinfo['f'] == shortdist:
-                            if p not in reporpms[v]:
-                                reporpms[v][p] = {}
-                            reporpms[v][p][arch] = max(int(ver), int(reporpms[v][p].get(arch, 0)))
-                            platforms[p]['found'] = True
-                            break
-                    else:
-                        # DEBUG
-                        # print "%s (%s) not found in platform list" % (familypath, shortdist)
-                        pass
-
-    # Filter all platforms that are not used
-    platforms = {k: v for k, v in platforms.items() if v['found']}
-    for k, v in platforms.items():
-        del v['found']
+    platforms = defaultdict(list)
+    for repodir in os.listdir('{0}/reporpms'.format(args.yumroot)):
+        m = re_platformdir.match(repodir)
+        if m:
+            platname = m.group(1)
+            platver = m.group(2)
+            arch = m.group(3)
+            platforms['{0}-{1}'.format(platname, platver)].append(arch)
 
-    j = json.dumps({'platforms': platforms, 'reporpms': reporpms})
+    j = json.dumps({'platforms': platforms})
 
     if args.target.startswith('http://') or args.target.startswith('https://'):
         r = requests.put(