From 18c7292eaa55fd4d1df9ea002408dcb083c6aad3 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Wed, 15 May 2019 11:40:11 +0200 Subject: [PATCH] Update yum spider and download page for new reporpms The yum project has switched to single-reporpm-for-multiple-versions as well as putting them in a dedicated directory. This makes a lot of the cruft in the code to deal with the distribution of reporpms unnecessary, so change it. We still need to keep track of which platforms exist for which versions, so we do still need some spidering, but it's alot simpler. Since all versions are also always availeble, the javascript that populates the version list can now just use the list of supported versions. This could be rendered directly in the HTML, but we might need to restrict versions-per-platform at some point in the future, and javascript is needed anyway for the rest of the page, so we keep doing it the old way. --- templates/downloads/js/yum.js | 93 ++++++++++++++++++++++++----------- tools/ftp/spider_yum.py | 89 +++++---------------------------- 2 files changed, 75 insertions(+), 107 deletions(-) diff --git a/templates/downloads/js/yum.js b/templates/downloads/js/yum.js index 2d571d1d..dee6b700 100644 --- a/templates/downloads/js/yum.js +++ b/templates/downloads/js/yum.js @@ -5,22 +5,64 @@ function sortNumeric(a,b) { return a-b; } -window.onload = function() { - versions = Object.keys(repodata['reporpms']).sort(sortNumeric).reverse(); - for (var p in versions) { - if (supported_versions.indexOf(Number(versions[p])) < 0) - continue; +function get_platform_name(plat) { + if (plat == 'EL') + return "RedHat Enterprise, CentOS, Scientific or Oracle"; + else if (plat == 'F') + return "Fedora"; + return "Undefined distribution"; +} + +function get_rpm_prefix(plat) { + if (plat.startsWith('EL-')) + return 'redhat'; + else if (plat.startsWith('F-')) + return 'fedora'; + return 'unknown'; +} + +function get_installer(plat) { + if (plat.startsWith('F-')) + return 'dnf'; + else if (plat.startsWith('EL-')) { + var a = plat.split('-'); + if (a[1] >= 8) + return 'dnf'; + } + return 'yum'; +} + +function uses_systemd(plat) { + if (plat.startsWith('EL-')) { + var a = plat.split('-'); + if (a[1] < 7) + return false; + } + return true; +} + +function get_platform_text(p) { + var a = p.split('-'); + return get_platform_name(a[0]) + ' version ' + a[1]; +} +window.onload = function() { + for (var p in supported_versions) { var opt = document.createElement('option'); - opt.text = versions[p]; + opt.text = supported_versions[p]; document.getElementById('version').add(opt); } - verChanged(); + loadPlatforms(); + archChanged(); } function verChanged() { - var newver = document.getElementById('version').value; + /* Just update like the architecture changed */ + archChanged(); +} + +function loadPlatforms() { var platbox = document.getElementById('platform'); while (platbox.options.length > 0) { @@ -31,15 +73,11 @@ function verChanged() { opt.value = -1; platbox.add(opt); - plats = Object.keys(repodata['reporpms'][newver]).sort( - function(a,b) { - return repodata['platforms'][a].s - repodata['platforms'][b].s; - } - ); - for (p in plats) { + platkeys = Object.keys(repodata['platforms']).sort(); + for (var pp in platkeys) { var opt = document.createElement('option'); - opt.text = repodata['platforms'][plats[p]].t; - opt.value = plats[p]; + opt.text = get_platform_text(platkeys[pp]); + opt.value = platkeys[pp]; platbox.add(opt); } @@ -47,7 +85,6 @@ function verChanged() { } function platChanged() { - var ver = document.getElementById('version').value; var plat = document.getElementById('platform').value; var archbox = document.getElementById('arch'); @@ -60,13 +97,9 @@ function platChanged() { return; } - var platname = repodata['platforms'][plat].t; - - archs = Object.keys(repodata['reporpms'][ver][plat]).sort().reverse(); - for (a in archs) { + for (a in repodata['platforms'][plat].sort().reverse()) { var opt = document.createElement('option'); - opt.text = archs[a]; - opt.value = archs[a]; + opt.text = opt.value = repodata['platforms'][plat][a]; archbox.add(opt); } @@ -78,7 +111,7 @@ function archChanged() { var plat = document.getElementById('platform').value; var arch = document.getElementById('arch').value; - if (plat == -1) { + if (!plat || plat == -1) { document.getElementById('reporpm').innerHTML = 'Select version and platform above'; document.getElementById('clientpackage').innerHTML = 'Select version and platform above'; document.getElementById('serverpackage').innerHTML = 'Select version and platform above'; @@ -89,12 +122,14 @@ function archChanged() { var pinfo = repodata['platforms'][plat]; var shortver = ver.replace('.', ''); - var url = 'https://download.postgresql.org/pub/repos/yum/' + ver + '/' + pinfo['p'] + '-' + arch + '/pgdg-' + pinfo['f'] + shortver + '-' + ver + '-' + repodata['reporpms'][ver][plat][arch] + '.noarch.rpm'; + var url = 'https://download.postgresql.org/pub/repos/yum/reporpms/' + plat + '-' + arch + '/pgdg-' + get_rpm_prefix(plat) +'-repo-latest.noarch.rpm'; + + var installer = get_installer(plat); + document.getElementById('reporpm').innerHTML = installer + ' install ' + url; + document.getElementById('clientpackage').innerHTML = installer + ' install postgresql' + shortver; + document.getElementById('serverpackage').innerHTML = installer + ' install postgresql' + shortver + '-server'; - document.getElementById('reporpm').innerHTML = pinfo['i'] + ' install ' + url; - document.getElementById('clientpackage').innerHTML = pinfo['i'] + ' install postgresql' + shortver; - document.getElementById('serverpackage').innerHTML = pinfo['i'] + ' install postgresql' + shortver + '-server'; - if (pinfo.d) { + if (uses_systemd(plat)) { var setupcmd = 'postgresql-' + shortver + '-setup'; if (ver < 10) { setupcmd = 'postgresql' + shortver + '-setup'; diff --git a/tools/ftp/spider_yum.py b/tools/ftp/spider_yum.py index f2b61364..4dfd6191 100755 --- a/tools/ftp/spider_yum.py +++ b/tools/ftp/spider_yum.py @@ -5,53 +5,10 @@ import os import re import json import requests -from decimal import Decimal +from collections import defaultdict from tempfile import NamedTemporaryFile -platform_names = { - 'redhat': 'Red Hat Enterprise Linux {0}', - 'centos': 'CentOS {0}', - 'sl': 'Scientific Linux {0}', - 'fedora': 'Fedora {0}', - 'oraclelinux': 'Oracle Enterprise Linux {0}', - 'ami201503-': 'Amazon Linux AMI201503 {0}', -} -platform_sort = { - 'redhat': 1, - 'centos': 2, - 'sl': 3, - 'fedora': 4, - 'oraclelinux': 5, - 'ami201503-': 6, -} -archs = ['x86_64', 'i386', 'i686', 'ppc64le'] - - -def generate_platform(dirname, familyprefix, ver, installer, systemd): - for f in list(platform_names.keys()): - yield ('%s-%s' % (f, ver), { - 't': platform_names[f].format(ver), - 'p': os.path.join(dirname, '{0}-{1}'.format(familyprefix, ver)), - 'f': f, - 'i': installer, - 'd': systemd, - 's': platform_sort[f] * 1000 - ver, - 'found': False, - }) - - -def get_redhat_systemd(ver): - return (ver >= 7) - - -platforms = {} -for v in range(5, 7 + 1): - platforms.update(dict(generate_platform('redhat', 'rhel', v, 'yum', get_redhat_systemd(v)))) -for v in range(24, 30 + 1): - platforms.update(dict(generate_platform('fedora', 'fedora', v, 'dnf', True))) - -re_reporpm = re.compile('^pgdg-([a-z0-9-]+)([0-9]{2})-[^-]+-(\d+)\.noarch\.rpm$') -re_versiondirs = re.compile(r'^\d+(\.\d+)?$') +re_platformdir = re.compile('^(\w+)-(\d+)-([^-]+)$') if __name__ == "__main__": parser = argparse.ArgumentParser(description="Spider repo RPMs") @@ -60,40 +17,16 @@ if __name__ == "__main__": args = parser.parse_args() - versions = sorted([v for v in os.listdir(args.yumroot) if re_versiondirs.match(v)], key=Decimal, reverse=True) - reporpms = {} - for v in versions: - reporpms[v] = {} - vroot = os.path.join(args.yumroot, v) - for dirpath, dirnames, filenames in os.walk(vroot): - rmatches = [_f for _f in (re_reporpm.match(f) for f in sorted(filenames, reverse=True)) if _f] - - if rmatches: - familypath = os.path.join(*dirpath.split('/')[-2:]) - (familypath, arch) = familypath.rsplit('-', 1) - - for r in rmatches: - shortdist, shortver, ver = r.groups(1) - - found = False - for p, pinfo in list(platforms.items()): - if pinfo['p'] == familypath and pinfo['f'] == shortdist: - if p not in reporpms[v]: - reporpms[v][p] = {} - reporpms[v][p][arch] = max(int(ver), int(reporpms[v][p].get(arch, 0))) - platforms[p]['found'] = True - break - else: - # DEBUG - # print "%s (%s) not found in platform list" % (familypath, shortdist) - pass - - # Filter all platforms that are not used - platforms = {k: v for k, v in platforms.items() if v['found']} - for k, v in platforms.items(): - del v['found'] + platforms = defaultdict(list) + for repodir in os.listdir('{0}/reporpms'.format(args.yumroot)): + m = re_platformdir.match(repodir) + if m: + platname = m.group(1) + platver = m.group(2) + arch = m.group(3) + platforms['{0}-{1}'.format(platname, platver)].append(arch) - j = json.dumps({'platforms': platforms, 'reporpms': reporpms}) + j = json.dumps({'platforms': platforms}) if args.target.startswith('http://') or args.target.startswith('https://'): r = requests.put( -- 2.39.5