From 795aca1531b325455349306da0ebe891bc73c659 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Thu, 28 Apr 2016 16:57:14 +0200 Subject: [PATCH] Remove old migrate commands, not been used for years --- tools/migrate/0_suck_data.sh | 27 --- tools/migrate/1_crunch_in_sql.sql | 209 ------------------------ tools/migrate/2_convert_to_markdown.pl | 123 -------------- tools/migrate/README | 1 - tools/migrate/migrate_single_account.py | 100 ------------ 5 files changed, 460 deletions(-) delete mode 100755 tools/migrate/0_suck_data.sh delete mode 100644 tools/migrate/1_crunch_in_sql.sql delete mode 100644 tools/migrate/2_convert_to_markdown.pl delete mode 100644 tools/migrate/README delete mode 100755 tools/migrate/migrate_single_account.py diff --git a/tools/migrate/0_suck_data.sh b/tools/migrate/0_suck_data.sh deleted file mode 100755 index 9e3c17e6..00000000 --- a/tools/migrate/0_suck_data.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# This script sucks data over from the old server - -if [ "$1" == "" -o "$2" == "" ]; then - echo "Usage: 0_suck_data.sh " - exit 1 -fi -O=$1 -N=$2 - - -( - echo "BEGIN TRANSACTION;DROP SCHEMA IF EXISTS oldweb CASCADE ; CREATE SCHEMA oldweb; SET search_path='oldweb';" - - cat <1) - AND id NOT IN ( - SELECT min(id) FROM profserv_professionalservice - GROUP BY organisation_id HAVING count(*)>1) -); - -ALTER TABLE profserv_professionalservice -ADD CONSTRAINT profserv_professionalservice_organisation_id_key -UNIQUE (organisation_id); - - --- Add product categories and license types -TRUNCATE TABLE downloads_category RESTART IDENTITY CASCADE; -TRUNCATE TABLE downloads_licencetype RESTART IDENTITY CASCADE; -INSERT INTO downloads_category (catname, blurb) SELECT name,blurb FROM oldweb.product_categories ORDER BY id; -COPY downloads_licencetype (id, typename) FROM stdin; -1 Open source -2 Freeware -3 Commercial -4 Multiple -\. -SELECT setval('downloads_category_id_seq', 5); - --- Add products -INSERT INTO downloads_product (name, approved, publisher_id, url, category_id, licencetype_id, description, price, lastconfirmed) -SELECT products.name, products.approved, -(SELECT id FROM core_organisation WHERE core_organisation.name=oo.name), -products.url, -(SELECT id FROM downloads_category WHERE downloads_category.catname=oldweb.product_categories.name), -(SELECT id FROM downloads_licencetype WHERE typename=CASE WHEN licence='o' THEN 'Open source' WHEN licence='c' THEN 'Commercial' WHEN licence='f' THEN 'Freeware' WHEN licence='m' THEN 'Multiple' END), -description, COALESCE(price,'') , products.lastconfirmed -FROM oldweb.products -INNER JOIN oldweb.organisations oo ON publisher=oo.id -INNER JOIN oldweb.product_categories ON category=product_categories.id; - - --- Surveys -TRUNCATE TABLE survey_surveyanswer CASCADE; -TRUNCATE TABLE survey_survey CASCADE; - -INSERT INTO survey_survey (id, question, opt1, opt2, opt3, opt4, opt5, opt6, opt7, opt8, posted, current) -SELECT surveyid, question, coalesce(opt1,''), coalesce(opt2,''), coalesce(opt3,''), coalesce(opt4,''), coalesce(opt5,''), coalesce(opt6,''), coalesce(opt7,''), coalesce(opt8,''), modified, current -FROM oldweb.survey_questions INNER JOIN oldweb.surveys ON surveys.id=survey_questions.surveyid; - -INSERT INTO survey_surveyanswer (survey_id, tot1, tot2, tot3, tot4, tot5, tot6, tot7, tot8) -SELECT id, tot1, tot2, tot3, tot4, tot5, tot6, tot7, tot8 -FROM oldweb.surveys; - -SELECT setval('survey_survey_id_seq', max(id)) FROM survey_survey; - --- mailinglists -TRUNCATE TABLE lists_mailinglist CASCADE; -TRUNCATE TABLE lists_mailinglistgroup CASCADE; - -INSERT INTO lists_mailinglistgroup (id, groupname, sortkey) -SELECT id, name, sortkey FROM oldweb.listgroups; - -INSERT INTO lists_mailinglist (id, group_id, listname, active, externallink, description, shortdesc) -SELECT id, grp, name, active::boolean, NULL, description, coalesce(shortdesc,'') FROM oldweb.lists; - -SELECT setval('lists_mailinglist_id_seq', max(id)) FROM lists_mailinglist; -SELECT setval('lists_mailinglistgroup_id_seq', max(id)) FROM lists_mailinglistgroup; - --- contributors -TRUNCATE TABLE contributors_contributor; -INSERT INTO contributors_contributor (ctype_id, lastname, firstname, email, company, companyurl, location, contribution) -SELECT ct.id, lastname, firstname, email, company, companyurl, location, contribution -FROM oldweb.developers d -INNER JOIN oldweb.developers_types dt ON dt.type=d.type -INNER JOIN contributors_contributortype ct ON ct.typename=dt.typename; -SELECT setval('contributors_contributor_id_seq', max(id)) FROM contributors_contributor; diff --git a/tools/migrate/2_convert_to_markdown.pl b/tools/migrate/2_convert_to_markdown.pl deleted file mode 100644 index 607ddaa9..00000000 --- a/tools/migrate/2_convert_to_markdown.pl +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env perl - -use HTML::WikiConverter; -use DBI; - -$| = 1; - -$conv = new HTML::WikiConverter(dialect=>'Markdown', link_style=>'inline'); -if (!$ARGV[0]) { - print "Usage: 2_convert_to_markdown.pl \n"; - exit(1); -} -$dbh = DBI->connect("dbi:Pg:" . $ARGV[0], '', '', { AutoCommit=> 0}); - -print "Converting news...\n"; -News(); -print "Converting events..\n"; -Events(); -print "Converting quotes...\n"; -Quotes(); #NOTE! Quotes need manual cleanup! (more than the others, but they all do, really) - -sub News() { - $dbh->do("TRUNCATE TABLE news_newsarticle"); - $r = $dbh->selectall_arrayref("SELECT id,posted,posted_by,headline,summary,story FROM oldweb.news INNER JOIN oldweb.news_text ON news.id=news_text.newsid AND news_text.language='en' WHERE approved ORDER BY id"); - $ins = $dbh->prepare("INSERT INTO news_newsarticle (id, org_id, approved, date, title, content) VALUES (?, 0, 't', ?, ?, ?)"); - print "Done loading, now starting conversion...\n"; - $last = -10;$now = 0; - for my $row (@$r) { - $now++; - if ($now - $last > 2) { - print "$now / " . scalar(@$r) . "\r"; - $last = $now; - } - - $ins->execute($row->[0], - $row->[1], - $row->[3], - ConvertHtmlToMarkdown($row->[5]) - ) || die "Failed to insert!\n"; - } - $dbh->do("SELECT setval('news_newsarticle_id_seq', max(id)+1) FROM news_newsarticle") || die "Failed to setval"; - $dbh->commit(); - print "Done.\n"; -} - -sub Events { - $dbh->do("TRUNCATE TABLE events_event") || die "Failed to truncate\n"; - $r = $dbh->selectall_arrayref("SELECT id,posted,posted_by,start_date,end_date,training,COALESCE(organisation,''),country,state,city,event,COALESCE(summary,''),COALESCE(details,'') FROM oldweb.events INNER JOIN oldweb.events_text ON events.id=events_text.eventid AND events_text.language='en' INNER JOIN oldweb.events_location ON events.id=events_location.eventid WHERE approved ORDER BY id"); - $ins = $dbh->prepare("INSERT INTO events_event (id,approved,org_id,title,city,state,country_id,training,startdate,enddate,summary,details) VALUES (?,'t',(SELECT id FROM core_organisation WHERE name=?),?,?,?,?,?,?,?,?,?)"); - $last = -10;$now = 0; - for my $row (@$r) { - $now++; - if ($now - $last > 2) { - print "$now / " . scalar(@$r) . "\r"; - $last = $now; - } - $ins->execute($row->[0], - $row->[6], - $row->[10], - $row->[9], - $row->[8], - $row->[7], - $row->[5], - $row->[3], - $row->[4], - ConvertHtmlToMarkdown($row->[11]), - ConvertHtmlToMarkdown($row->[12]) - ) || die "Failed to insert\n"; - } - $dbh->do("SELECT setval('events_event_id_seq', max(id)+1) FROM events_event") || die "Failed to setval\n"; - $dbh->commit(); -} - -sub Quotes { - $dbh->do("TRUNCATE TABLE quotes_quote") || die "Failed to truncate\n"; - $r = $dbh->selectall_arrayref("SELECT quoteid,quote,tagline FROM oldweb.quotes_text WHERE language='en' ORDER BY 1"); - $ins = $dbh->prepare("INSERT INTO quotes_quote(id,approved,quote,who,org,link) VALUES (?,'t',?,?,?,?)"); - $last = -10;$now = 0; - for my $row (@$r) { - $now++; - if ($now - $last > 2) { - print "$now / " . scalar(@$r) . "\r"; - $last = $now; - } - $tag = $row->[2]; - if ($tag =~ /^([^<]+), ([^<]+)<\/a>(.*)$/) { - print "match $tag\n"; - $who = $1; - $link = $2; - $org = $3 . $4; - } - elsif ($tag =~ /^([^,]+), (.*)$/s) { - $who = $1; - $org = $2; - $link = ''; - } - elsif ($tag =~ /^([^<]+)<\/a>(.*)$/) { - $who = ''; - $link = $1; - $org = $3 . $4; - } - else { - die "Could not parse $tag\n"; - } - $ins->execute($row->[0], $row->[1], $who, $org, $link) || die "Failed to insert\n"; - } - $dbh->do("SELECT setval('quotes_quote_id_seq', max(id)+1) FROM quotes_quote") || die "Failed to setval\n"; - $dbh->commit(); -} - -sub ConvertHtmlToMarkdown { - $html = shift; - # Blank? - return "" if $html =~ /^\s*$/; - # First apply our website style translation thingy - $html =~ s/[\r\n]+/

\n/g; -# print "Attempt to convert\n"; -# print $html ."\n"; -# print "result:\n"; -# print $conv->html2wiki($html); - - return $conv->html2wiki($html); -} diff --git a/tools/migrate/README b/tools/migrate/README deleted file mode 100644 index 00494c87..00000000 --- a/tools/migrate/README +++ /dev/null @@ -1 +0,0 @@ -Tools to migrate data from the old website diff --git a/tools/migrate/migrate_single_account.py b/tools/migrate/migrate_single_account.py deleted file mode 100755 index 79f44951..00000000 --- a/tools/migrate/migrate_single_account.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# -# This script will migrate a single user from the old system to the new one. This will -# reset the users password in the process - there is no way around that. -# -# This process is automatically done when the user logs in to the new website, but it -# is useful to do this if the user has lost his password before he/she logs into the new -# site for the first time, since the password recovery feature only works once the account -# has been migrated. -# - - -import sys -import os - -# Set up for accessing django -from django.core.management import setup_environ -sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../../pgweb')) -import settings -setup_environ(settings) - -from django.contrib.auth.models import User -from django.db import connection, transaction - -from pgweb.core.models import UserProfile - - -from random import choice -import string - -# This does not generate a strong password. But it's only a temporary one anyway, -# so this doesn't matter. -def GenPasswd(l): - return ''.join([choice(string.letters + string.digits) for i in xrange(l)]) - -if __name__=="__main__": - if len(sys.argv) != 2: - print "Usage: migrate_single_account.py " - sys.exit(1) - - u = sys.argv[1].lower() - - try: - user = User.objects.get(username=u) - print "User %s (%s %s) already exists!" % (u, user.first_name, user.last_name) - sys.exit(1) - except User.DoesNotExist: - print "User does not exist in new system, that's expected..." - pass - - transaction.enter_transaction_management() - transaction.managed() - - # Attempt login against old system - curs = connection.cursor() - curs.execute("SELECT userid, fullname, email, sshkey FROM users_old WHERE userid=%s", (u,)) - rows = curs.fetchall() - if len(rows) == 0: - print "User %s does not exist in the old system." % u - sys.exit(1) - if len(rows) != 1: - print "Userid lookup returned %s rows, not 1!" % len(rows) - sys.exit(1) - print "Found user %s in the old system" % u - print "Fullname: %s" % rows[0][1] - print "Email: %s" % rows[0][2] - print "" - while True: - yn = raw_input("Are you sure you want to migrate this user, resetting his/her password? [y/n]?") - if yn == "y": - print "Ok, migrating..." - break - elif yn == "n": - print "Aborting" - sys.exit(1) - else: - continue - - pwd = GenPasswd(12) - print "New password: %s" % pwd - - namepieces = rows[0][1].split(None, 2) - if len(namepieces) == 0: namepieces = ['', ''] - if len(namepieces) == 1: namepieces.append('') - print "Creating new user record..." - user = User(username=u, email=rows[0][2], first_name=namepieces[0], last_name=namepieces[1]) - user.set_password(pwd) - user.save() - if rows[0][3]: - print "Migrating SSH key..." - profile = UserProfile(user=user) - profile.sshkey = rows[0][3] - profile.save() - - print "Removing user from the old system..." - curs.execute("SELECT * FROM community_login_old_delete(%s)", (u, )) - - transaction.commit() - - print "Done. Don't forget to email the user at %s, informing him/her about the new password %s" % (rows[0][2], pwd) -- 2.39.5