--- /dev/null
+
+alter table mydata add column foodata text;
+
--- /dev/null
+[londiste]
+job_name = londiste_db1
+db = dbname=db1
+queue_name = replika
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+
--- /dev/null
+[londiste]
+job_name = londiste_db2
+db = dbname=db2
+queue_name = replika
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+
--- /dev/null
+[londiste]
+job_name = londiste_db3
+db = dbname=db3
+queue_name = replika
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+
--- /dev/null
+[nop_consumer]
+
+job_name = nop_consumer
+
+#_provider_db = dbname=db1
+dst_db = dbname=db2
+
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+queue_name = fooqueue
+
--- /dev/null
+[cascade_admin]
+
+node_db = dbname=db2
+
+queue_name = fooqueue
+
--- /dev/null
+[pgqadm]
+
+job_name = ticker_branch
+
+db = dbname=db_branch
+
+# how often to run maintenance [minutes]
+maint_delay_min = 1
+
+# how often to check for activity [secs]
+loop_delay = 0.5
+
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+use_skylog = 0
+
+connection_lifetime = 21
+
+queue_refresh_period = 10
+
--- /dev/null
+[pgqadm]
+job_name = ticker_db1
+db = dbname=db1
+loop_delay = 0.5
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
--- /dev/null
+[pgqadm]
+job_name = ticker_db2
+db = dbname=db2
+loop_delay = 0.5
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
--- /dev/null
+[pgqadm]
+job_name = ticker_db3
+db = dbname=db3
+loop_delay = 0.5
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
--- /dev/null
+[nop_worker]
+
+job_name = node1_worker
+
+dst_db = dbname=db1
+
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+queue_name = fooqueue
+
--- /dev/null
+[nop_worker]
+
+job_name = node2_worker
+
+dst_db = dbname=db2
+
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+queue_name = fooqueue
+
--- /dev/null
+[nop_worker]
+
+job_name = node3_worker
+
+dst_db = dbname=db3
+
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+
+queue_name = fooqueue
+
--- /dev/null
+
+create table footable (
+ id serial primary key,
+ username text not null,
+ utype int4 not null check (utype in (1,2,3))
+);
+create index uindex on footable (username);
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+dropdb db1
+dropdb db2
+dropdb db3
+
+createdb db1
+createdb db2
+createdb db3
+
+pgqadm.py conf/ticker_db1.ini install
+pgqadm.py conf/ticker_db2.ini install
+pgqadm.py conf/ticker_db3.ini install
+
+lst="part1 part2 part3 part4 full1 full2 full3 full4"
+
+for db in $lst; do
+ echo dropdb $db
+ dropdb $db
+done
+for db in $lst; do
+ echo createdb $db
+ createdb $db
+done
--- /dev/null
+#! /usr/bin/env python
+
+import sys, time, skytools
+
+from pgq.cascade.consumer import CascadedConsumer
+
+class PlainCascadedConsumer(CascadedConsumer):
+ def process_remote_event(self, src_curs, dst_curs, ev):
+ ev.tag_done()
+
+if __name__ == '__main__':
+ script = PlainCascadedConsumer('nop_consumer', 'dst_db', sys.argv[1:])
+ script.start()
+
--- /dev/null
+#! /usr/bin/env python
+
+import sys, time, skytools
+
+from pgq.cascade.worker import CascadedWorker
+
+class PlainCascadedWorker(CascadedWorker):
+ def process_remote_event(self, src_db, dst_db, ev):
+ self.log.info("got events: %s / %s" % (ev.ev_type, ev.ev_data))
+ ev.tag_done()
+
+if __name__ == '__main__':
+ script = PlainCascadedWorker('nop_worker', 'dst_db', sys.argv[1:])
+ script.start()
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+mkdir -p log pid
+
+./zstop.sh
+
+v=
+v=-v
+v=-q
+
+cleardb() {
+ psql -q -d $db -c '
+ set client_min_messages=warning;
+ drop schema if exists londiste cascade;
+ drop schema if exists pgq_node cascade;
+ drop schema if exists pgq cascade;
+ drop table if exists mydata;
+ drop table if exists footable;
+ drop sequence if exists footable_id_seq;
+ '
+}
+
+run() {
+ echo "$ $*"
+ "$@"
+}
+
+for db in db1 db2 db3; do
+ pgqadm.py conf/ticker_$db.ini -k
+ cleardb $db
+done
+
+run ./plainconsumer.py -s conf/nop_consumer.ini
+
+rm -f log/*.log
+
+set -e
+
+run cat conf/ticker_db1.ini
+
+#echo " # pgqadm install # "
+run pgqadm.py $v conf/ticker_db1.ini install
+run pgqadm.py $v conf/ticker_db2.ini install
+run pgqadm.py $v conf/ticker_db3.ini install
+
+#echo " # pgqadm ticker # "
+run pgqadm.py $v -d conf/ticker_db1.ini ticker
+run pgqadm.py $v -d conf/ticker_db2.ini ticker
+run pgqadm.py $v -d conf/ticker_db3.ini ticker
+
+if false; then
+
+#echo " # setadm create-node # "
+run setadm.py $v --worker=node1_worker conf/setadm.ini create-root node1 'dbname=db1'
+run setadm.py $v --worker=node2_worker conf/setadm.ini create-branch node2 'dbname=db2' --provider='dbname=db1'
+run setadm.py $v --worker=node3_worker conf/setadm.ini create-branch node3 'dbname=db3' --provider='dbname=db2'
+
+#echo " # setadm status # "
+run setadm.py $v conf/setadm.ini status
+
+#echo " # plainconsumer # "
+run ./plainconsumer.py $v conf/nop_consumer.ini --register --provider='dbname=db1'
+run ./plainconsumer.py $v -d conf/nop_consumer.ini
+
+#echo " # plainworker # "
+run ./plainworker.py $v -d conf/worker_db1.ini
+run ./plainworker.py $v -d conf/worker_db2.ini
+run ./plainworker.py $v -d conf/worker_db3.ini
+
+
+#echo " # insert_event() # "
+run psql db1 -c "select pgq.insert_event('fooqueue', 'tmp', 'data')"
+
+fi
+
+grep -E 'ERR|WARN|CRIT' log/*.log || true
+
+#exit 0
+
+# -- londiste
+
+for db in db1 db2 db3; do
+ run psql $db -c "create table mydata (id serial primary key, data text)"
+done
+
+run cat conf/londiste_db1.ini
+
+run londiste.py $v conf/londiste_db1.ini create-root rnode1 'dbname=db1'
+run londiste.py $v conf/londiste_db2.ini create-branch rnode2 'dbname=db2' --provider='dbname=db1'
+run londiste.py $v conf/londiste_db3.ini create-branch rnode3 'dbname=db3' --provider='dbname=db2'
+
+run londiste.py $v -d conf/londiste_db1.ini replay
+run londiste.py $v -d conf/londiste_db2.ini replay
+run londiste.py $v -d conf/londiste_db3.ini replay
+
+run londiste.py $v conf/londiste_db1.ini add-table mydata
+run londiste.py $v conf/londiste_db2.ini add-table mydata
+run londiste.py $v conf/londiste_db3.ini add-table mydata
+
+run cat addcol.sql
+run londiste.py $v conf/londiste_db1.ini execute addcol.sql
+
+run sleep 10
+run psql db3 -c '\d mydata'
+
+run londiste.py $v conf/londiste_db1.ini status
+run londiste.py $v conf/londiste_db3.ini change-provider --provider=rnode1
+run londiste.py $v conf/londiste_db1.ini status
+run londiste.py $v conf/londiste_db1.ini switchover --target=rnode2
+run londiste.py $v conf/londiste_db1.ini status
+
+run cat footable.sql
+
+run psql db2 -f footable.sql
+run londiste.py $v conf/londiste_db2.ini add-seq footable_id_seq
+run londiste.py $v conf/londiste_db2.ini add-table footable
+run londiste.py $v conf/londiste_db1.ini add-seq footable_id_seq --create
+run londiste.py $v conf/londiste_db1.ini add-table footable --create
+run londiste.py $v conf/londiste_db3.ini add-seq footable_id_seq --create
+run londiste.py $v conf/londiste_db3.ini add-table footable --create
+
+#run psql db2 -f footable.sql
+#run londiste.py $v conf/londiste_db2.ini add-seq footable_id_seq
+#run londiste.py $v conf/londiste_db2.ini add-table footable
+#run londiste.py $v conf/londiste_db2.ini add-seq footable_id_seq --create
+#run londiste.py $v conf/londiste_db1.ini add-table footable --create
+#run londiste.py $v conf/londiste_db2.ini add-seq footable_id_seq --create
+#run londiste.py $v conf/londiste_db3.ini add-table footable --create
+
+run psql db3 -c '\d footable'
+
+sleep 3
+
+grep -E 'ERR|WARN|CRIT' log/*.log || echo "All OK"
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+pgqadm.py conf/ticker_db1.ini status
+pgqadm.py conf/ticker_db2.ini status
+pgqadm.py conf/ticker_db3.ini status
+
+setadm.py -v conf/setadm.ini status
+
+
--- /dev/null
+#! /bin/sh
+
+grep -E 'ERR|WARN|CRIT' log/*.log || echo "All OK"
+
--- /dev/null
+#! /bin/sh
+
+#. ../env.sh
+
+for p in pid/*.pid*; do
+ test -f "$p" || continue
+ pid=`cat "$p"`
+ test -d "/proc/$pid" || {
+ rm -f "$p"
+ continue
+ }
+ kill "$pid"
+done
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+./plainconsumer.py -v conf/nop_consumer.ini --register
+./plainconsumer.py -v conf/nop_consumer.ini
+
PYTHONPATH=../../python:$PYTHONPATH
PATH=../../python:../../python/bin:../../scripts:$PATH
-export PYTHONPATH PATH
+#LD_LIBRARY_PATH=/opt/apps/py26/lib:$LD_LIBRARY_PATH
+#PATH=/opt/apps/py26/bin:$PATH
+export PYTHONPATH PATH LD_LIBRARY_PATH PATH
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+lst="part1 part2 part3 part4 full1 full2 full3 full4"
+
+for db in $lst; do
+ echo dropdb $db
+ dropdb $db
+done
+for db in $lst; do
+ echo createdb $db
+ createdb $db
+done
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+mkdir -p log pid conf
+
+./zstop.sh
+
+v=
+v=-q
+v=-v
+
+cleardb() {
+ echo "Clearing database $1"
+ psql -q -d $1 -c '
+ set client_min_messages=warning;
+ drop schema if exists londiste cascade;
+ drop schema if exists pgq_node cascade;
+ drop schema if exists pgq cascade;
+ drop table if exists mydata;
+ drop table if exists footable;
+ drop sequence if exists footable_id_seq;
+ '
+}
+
+run() {
+ echo "$ $*"
+ "$@"
+}
+
+part_list="part1 part2 part3 part4"
+full_list="full1 full2 full3 full4"
+merge_list=""
+for dst in $full_list; do
+ for src in $part_list; do
+ merge_list="$merge_list ${src}_${dst}"
+ done
+done
+all_list="$part_list $full_list"
+
+echo " * create configs * "
+
+# create ticker conf
+for db in $all_list; do
+cat > conf/ticker_$db.ini << EOF
+[pgqadm]
+job_name = ticker_$db
+db = dbname=$db
+loop_delay = 0.5
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+done
+
+# partition replicas
+for db in $part_list; do
+
+# londiste on part node
+cat > conf/londiste_$db.ini << EOF
+[londiste]
+job_name = londiste_$db
+db = dbname=$db
+queue_name = replika_$db
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+# londiste on combined nodes
+for dst in full1 full2; do
+cat > conf/londiste_${db}_${dst}.ini << EOF
+[londiste]
+job_name = londiste_${db}_${dst}
+db = dbname=$dst
+queue_name = replika_$db
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+done
+done
+
+# full replicas
+for db in $full_list; do
+
+cat > conf/londiste_$db.ini << EOF
+[londiste]
+job_name = londiste_$db
+db = dbname=$db
+queue_name = replika
+logfile = log/%(job_name)s.log
+pidfile = pid/%(job_name)s.pid
+EOF
+
+done
+
+
+
+for db in $part_list $full_list; do
+ cleardb $db
+done
+
+echo "clean logs"
+rm -f log/*.log
+
+set -e
+
+for db in $all_list; do
+ run pgqadm.py $v conf/ticker_$db.ini install
+done
+
+run londiste.py $v conf/londiste_full1.ini create-root fnode1 'dbname=full1'
+run londiste.py $v conf/londiste_full2.ini create-branch fnode2 'dbname=full2' --provider='dbname=full1'
+run londiste.py $v conf/londiste_full3.ini create-branch fnode3 'dbname=full3' --provider='dbname=full1'
+run londiste.py $v conf/londiste_full4.ini create-leaf fnode4 'dbname=full4' --provider='dbname=full2'
+
+run londiste.py $v conf/londiste_part1.ini create-root p1root 'dbname=part1'
+run londiste.py $v conf/londiste_part2.ini create-root p2root 'dbname=part2'
+run londiste.py $v conf/londiste_part3.ini create-root p3root 'dbname=part3'
+run londiste.py $v conf/londiste_part4.ini create-root p4root 'dbname=part4'
+
+
+for dst in full1 full2; do
+ for src in $part_list; do
+ run londiste.py $v conf/londiste_${src}_${dst}.ini \
+ create-leaf merge_${src}_${dst} "dbname=$dst" \
+ --provider="dbname=$src" --merge="replika"
+ done
+done
+
+for db in $all_list; do
+ run pgqadm.py $v -d conf/ticker_$db.ini ticker
+ run londiste.py $v -d conf/londiste_$db.ini replay
+done
+
+for dst in full1 full2; do
+ for src in $part_list; do
+ run londiste.py $v -d conf/londiste_${src}_${dst}.ini replay
+ done
+done
+
+for db in $part_list; do
+ run psql $db -c "create table mydata (id int4 primary key, data text)"
+ run londiste.py $v conf/londiste_$db.ini add-table mydata
+done
+for n in 1 2 3 4; do
+ run psql -d part$n -c "insert into mydata values ($n, 'part$n')"
+done
+
+for db in full1; do
+ run psql $db -c "create table mydata (id int4 primary key, data text)"
+ run londiste.py $v conf/londiste_$db.ini add-table mydata
+ for src in $part_list; do
+ run londiste.py $v conf/londiste_${src}_${db}.ini add-table mydata
+ done
+done
+
+run sleep 10
+
+for n in 1 2 3 4; do
+ run psql -d part$n -c "insert into mydata values (4 + $n, 'part$n')"
+done
+
+run sleep 10
+
+for db in full1; do
+run psql -d $db -c "select * from mydata order by id"
+run psql -d $db -c "select * from londiste.table_info order by queue_name"
+done
+run psql -d full1 -c "select * from londiste.get_table_list('replika_part1')"
+run psql -d full1 -c "select * from londiste.get_table_list('replika_part2')"
+
+./zcheck.sh
+
--- /dev/null
+#! /bin/sh
+
+grep -E 'ERR|WARN|CRIT' log/*.log || echo "All OK"
+
--- /dev/null
+#! /bin/sh
+
+#. ../env.sh
+
+for p in pid/*.pid*; do
+ test -f "$p" || continue
+ pid=`cat "$p"`
+ test -d "/proc/$pid" || {
+ rm -f "$p"
+ continue
+ }
+ kill "$pid"
+done
+
--- /dev/null
+#! /usr/bin/env python
+
+import sys, time
+import skytools.psycopgwrapper
+import skytools._cquoting, skytools._pyquoting
+from decimal import Decimal
+
+# create a DictCursor row
+class fake_cursor:
+ index = {'id': 0, 'data': 1}
+ description = ['x', 'x']
+dbrow = skytools.psycopgwrapper._CompatRow(fake_cursor())
+dbrow[0] = '123'
+dbrow[1] = 'value'
+
+def regtest(name, func, cases):
+ bad = 0
+ for dat, res in cases:
+ res2 = func(dat)
+ if res != res2:
+ print("failure: %s(%s) = %s (expected %s)" % (name, repr(dat), repr(res2), repr(res)))
+ bad += 1
+ if bad:
+ print("%-20s: failed" % name)
+ else:
+ print("%-20s: OK" % name)
+
+
+sql_literal = [
+ [None, "null"],
+ ["", "''"],
+ ["a'b", "'a''b'"],
+ [r"a\'b", r"E'a\\''b'"],
+ [1, "'1'"],
+ [True, "'True'"],
+ [Decimal(1), "'1'"],
+]
+regtest("quote_literal/c", skytools._cquoting.quote_literal, sql_literal)
+regtest("quote_literal/py", skytools._pyquoting.quote_literal, sql_literal)
+
+sql_copy = [
+ [None, "\\N"],
+ ["", ""],
+ ["a'\tb", "a'\\tb"],
+ [r"a\'b", r"a\\'b"],
+ [1, "1"],
+ [True, "True"],
+ [u"qwe", "qwe"],
+ [Decimal(1), "1"],
+]
+regtest("quote_copy/c", skytools._cquoting.quote_copy, sql_copy)
+regtest("quote_copy/py", skytools._pyquoting.quote_copy, sql_copy)
+
+sql_bytea_raw = [
+ [None, None],
+ ["", ""],
+ ["a'\tb", "a'\\011b"],
+ [r"a\'b", r"a\\'b"],
+ ["\t\344", r"\011\344"],
+]
+regtest("quote_bytea_raw/c", skytools._cquoting.quote_bytea_raw, sql_bytea_raw)
+regtest("quote_bytea_raw/py", skytools._pyquoting.quote_bytea_raw, sql_bytea_raw)
+
+t_urlenc = [
+ [{}, ""],
+ [{'a': 1}, "a=1"],
+ [{'a': None}, "a"],
+ [{'qwe': 1, u'zz': u"qwe"}, "qwe=1&zz=qwe"],
+ [{'a': '\000%&'}, "a=%00%25%26"],
+ [dbrow, 'data=value&id=123'],
+ [{'a': Decimal("1")}, "a=1"],
+]
+regtest("db_urlencode/c", skytools._cquoting.db_urlencode, t_urlenc)
+regtest("db_urlencode/py", skytools._pyquoting.db_urlencode, t_urlenc)
+
+t_urldec = [
+ ["", {}],
+ ["a=b&c", {'a': 'b', 'c': None}],
+ ["&&b=f&&", {'b': 'f'}],
+ [u"abc=qwe", {'abc': 'qwe'}],
+ ["b=", {'b': ''}],
+ ["b=%00%45", {'b': '\x00E'}],
+]
+regtest("db_urldecode/c", skytools._cquoting.db_urldecode, t_urldec)
+regtest("db_urldecode/py", skytools._pyquoting.db_urldecode, t_urldec)
+
+t_unesc = [
+ ["", ""],
+ ["\\N", "N"],
+ ["abc", "abc"],
+ [u"abc", "abc"],
+ [r"\0\000\001\01\1", "\0\000\001\001\001"],
+ [r"a\001b\tc\r\n", "a\001b\tc\r\n"],
+]
+regtest("unescape/c", skytools._cquoting.unescape, t_unesc)
+regtest("unescape/py", skytools._pyquoting.unescape, t_unesc)
+
projects / skytools.git / commitdiff