From 50abdba44a031ad40b1886f941479f203ca92039 Mon Sep 17 00:00:00 2001 From: Marko Kreen Date: Tue, 13 Mar 2007 11:52:09 +0000 Subject: [PATCH] final public release --- AUTHORS | 4 + COPYRIGHT | 16 + Makefile | 79 +++ NEWS | 5 + README | 47 ++ config.mak.in | 10 + configure.ac | 24 + debian/changelog | 6 + debian/packages.in | 44 ++ doc/Makefile | 41 ++ doc/TODO.txt | 44 ++ doc/londiste.txt | 64 ++ doc/overview.txt | 179 +++++ doc/pgq-admin.txt | 42 ++ doc/pgq-nodupes.txt | 33 + doc/pgq-sql.txt | 191 ++++++ doc/walmgr.txt | 82 +++ python/conf/londiste.ini | 16 + python/conf/pgqadm.ini | 18 + python/conf/skylog.ini | 76 ++ python/conf/wal-master.ini | 18 + python/conf/wal-slave.ini | 15 + python/londiste.py | 130 ++++ python/londiste/__init__.py | 12 + python/londiste/compare.py | 45 ++ python/londiste/file_read.py | 52 ++ python/londiste/file_write.py | 67 ++ python/londiste/installer.py | 26 + python/londiste/playback.py | 558 +++++++++++++++ python/londiste/repair.py | 284 ++++++++ python/londiste/setup.py | 580 ++++++++++++++++ python/londiste/syncer.py | 177 +++++ python/londiste/table_copy.py | 107 +++ python/pgq/__init__.py | 6 + python/pgq/consumer.py | 410 +++++++++++ python/pgq/event.py | 60 ++ python/pgq/maint.py | 99 +++ python/pgq/producer.py | 41 ++ python/pgq/status.py | 93 +++ python/pgq/ticker.py | 172 +++++ python/pgqadm.py | 162 +++++ python/skytools/__init__.py | 10 + python/skytools/config.py | 139 ++++ python/skytools/dbstruct.py | 380 ++++++++++ python/skytools/gzlog.py | 39 ++ python/skytools/quoting.py | 156 +++++ python/skytools/scripting.py | 523 ++++++++++++++ python/skytools/skylog.py | 173 +++++ python/skytools/sqltools.py | 398 +++++++++++ python/walmgr.py | 648 ++++++++++++++++++ scripts/bulk_loader.ini.templ | 13 + scripts/bulk_loader.py | 181 +++++ scripts/catsql.py | 141 ++++ scripts/cube_dispatcher.ini.templ | 23 + scripts/cube_dispatcher.py | 175 +++++ scripts/queue_mover.ini.templ | 14 + scripts/queue_mover.py | 30 + scripts/queue_splitter.ini.templ | 13 + scripts/queue_splitter.py | 33 + scripts/scriptmgr.ini.templ | 43 ++ scripts/scriptmgr.py | 220 ++++++ scripts/table_dispatcher.ini.templ | 31 + scripts/table_dispatcher.py | 124 ++++ setup.py | 36 + source.cfg | 13 + sql/Makefile | 10 + sql/logtriga/Makefile | 12 + sql/logtriga/README.logtriga | 47 ++ sql/logtriga/expected/logtriga.out | 95 +++ sql/logtriga/logtriga.c | 500 ++++++++++++++ sql/logtriga/logtriga.sql.in | 10 + sql/logtriga/sql/logtriga.sql | 74 ++ sql/logtriga/textbuf.c | 334 +++++++++ sql/logtriga/textbuf.h | 26 + sql/londiste/Makefile | 20 + sql/londiste/README.londiste | 29 + .../expected/londiste_denytrigger.out | 40 ++ sql/londiste/expected/londiste_install.out | 1 + sql/londiste/expected/londiste_provider.out | 135 ++++ sql/londiste/expected/londiste_subscriber.out | 128 ++++ .../functions/londiste.denytrigger.sql | 19 + .../functions/londiste.find_column_types.sql | 26 + .../functions/londiste.find_table_oid.sql | 49 ++ .../functions/londiste.get_last_tick.sql | 13 + sql/londiste/functions/londiste.link.sql | 112 +++ .../functions/londiste.provider_add_seq.sql | 27 + .../functions/londiste.provider_add_table.sql | 48 ++ .../londiste.provider_create_trigger.sql | 33 + .../londiste.provider_get_seq_list.sql | 17 + .../londiste.provider_get_table_list.sql | 18 + .../londiste.provider_notify_change.sql | 26 + .../londiste.provider_refresh_trigger.sql | 44 ++ .../londiste.provider_remove_seq.sql | 26 + .../londiste.provider_remove_table.sql | 30 + .../functions/londiste.set_last_tick.sql | 18 + .../functions/londiste.subscriber_add_seq.sql | 23 + .../londiste.subscriber_add_table.sql | 14 + .../londiste.subscriber_get_seq_list.sql | 17 + .../londiste.subscriber_get_table_list.sql | 35 + .../londiste.subscriber_remove_seq.sql | 27 + .../londiste.subscriber_remove_table.sql | 27 + .../londiste.subscriber_set_table_state.sql | 58 ++ sql/londiste/sql/londiste_denytrigger.sql | 19 + sql/londiste/sql/londiste_install.sql | 8 + sql/londiste/sql/londiste_provider.sql | 68 ++ sql/londiste/sql/londiste_subscriber.sql | 53 ++ sql/londiste/structure/tables.sql | 53 ++ sql/londiste/structure/types.sql | 13 + sql/pgq/Makefile | 48 ++ sql/pgq/README.pgq | 19 + sql/pgq/docs/Languages.txt | 113 +++ sql/pgq/docs/Menu.txt | 43 ++ sql/pgq/docs/Topics.txt | 107 +++ sql/pgq/expected/logutriga.out | 22 + sql/pgq/expected/pgq_init.out | 253 +++++++ sql/pgq/expected/sqltriga.out | 86 +++ sql/pgq/functions/pgq.batch_event_sql.sql | 106 +++ sql/pgq/functions/pgq.batch_event_tables.sql | 67 ++ sql/pgq/functions/pgq.create_queue.sql | 71 ++ sql/pgq/functions/pgq.current_event_table.sql | 25 + sql/pgq/functions/pgq.drop_queue.sql | 56 ++ sql/pgq/functions/pgq.event_failed.sql | 41 ++ sql/pgq/functions/pgq.event_retry.sql | 68 ++ sql/pgq/functions/pgq.event_retry_raw.sql | 66 ++ sql/pgq/functions/pgq.failed_queue.sql | 201 ++++++ sql/pgq/functions/pgq.finish_batch.sql | 32 + sql/pgq/functions/pgq.get_batch_events.sql | 26 + sql/pgq/functions/pgq.get_batch_info.sql | 36 + sql/pgq/functions/pgq.get_consumer_info.sql | 108 +++ sql/pgq/functions/pgq.get_queue_info.sql | 51 ++ sql/pgq/functions/pgq.grant_perms.sql | 37 + sql/pgq/functions/pgq.insert_event.sql | 49 ++ sql/pgq/functions/pgq.insert_event_raw.sql | 87 +++ sql/pgq/functions/pgq.maint_retry_events.sql | 42 ++ sql/pgq/functions/pgq.maint_rotate_tables.sql | 98 +++ .../functions/pgq.maint_tables_to_vacuum.sql | 33 + sql/pgq/functions/pgq.next_batch.sql | 66 ++ sql/pgq/functions/pgq.register_consumer.sql | 120 ++++ sql/pgq/functions/pgq.ticker.sql | 86 +++ sql/pgq/functions/pgq.unregister_consumer.sql | 44 ++ sql/pgq/functions/pgq.version.sql | 12 + sql/pgq/sql/logutriga.sql | 22 + sql/pgq/sql/pgq_init.sql | 66 ++ sql/pgq/sql/sqltriga.sql | 58 ++ sql/pgq/structure/func_internal.sql | 23 + sql/pgq/structure/func_public.sql | 36 + sql/pgq/structure/install.sql | 7 + sql/pgq/structure/tables.sql | 217 ++++++ sql/pgq/structure/triggers.sql | 8 + sql/pgq/structure/types.sql | 47 ++ sql/pgq/triggers/pgq.logutriga.sql | 103 +++ sql/pgq/triggers/pgq.sqltriga.sql | 195 ++++++ sql/pgq_ext/Makefile | 16 + sql/pgq_ext/README.pgq_ext | 52 ++ sql/pgq_ext/expected/test_pgq_ext.out | 85 +++ sql/pgq_ext/functions/track_batch.sql | 39 ++ sql/pgq_ext/functions/track_event.sql | 60 ++ sql/pgq_ext/sql/test_pgq_ext.sql | 26 + sql/pgq_ext/structure/tables.sql | 48 ++ sql/txid/Makefile | 30 + sql/txid/README.txid | 65 ++ sql/txid/epoch.c | 240 +++++++ sql/txid/expected/txid.out | 88 +++ sql/txid/sql/txid.sql | 43 ++ sql/txid/txid.c | 364 ++++++++++ sql/txid/txid.h | 43 ++ sql/txid/txid.schema.sql | 50 ++ sql/txid/txid.std.sql | 77 +++ sql/txid/uninstall_txid.sql | 10 + tests/env.sh | 6 + tests/londiste/conf/fread.ini | 17 + tests/londiste/conf/fwrite.ini | 17 + tests/londiste/conf/linkticker.ini | 17 + tests/londiste/conf/replic.ini | 15 + tests/londiste/conf/tester.ini | 16 + tests/londiste/conf/ticker.ini | 17 + tests/londiste/data.sql | 24 + tests/londiste/env.sh | 7 + tests/londiste/gendb.sh | 47 ++ tests/londiste/run-tests.sh | 58 ++ tests/londiste/stop.sh | 12 + tests/londiste/testing.py | 80 +++ tests/scripts/conf/cube.ini | 18 + tests/scripts/conf/mover.ini | 13 + tests/scripts/conf/table.ini | 29 + tests/scripts/conf/ticker.ini | 26 + tests/scripts/data.sql | 22 + tests/scripts/env.sh | 7 + tests/scripts/gendb.sh | 45 ++ tests/scripts/install.sql | 7 + tests/scripts/run-tests.sh | 15 + tests/scripts/stop.sh | 14 + tests/skylog/logtest.py | 17 + tests/skylog/runtest.sh | 6 + tests/skylog/skylog.ini | 73 ++ tests/skylog/test.ini | 6 + tests/walmgr/conf.master/pg_hba.conf | 75 ++ tests/walmgr/conf.master/pg_ident.conf | 36 + tests/walmgr/conf.master/postgresql.conf | 17 + tests/walmgr/conf.slave/pg_hba.conf | 75 ++ tests/walmgr/conf.slave/pg_ident.conf | 36 + tests/walmgr/conf.slave/postgresql.conf | 434 ++++++++++++ tests/walmgr/run-test.sh | 85 +++ 203 files changed, 15899 insertions(+) create mode 100644 AUTHORS create mode 100644 COPYRIGHT create mode 100644 Makefile create mode 100644 NEWS create mode 100644 README create mode 100644 config.mak.in create mode 100644 configure.ac create mode 100644 debian/changelog create mode 100644 debian/packages.in create mode 100644 doc/Makefile create mode 100644 doc/TODO.txt create mode 100644 doc/londiste.txt create mode 100644 doc/overview.txt create mode 100644 doc/pgq-admin.txt create mode 100644 doc/pgq-nodupes.txt create mode 100644 doc/pgq-sql.txt create mode 100644 doc/walmgr.txt create mode 100644 python/conf/londiste.ini create mode 100644 python/conf/pgqadm.ini create mode 100644 python/conf/skylog.ini create mode 100644 python/conf/wal-master.ini create mode 100644 python/conf/wal-slave.ini create mode 100755 python/londiste.py create mode 100644 python/londiste/__init__.py create mode 100644 python/londiste/compare.py create mode 100644 python/londiste/file_read.py create mode 100644 python/londiste/file_write.py create mode 100644 python/londiste/installer.py create mode 100644 python/londiste/playback.py create mode 100644 python/londiste/repair.py create mode 100644 python/londiste/setup.py create mode 100644 python/londiste/syncer.py create mode 100644 python/londiste/table_copy.py create mode 100644 python/pgq/__init__.py create mode 100644 python/pgq/consumer.py create mode 100644 python/pgq/event.py create mode 100644 python/pgq/maint.py create mode 100644 python/pgq/producer.py create mode 100644 python/pgq/status.py create mode 100644 python/pgq/ticker.py create mode 100755 python/pgqadm.py create mode 100644 python/skytools/__init__.py create mode 100644 python/skytools/config.py create mode 100644 python/skytools/dbstruct.py create mode 100644 python/skytools/gzlog.py create mode 100644 python/skytools/quoting.py create mode 100644 python/skytools/scripting.py create mode 100644 python/skytools/skylog.py create mode 100644 python/skytools/sqltools.py create mode 100755 python/walmgr.py create mode 100644 scripts/bulk_loader.ini.templ create mode 100755 scripts/bulk_loader.py create mode 100755 scripts/catsql.py create mode 100644 scripts/cube_dispatcher.ini.templ create mode 100755 scripts/cube_dispatcher.py create mode 100644 scripts/queue_mover.ini.templ create mode 100755 scripts/queue_mover.py create mode 100644 scripts/queue_splitter.ini.templ create mode 100755 scripts/queue_splitter.py create mode 100644 scripts/scriptmgr.ini.templ create mode 100755 scripts/scriptmgr.py create mode 100644 scripts/table_dispatcher.ini.templ create mode 100755 scripts/table_dispatcher.py create mode 100755 setup.py create mode 100644 source.cfg create mode 100644 sql/Makefile create mode 100644 sql/logtriga/Makefile create mode 100644 sql/logtriga/README.logtriga create mode 100644 sql/logtriga/expected/logtriga.out create mode 100644 sql/logtriga/logtriga.c create mode 100644 sql/logtriga/logtriga.sql.in create mode 100644 sql/logtriga/sql/logtriga.sql create mode 100644 sql/logtriga/textbuf.c create mode 100644 sql/logtriga/textbuf.h create mode 100644 sql/londiste/Makefile create mode 100644 sql/londiste/README.londiste create mode 100644 sql/londiste/expected/londiste_denytrigger.out create mode 100644 sql/londiste/expected/londiste_install.out create mode 100644 sql/londiste/expected/londiste_provider.out create mode 100644 sql/londiste/expected/londiste_subscriber.out create mode 100644 sql/londiste/functions/londiste.denytrigger.sql create mode 100644 sql/londiste/functions/londiste.find_column_types.sql create mode 100644 sql/londiste/functions/londiste.find_table_oid.sql create mode 100644 sql/londiste/functions/londiste.get_last_tick.sql create mode 100644 sql/londiste/functions/londiste.link.sql create mode 100644 sql/londiste/functions/londiste.provider_add_seq.sql create mode 100644 sql/londiste/functions/londiste.provider_add_table.sql create mode 100644 sql/londiste/functions/londiste.provider_create_trigger.sql create mode 100644 sql/londiste/functions/londiste.provider_get_seq_list.sql create mode 100644 sql/londiste/functions/londiste.provider_get_table_list.sql create mode 100644 sql/londiste/functions/londiste.provider_notify_change.sql create mode 100644 sql/londiste/functions/londiste.provider_refresh_trigger.sql create mode 100644 sql/londiste/functions/londiste.provider_remove_seq.sql create mode 100644 sql/londiste/functions/londiste.provider_remove_table.sql create mode 100644 sql/londiste/functions/londiste.set_last_tick.sql create mode 100644 sql/londiste/functions/londiste.subscriber_add_seq.sql create mode 100644 sql/londiste/functions/londiste.subscriber_add_table.sql create mode 100644 sql/londiste/functions/londiste.subscriber_get_seq_list.sql create mode 100644 sql/londiste/functions/londiste.subscriber_get_table_list.sql create mode 100644 sql/londiste/functions/londiste.subscriber_remove_seq.sql create mode 100644 sql/londiste/functions/londiste.subscriber_remove_table.sql create mode 100644 sql/londiste/functions/londiste.subscriber_set_table_state.sql create mode 100644 sql/londiste/sql/londiste_denytrigger.sql create mode 100644 sql/londiste/sql/londiste_install.sql create mode 100644 sql/londiste/sql/londiste_provider.sql create mode 100644 sql/londiste/sql/londiste_subscriber.sql create mode 100644 sql/londiste/structure/tables.sql create mode 100644 sql/londiste/structure/types.sql create mode 100644 sql/pgq/Makefile create mode 100644 sql/pgq/README.pgq create mode 100644 sql/pgq/docs/Languages.txt create mode 100644 sql/pgq/docs/Menu.txt create mode 100644 sql/pgq/docs/Topics.txt create mode 100644 sql/pgq/expected/logutriga.out create mode 100644 sql/pgq/expected/pgq_init.out create mode 100644 sql/pgq/expected/sqltriga.out create mode 100644 sql/pgq/functions/pgq.batch_event_sql.sql create mode 100644 sql/pgq/functions/pgq.batch_event_tables.sql create mode 100644 sql/pgq/functions/pgq.create_queue.sql create mode 100644 sql/pgq/functions/pgq.current_event_table.sql create mode 100644 sql/pgq/functions/pgq.drop_queue.sql create mode 100644 sql/pgq/functions/pgq.event_failed.sql create mode 100644 sql/pgq/functions/pgq.event_retry.sql create mode 100644 sql/pgq/functions/pgq.event_retry_raw.sql create mode 100644 sql/pgq/functions/pgq.failed_queue.sql create mode 100644 sql/pgq/functions/pgq.finish_batch.sql create mode 100644 sql/pgq/functions/pgq.get_batch_events.sql create mode 100644 sql/pgq/functions/pgq.get_batch_info.sql create mode 100644 sql/pgq/functions/pgq.get_consumer_info.sql create mode 100644 sql/pgq/functions/pgq.get_queue_info.sql create mode 100644 sql/pgq/functions/pgq.grant_perms.sql create mode 100644 sql/pgq/functions/pgq.insert_event.sql create mode 100644 sql/pgq/functions/pgq.insert_event_raw.sql create mode 100644 sql/pgq/functions/pgq.maint_retry_events.sql create mode 100644 sql/pgq/functions/pgq.maint_rotate_tables.sql create mode 100644 sql/pgq/functions/pgq.maint_tables_to_vacuum.sql create mode 100644 sql/pgq/functions/pgq.next_batch.sql create mode 100644 sql/pgq/functions/pgq.register_consumer.sql create mode 100644 sql/pgq/functions/pgq.ticker.sql create mode 100644 sql/pgq/functions/pgq.unregister_consumer.sql create mode 100644 sql/pgq/functions/pgq.version.sql create mode 100644 sql/pgq/sql/logutriga.sql create mode 100644 sql/pgq/sql/pgq_init.sql create mode 100644 sql/pgq/sql/sqltriga.sql create mode 100644 sql/pgq/structure/func_internal.sql create mode 100644 sql/pgq/structure/func_public.sql create mode 100644 sql/pgq/structure/install.sql create mode 100644 sql/pgq/structure/tables.sql create mode 100644 sql/pgq/structure/triggers.sql create mode 100644 sql/pgq/structure/types.sql create mode 100644 sql/pgq/triggers/pgq.logutriga.sql create mode 100644 sql/pgq/triggers/pgq.sqltriga.sql create mode 100644 sql/pgq_ext/Makefile create mode 100644 sql/pgq_ext/README.pgq_ext create mode 100644 sql/pgq_ext/expected/test_pgq_ext.out create mode 100644 sql/pgq_ext/functions/track_batch.sql create mode 100644 sql/pgq_ext/functions/track_event.sql create mode 100644 sql/pgq_ext/sql/test_pgq_ext.sql create mode 100644 sql/pgq_ext/structure/tables.sql create mode 100644 sql/txid/Makefile create mode 100644 sql/txid/README.txid create mode 100644 sql/txid/epoch.c create mode 100644 sql/txid/expected/txid.out create mode 100644 sql/txid/sql/txid.sql create mode 100644 sql/txid/txid.c create mode 100644 sql/txid/txid.h create mode 100644 sql/txid/txid.schema.sql create mode 100644 sql/txid/txid.std.sql create mode 100644 sql/txid/uninstall_txid.sql create mode 100644 tests/env.sh create mode 100644 tests/londiste/conf/fread.ini create mode 100644 tests/londiste/conf/fwrite.ini create mode 100644 tests/londiste/conf/linkticker.ini create mode 100644 tests/londiste/conf/replic.ini create mode 100644 tests/londiste/conf/tester.ini create mode 100644 tests/londiste/conf/ticker.ini create mode 100644 tests/londiste/data.sql create mode 100644 tests/londiste/env.sh create mode 100755 tests/londiste/gendb.sh create mode 100755 tests/londiste/run-tests.sh create mode 100755 tests/londiste/stop.sh create mode 100755 tests/londiste/testing.py create mode 100644 tests/scripts/conf/cube.ini create mode 100644 tests/scripts/conf/mover.ini create mode 100644 tests/scripts/conf/table.ini create mode 100644 tests/scripts/conf/ticker.ini create mode 100644 tests/scripts/data.sql create mode 100644 tests/scripts/env.sh create mode 100755 tests/scripts/gendb.sh create mode 100644 tests/scripts/install.sql create mode 100755 tests/scripts/run-tests.sh create mode 100755 tests/scripts/stop.sh create mode 100755 tests/skylog/logtest.py create mode 100755 tests/skylog/runtest.sh create mode 100644 tests/skylog/skylog.ini create mode 100644 tests/skylog/test.ini create mode 100644 tests/walmgr/conf.master/pg_hba.conf create mode 100644 tests/walmgr/conf.master/pg_ident.conf create mode 100644 tests/walmgr/conf.master/postgresql.conf create mode 100644 tests/walmgr/conf.slave/pg_hba.conf create mode 100644 tests/walmgr/conf.slave/pg_ident.conf create mode 100644 tests/walmgr/conf.slave/postgresql.conf create mode 100755 tests/walmgr/run-test.sh diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..f00e3a50 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ + +Marko Kreen - main coder +Martin Pihlak - walmgr + diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 00000000..c20f0b8b --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,16 @@ +SkyTools - tool collection for PostgreSQL + +Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..60ecb1a9 --- /dev/null +++ b/Makefile @@ -0,0 +1,79 @@ + +-include config.mak + +PYTHON ?= python + +pyver = $(shell $(PYTHON) -V 2>&1 | sed 's/^[^ ]* \([0-9]*\.[0-9]*\).*/\1/') + +SUBDIRS = sql + +all: python-all modules-all + +modules-all: config.mak + make -C sql all + +python-all: config.mak + $(PYTHON) setup.py build + +clean: + make -C sql clean + make -C doc clean + $(PYTHON) setup.py clean + rm -rf build + find python -name '*.py[oc]' -print | xargs rm -f + +install: python-install modules-install + +installcheck: + make -C sql installcheck + +modules-install: config.mak + make -C sql install DESTDIR=$(DESTDIR) + test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR) + +python-install: config.mak + $(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR)/ + test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR) + +distclean: clean + for dir in $(SUBDIRS); do make -C $$dir $@ || exit 1; done + make -C doc $@ + rm -rf source.list dist skytools-* + find python -name '*.pyc' | xargs rm -f + rm -rf dist build + rm -rf autom4te.cache config.log config.status config.mak + +deb80: + ./configure + sed -e s/PGVER/8.0/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +deb81: + ./configure + sed -e s/PGVER/8.1/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +deb82: + ./configure + sed -e s/PGVER/8.2/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages + yada rebuild + debuild -uc -us -b + +tgz: config.mak + $(PYTHON) setup.py sdist -t source.cfg -m source.list + +debclean: distclean + rm -rf debian/tmp-* debian/build* debian/control debian/packages-tmp* + rm -f debian/files debian/rules debian/sub* debian/packages + +boot: configure + +configure: configure.ac + autoconf + + +.PHONY: all clean distclean install deb debclean tgz +.PHONY: python-all python-clean python-install + diff --git a/NEWS b/NEWS new file mode 100644 index 00000000..1e959cce --- /dev/null +++ b/NEWS @@ -0,0 +1,5 @@ + +2007-03-xx ver 2.1 "Radioactive Candy" + + * Final public release. + diff --git a/README b/README new file mode 100644 index 00000000..c9460966 --- /dev/null +++ b/README @@ -0,0 +1,47 @@ + +SkyTools - tools for PostgreSQL +=============================== + +This is a package of tools in use in Skype for replication and +failover. Also it includes a generic queuing mechanism PgQ and +utility library for Python scripts. + +It contains following tools: + +PgQ +--- + +This is the queue machanism we use. Consists of PL/pgsql, PL/python +and C code in database, with Python framework on top of it. It is +based on snapshot based event handling ideas from Slony-I, +written for general usage. + +Features: + + * There can be several queues in database. + * There can be several producers than can insert into any queue. + * There can be several consumers on one queue and all consumers + see all events. + + +Londiste +-------- + +Replication tool written in Python, using PgQ as event transport. + +Features: +- Tables can be added one-by-one into set. +- Initial COPY for one table does not block event replay + for other tables. +- Can compare tables on both sides. + + +walmgr +------ + +This script will setup WAL archiving, does initial backup and +runtime WAL archive and restore. + + + + diff --git a/config.mak.in b/config.mak.in new file mode 100644 index 00000000..233f2cdf --- /dev/null +++ b/config.mak.in @@ -0,0 +1,10 @@ + +prefix = @prefix@ + +override PYTHON = @PYTHON@ +override PG_CONFIG = @PG_CONFIG@ + +PGXS = $(shell $(PG_CONFIG) --pgxs) + +DESTDIR = / + diff --git a/configure.ac b/configure.ac new file mode 100644 index 00000000..4aaa879e --- /dev/null +++ b/configure.ac @@ -0,0 +1,24 @@ +dnl Process this file with autoconf to produce a configure script. + +AC_INIT(skytools, 2.1) +AC_CONFIG_SRCDIR(python/pgqadm.py) + +dnl Find Python interpreter +AC_ARG_WITH(python, [ --with-python=PYTHON name of the Python executable (default: python)], +[ AC_MSG_CHECKING(for python) + PYTHON=$withval + AC_MSG_RESULT($PYTHON)], +[ AC_PATH_PROGS(PYTHON, python) ]) +test -n "$PYTHON" || AC_MSG_ERROR([Cannot continue without Python]) + +dnl Find PostgreSQL pg_config +AC_ARG_WITH(pgconfig, [ --with-pgconfig=PG_CONFIG path to pg_config (default: pg_config)], +[ AC_MSG_CHECKING(for pg_config) + PG_CONFIG=$withval + AC_MSG_RESULT($PG_CONFIG)], +[ AC_PATH_PROGS(PG_CONFIG, pg_config) ]) +test -n "$PG_CONFIG" || AC_MSG_ERROR([Cannot continue without pg_config]) + +dnl Write result +AC_OUTPUT([config.mak]) + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..44774483 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,6 @@ +skytools (2.1) unstable; urgency=low + + * cleanup + + -- Marko Kreen Fri, 02 Feb 2007 12:38:17 +0200 + diff --git a/debian/packages.in b/debian/packages.in new file mode 100644 index 00000000..a9b07847 --- /dev/null +++ b/debian/packages.in @@ -0,0 +1,44 @@ +## debian/packages for skytools + +Source: skytools +Section: contrib/misc +Priority: extra +Maintainer: Marko Kreen +Standards-Version: 3.6.2 +Description: PostgreSQL +Copyright: BSD + Copyright 2006 Marko Kreen +Build: sh + PG_CONFIG=/usr/lib/postgresql/PGVER/bin/pg_config \ + ./configure --prefix=/usr + make DESTDIR=$ROOT +Clean: sh + make distclean || make clean || true +Build-Depends: python-dev, postgresql-server-dev-PGVER + +Package: skytools +Architecture: any +Depends: python-psycopg | pythonPYVER-psycopg, skytools-modules-8.1 | skytools-modules-8.0, [] +Description: Skype database tools - Python parts + . + londiste - replication + pgqadm - generic event queue + walmgr - failover server scripts +Install: sh + make python-install DESTDIR=$ROOT prefix=/usr + +Package: skytools-modules-PGVER +Architecture: any +Depends: postgresql-PGVER, [] +Conflicts: postgresql-extras-PGVER +Description: Extra modules for PostgreSQL + It includes various extra modules for PostgreSQL: + . + txid - 8-byte transaction id's + logtriga - Trigger function to log change in SQL format. + logutriga - Trigger function to log change in urlencoded format. + londiste - Database parts of replication engine. + pgq - Generic queue in database. +Install: sh + make modules-install DESTDIR=$ROOT + diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..d9d94a3d --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,41 @@ + +wiki = https://developer.skype.com/SkypeGarage/DbProjects/SkyTools + +web = mkz@shell.pgfoundry.org:/home/pgfoundry.org/groups/skytools/htdocs/ + +EPYARGS = --no-private -u "http://pgfoundry.org/projects/skytools/" \ + -n "Skytools" + +all: + +upload: + devupload.sh overview.txt $(wiki) + devupload.sh londiste.txt $(wiki)/LondisteUsage + devupload.sh pgq-sql.txt $(wiki)/PgQdocs + devupload.sh pgq-nodupes.txt $(wiki)/PgqNoDupes + devupload.sh walmgr.txt $(wiki)/WalMgr + devupload.sh pgq-admin.txt $(wiki)/PgqAdm + +PY_PKGS = skytools skytools.config skytools.dbstruct skytools.gzlog \ + skytools.quoting skytools.scripting skytools.sqltools \ + pgq pgq.consumer pgq.event pgq.maint pgq.producer pgq.status pgq.ticker \ + londiste londiste.compare londiste.file_read londiste.file_write \ + londiste.installer londiste.playback londiste.repair londiste.setup \ + londiste.syncer londiste.table_copy + +apidoc: + rm -rf api + mkdir -p api + cd ../python && epydoc3 -o ../doc/api --html --no-private $(PY_PKGS) + +apiupload: apidoc + cd ../sql/pgq && rm -rf docs/pgq && make dox && mv docs/html docs/pgq + rsync -rtlz api $(web) + rsync -rtlz ../sql/pgq/docs/pgq $(web) + +clean: + rm -rf api + +distclean: + rm -rf ../sql/pgq/docs/pgq api + diff --git a/doc/TODO.txt b/doc/TODO.txt new file mode 100644 index 00000000..efc8a54e --- /dev/null +++ b/doc/TODO.txt @@ -0,0 +1,44 @@ + +web: + - walmgr + - pgqadm + - todo + + +londiste link +londiste unlink + +Immidiate +========= + +* londiste swithcover support / deny triggers +* deb: /etc/skylog.ini should be conffile +* RemoteConsumer/SerialConsumer/pgq_ext sanity, too much duplication + +Near future +============ + +* londiste: create tables on subscriber +* skytools: switch for silence for cron scripts +* docs: londiste, pgq/python, pgq/sql, skytools +* txid: decide on renaming functions +* logtriga: way to switch off logging for some connection +* pgq: separately installable fkeys for all tables for testing +* logdb: hostname +* pgq_ext: solve event tracking +* contrib/*.sql loading from python - need to check db version +* ideas from SlonyI: + - force timestamps to ISO + - when buffering queries, check their size +* DBScript: failure to write pidfile should be logged (crontscripts) + +Just ideas +=========== + +* logtriga: use pgq.insert_event_directly? +* pgq/sql: rewrite pgq.insert_event in C or logtriga in plpython? +* skytools: config-less operation? +* skytools: config from database? +* skytools: partial sql parser for log processing +* pgqadm ticker logic into db, to make easier other implementations? + diff --git a/doc/londiste.txt b/doc/londiste.txt new file mode 100644 index 00000000..bdb26a52 --- /dev/null +++ b/doc/londiste.txt @@ -0,0 +1,64 @@ + + +== Config file == + +{{{ +[londiste] +job_name = test_to_subcriber + +# source database, where the queue resides +provider_db = dbname=provider port=6000 host=127.0.0.1 + +# destination database +subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 + +# the queue where to listen on +pgq_queue_name = londiste.replika + +# where to log +logfile = ~/log/%(job_name)s.log + +# pidfile is used for avoiding duplicate processes +pidfile = ~/pid/%(job_name)s.pid + +}}} + +== Command line overview == + +{{{ +$ londiste.py --help +usage: londiste.py [options] INI CMD [subcmd args] + +commands: + provider install installs modules, creates queue + provider add TBL ... add table to queue + provider remove TBL ... remove table from queue + provider tables show all tables linked to queue + subscriber install installs schema + subscriber register attaches subscriber to queue (also done by replay) + subscriber unregister detach subscriber from queue + subscriber add TBL ... add table to subscriber + subscriber remove TBL ... remove table from subscriber + subscriber resync TBL ... do full copy again + subscriber tables list tables subscriber has attached to + subscriber missing list tables subscriber has not yet attached to + subscriber check compare table structure on both sides + subscriber fkeys print out fkey drop/create commands + replay replay events to subscriber + copy full copy of table, internal cmd + compare [TBL ...] compare table contents on both sides + repair [TBL ...] repair data on subscriber + +options: + -h, --help show this help message and exit + -q, --quiet make program silent + -v, --verbose make program verbose + -d, --daemon go background + --expect-sync no copy needed (for add command) + --force ignore some warnings + + control running process: + -r, --reload reload config (send SIGHUP) + -s, --stop stop program safely (send SIGINT) + -k, --kill kill program immidiately (send SIGTERM) +}}} diff --git a/doc/overview.txt b/doc/overview.txt new file mode 100644 index 00000000..98235564 --- /dev/null +++ b/doc/overview.txt @@ -0,0 +1,179 @@ +#pragma section-numbers 2 + += SkyTools = + +[[TableOfContents]] + +== Intro == + +This is package of tools we use at Skype to manage our cluster of [http://www.postgresql.org PostgreSQL] +servers. They are put together for our own convinience and also because they build on each other, +so managing them separately is pain. + +The code is hosted at [http://pgfoundry.org PgFoundry] site: + + http://pgfoundry.org/projects/skytools/ + +There are our [http://pgfoundry.org/frs/?group_id=1000206 downloads] and +[http://lists.pgfoundry.org/mailman/listinfo/skytools-users mailing list]. +Also [http://pgfoundry.org/scm/?group_id=1000206 CVS] +and [http://pgfoundry.org/tracker/?group_id=1000206 bugtracker]. + +== High-level tools == + +Those are script that are meant for end-user. +In our case that means database administrators. + +=== Londiste === + +Replication engine written in Python. It uses PgQ as transport mechanism. +Its main goals are robustness and easy usage. Thus its not as complete +and featureful as Slony-I. + +Docs: ./LondisteUsage + +''' Features ''' + + * Tables can be added one-by-one into set. + * Initial COPY for one table does not block event replay for other tables. + * Can compare tables on both sides. + * Easy installation. + +''' Missing features ''' + + * No support for sequences. Thus its not possible to use it for keeping + failover server up-to-date. We use WalMgr for that. + + * Does not understand cascaded replication, when one subscriber acts + as provider to another one and it dies, the last one loses sync with the first one. + In other words - it understands only pair of servers. + +''' Sample usage ''' +{{{ +$ londiste.py replic.ini provider install +$ londiste.py replic.ini subscriber install +$ londiste.py replic.ini replay -d +$ londiste.py replic.ini provider add users orders +$ londiste.py replic.ini subscriber add users +}}} + +=== PgQ === + +Generic queue implementation. Based on ideas from [http://www.slony1.info/ Slony-I] - +snapshot based event batching. + +''' Features ''' + + * Generic multi-consumer, multi-producer queue. + * There can be several consumers on one queue. + * It is guaranteed that each of them sees a event at least once. + But it's not guaranteed that it sees it only once. + * The goal is to provide a clean API as SQL functions. The frameworks + on top of that don't need to understand internal details. + +''' Technical design ''' + + * Events are batched using snapshots (like Slony-I). + * Consumers are poll-only, they don't need to do any administrative work. + * Queue administration is separate process from consumers. + * Tolerant of long transactions. + * Easy to monitor. + +''' Docs ''' + + * SQL API overview: ./PgQdocs + * SQL API detailed docs: http://skytools.projects.postgresql.org/pgq/ + * Administrative tool usage: ./PgqAdm + +=== WalMgr === + +Python script for hot failover. Tries to make setup +initial copy and later switch easy for admins. + + * Docs: ./WalMgr + +Sample: + +{{{ + [ .. prepare config .. ] + + master$ walmgr setup + master$ walmgr backup + slave$ walmgr restore + + [ .. main server down, switch failover server to normal mode: ] + + slave$ walmgr restore +}}} + +== Low-level tools == + +Those are building blocks for the PgQ and Londiste. +Useful for database developers. + +=== txid === + + Provides 8-byte transaction id-s for external usage. + +=== logtriga === + + Trigger function for table event logging in "partial SQL" format. + Based on Slony-I logtrigger. Used in londiste for replication. + +=== logutriga === + + Trigger function for table event logging in urlencoded format. + Written in PL/Python. For cases where data manipulation is necessary. + +== Developement frameworks == + +=== skytools - Python framework for database scripting === + +This collect various utilities for Python scripts for databases. + +''' Topics ''' + + * Daemonization + * Logging + * Configuration. + * Skeleton class for scripts. + * Quoting (SQL/COPY) + * COPY helpers. + * Database object lookup. + * Table structure detection. + +Documentation: http://skytools.projects.postgresql.org/api/ + +=== pgq - Python framework for PgQ consumers === + +This builds on scripting framework above. + +Documentation: http://skytools.projects.postgresql.org/api/ + +== Sample scripts == + +Those are specialized script that are based on skytools/pgq framework. +Can be considered examples, although they are used in production in Skype. + +=== Special data moving scripts === + +There are couple of scripts for situations where regular replication +does not fit. They all operate on `logutriga()` urlencoded queues. + + * `cube_dispatcher`: Multi-table partitioning on change date, with optional keep-all-row-versions mode. + * `table_dispatcher`: configurable partitioning for one table. + * `bulk_loader`: aggregates changes for slow databases. Instead of each change in separate statement, + does minimal amount of DELETE-s and then big COPY. + +|| Script || Supported operations || Number of tables || Partitioning || +|| table_dispatcher || INSERT || 1 || any || +|| cube_dispatcher || INSERT/UPDATE || any || change time || +|| bulk_loader || INSERT/UPDATE/DELETE || any || none || + +=== queue_mover === + +Simply copies all events from one queue to another. + +=== scriptmgr === + +Allows to start and stop several scripts together. diff --git a/doc/pgq-admin.txt b/doc/pgq-admin.txt new file mode 100644 index 00000000..1ea9fddb --- /dev/null +++ b/doc/pgq-admin.txt @@ -0,0 +1,42 @@ += PgqAdm = + +== Config == + +{{{ +[pgqadm] +job_name = pgqadm_somedb + +db = dbname=somedb + +# how often to run maintenance [minutes] +maint_delay_min = 5 + +# how often to check for activity [secs] +loop_delay = 0.1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +}}} + +== Command line usage == + +{{{ +$ pgqadm.py --help +usage: pgqadm.py [options] INI CMD [subcmd args] + +commands: + ticker start ticking & maintenance process + status show overview of queue healt + +options: + -h, --help show this help message and exit + -q, --quiet make program silent + -v, --verbose make program verbose + -d, --daemon go background + + control running process: + -r, --reload reload config (send SIGHUP) + -s, --stop stop program safely (send SIGINT) + -k, --kill kill program immidiately (send SIGTERM) +}}} diff --git a/doc/pgq-nodupes.txt b/doc/pgq-nodupes.txt new file mode 100644 index 00000000..933cab56 --- /dev/null +++ b/doc/pgq-nodupes.txt @@ -0,0 +1,33 @@ += Avoiding duplicate events = + +It is pretty burdensome to check if event is already processed, +especially on bulk data moving. Here's a way how this can be avoided. + +First, consumer must guarantee that it processes all events in one tx. + +Consumer itself can tag events for retry, but then it must be able to handle them later. + + * If the PgQ queue and event data handling happen in same database, + the consumer must simply call pgq.finish_batch() inside the event-processing + transaction. + + * If the event processing happens in different database, the consumer + must store the batch_id into destination database, inside the same + transaction as the event processing happens. + + Only after committing it, consumer can call pgq.finish_batch() in queue database + and commit that. + + As the batches come in sequence, there's no need to remember full log of batch_id's, + it's enough to keep the latest batch_id. + + Then at the start of every batch, consumer can check if the batch_id already + exists in destination database, and if it does, then just tag batch done, + without processing. + +With this, there's no need for consumer to check for already processed +events. + +NB: This assumes the event processing is transaction-able - failures +will be rollbacked. If event processing includes communication with +world outside database, eg. sending email, such handling won't work. diff --git a/doc/pgq-sql.txt b/doc/pgq-sql.txt new file mode 100644 index 00000000..9414594e --- /dev/null +++ b/doc/pgq-sql.txt @@ -0,0 +1,191 @@ += PgQ - queue for PostgreSQL = + +== Queue creation == + +{{{ + pgq.create_queue(queue_name text) +}}} + +Initialize event queue. + +Returns 0 if event queue already exists, 1 otherwise. + +== Producer == + +{{{ + pgq.insert_event(queue_name text, ev_type, ev_data) + pgq.insert_event(queue_name text, ev_type, ev_data, extra1, extra2, extra3, extra4) +}}} + +Generate new event. This should be called inside main tx - thus +rollbacked with it if needed. + + +== Consumer == + +{{{ + pgq.register_consumer(queue_name text, consumer_id text) +}}} + +Attaches this consumer to particular event queue. + +Returns 0 if the consumer was already attached, 1 otherwise. + +{{{ + pgq.unregister_consumer(queue_name text, consumer_id text) +}}} + +Unregister and drop resources allocated to customer. + + +{{{ + pgq.next_batch(queue_name text, consumer_id text) +}}} + +Allocates next batch of events to consumer. + +Returns batch id (int8), to be used in processing functions. If no batches +are available, returns NULL. That means that the ticker has not cut them yet. +This is the appropriate moment for consumer to sleep. + +{{{ + pgq.get_batch_events(batch_id int8) +}}} + +`pgq.get_batch_events()` returns a set of events in this batch. + +There may be no events in the batch. This is normal. The batch must still be closed +with pgq.finish_batch(). + +Event fields: (ev_id int8, ev_time timestamptz, ev_txid int8, ev_retry int4, ev_type text, + ev_data text, ev_extra1, ev_extra2, ev_extra3, ev_extra4) + +{{{ + pgq.event_failed(batch_id int8, event_id int8, reason text) +}}} + +Tag event as 'failed' - it will be stored, but not further processing is done. + +{{{ + pgq.event_retry(batch_id int8, event_id int8, retry_seconds int4) +}}} + +Tag event for 'retry' - after x seconds the event will be re-inserted +into main queue. + +{{{ + pgq.finish_batch(batch_id int8) +}}} + +Tag batch as finished. Until this is not done, the consumer will get +same batch again. + +After calling finish_batch consumer cannot do any operations with events +of that batch. All operations must be done before. + +== Failed queue operation == + +Events tagged as failed just stay on their queue. Following +functions can be used to manage them. + +{{{ + pgq.failed_event_list(queue_name, consumer) + pgq.failed_event_list(queue_name, consumer, cnt, offset) + pgq.failed_event_count(queue_name, consumer) +}}} + +Get info about the queue. + +Event fields are same as for pgq.get_batch_events() + +{{{ + pgq.failed_event_delete(queue_name, consumer, event_id) + pgq.failed_event_retry(queue_name, consumer, event_id) +}}} + +Remove an event from queue, or retry it. + +== Info operations == + +{{{ + pgq.get_queue_info() +}}} + +Get list of queues. + +Result: () + +{{{ + pgq.get_consumer_info() + pgq.get_consumer_info(queue_name) + pgq.get_consumer_info(queue_name, consumer) +}}} + +Get list of active consumers. + +Result: () + +{{{ + pgq.get_batch_info(batch_id) +}}} + +Get info about batch. + +Result fields: () + +== Notes == + +Consumer '''must''' be able to process same event several times. + +== Example == + +First, create event queue: + +{{{ + select pgq.create_queue('LogEvent'); +}}} + +Then, producer side can do whenever it wishes: + +{{{ + select pgq.insert_event('LogEvent', 'data', 'DataFor123'); +}}} + +First step for consumer is to register: + +{{{ + select pgq.register_consumer('LogEvent', 'TestConsumer'); +}}} + +Then it can enter into consuming loop: + +{{{ + begin; + select pgq.next_batch('LogEvent', 'TestConsumer'); [into batch_id] + commit; +}}} + +That will reserve a batch of events for this consumer. + +To see the events in batch: + +{{{ + select * from pgq.get_batch_events(batch_id); +}}} + +That will give all events in batch. The processing does not need to be happen +all in one transaction, framework can split the work how it wants. + +If a events failed or needs to be tried again, framework can call: + +{{{ + select pgq.event_retry(batch_id, event_id, 60); + select pgq.event_failed(batch_id, event_id, 'Record deleted'); +}}} + +When all done, notify core about it: + +{{{ + select pgq.finish_batch(batch_id) +}}} + diff --git a/doc/walmgr.txt b/doc/walmgr.txt new file mode 100644 index 00000000..a05ea644 --- /dev/null +++ b/doc/walmgr.txt @@ -0,0 +1,82 @@ +#pragma section-numbers 2 + += WalMgr = + +[[TableOfContents]] + +== Step-by-step instructions == + +=== no-password ssh access from one to other === + + master$ test -f ~/.ssh/id_dsa.pub || ssh-keygen -t dsa + master$ scp .ssh/id_dsa.pub slave: + slave$ cat id_dsa.pub >> ~/.ssh/authorized_keys + +=== Configure paths === + + master$ edit master.ini + slave$ edit slave.ini + slave$ mkdir data.master logs.full logs.partial + +=== Start archival process === + + master$ ./walmgr.py setup + +=== Do full backup+restore === + + master$ ./walmgr.py backup + slave$ ./walmgr.py restore + + 'walmgr.py restore' moves data in place and starts postmaster, + that starts replaying logs as they appear. + +=== In-progress WAL segments can be backup by command: === + + master$ ./walmgr.py sync + +=== If need to stop replay on slave and boot into normal mode, do: === + + slave$ ./walmgr.py boot + +== Configuration == + +=== master.ini === + +{{{ +[wal-master] +logfile = master.log +use_skylog = 0 + +master_db = dbname=template1 +master_data = /var/lib/postgresql/8.0/main +master_config = /etc/postgresql/8.0/main/postgresql.conf + +slave = slave:/var/lib/postgresql/walshipping + +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +# syncdaemon update frequency +loop_delay = 10.0 + +}}} + +=== slave.ini === + +{{{ +[wal-slave] +logfile = slave.log +use_skylog = 0 + +slave_data = /var/lib/postgresql/8.0/main +slave_stop_cmd = /etc/init.d/postgresql-8.0 stop +slave_start_cmd = /etc/init.d/postgresql-8.0 start + +slave = /var/lib/postgresql/walshipping +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +keep_old_logs = 0 +}}} diff --git a/python/conf/londiste.ini b/python/conf/londiste.ini new file mode 100644 index 00000000..a1506a32 --- /dev/null +++ b/python/conf/londiste.ini @@ -0,0 +1,16 @@ + +[londiste] +job_name = test_to_subcriber + +provider_db = dbname=provider port=6000 host=127.0.0.1 +subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste.replika + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# both events and ticks will be copied there +#mirror_queue = replika_mirror + diff --git a/python/conf/pgqadm.ini b/python/conf/pgqadm.ini new file mode 100644 index 00000000..a2e92f6b --- /dev/null +++ b/python/conf/pgqadm.ini @@ -0,0 +1,18 @@ + +[pgqadm] + +job_name = pgqadm_somedb + +db = dbname=provider port=6000 host=127.0.0.1 + +# how often to run maintenance [minutes] +maint_delay_min = 5 + +# how often to check for activity [secs] +loop_delay = 0.1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/python/conf/skylog.ini b/python/conf/skylog.ini new file mode 100644 index 00000000..150ef934 --- /dev/null +++ b/python/conf/skylog.ini @@ -0,0 +1,76 @@ +; notes: +; - 'args' is mandatory in [handler_*] sections +; - in lists there must not be spaces + +; +; top-level config +; + +; list of all loggers +[loggers] +keys=root +; root logger sees everything. there can be per-job configs by +; specifing loggers with job_name of the script + +; list of all handlers +[handlers] +;; seems logger module immidiately initalized all handlers, +;; whether they are actually used or not. so better +;; keep this list in sync with actual handler list +;keys=stderr,logdb,logsrv,logfile +keys=stderr + +; list of all formatters +[formatters] +keys=short,long,none + +; +; map specific loggers to specifig handlers +; +[logger_root] +level=DEBUG +;handlers=stderr,logdb,logsrv,logfile +handlers=stderr + +; +; configure formatters +; +[formatter_short] +format=%(asctime)s %(levelname)s %(message)s +datefmt=%H:%M + +[formatter_long] +format=%(asctime)s %(process)s %(levelname)s %(message)s + +[formatter_none] +format=%(message)s + +; +; configure handlers +; + +; file. args: stream +[handler_stderr] +class=StreamHandler +args=(sys.stderr,) +formatter=short + +; log into db. args: conn_string +[handler_logdb] +class=skylog.LogDBHandler +args=("host=127.0.0.1 port=5432 user=logger dbname=logdb",) +formatter=none +level=INFO + +; JSON messages over UDP. args: host, port +[handler_logsrv] +class=skylog.UdpLogServerHandler +args=('127.0.0.1', 6666) +formatter=none + +; rotating logfile. args: filename, maxsize, maxcount +[handler_logfile] +class=skylog.EasyRotatingFileHandler +args=('~/log/%(job_name)s.log', 100*1024*1024, 3) +formatter=long + diff --git a/python/conf/wal-master.ini b/python/conf/wal-master.ini new file mode 100644 index 00000000..5ae8cb2b --- /dev/null +++ b/python/conf/wal-master.ini @@ -0,0 +1,18 @@ +[wal-master] +logfile = master.log +use_skylog = 0 + +master_db = dbname=template1 +master_data = /var/lib/postgresql/8.0/main +master_config = /etc/postgresql/8.0/main/postgresql.conf + + +slave = slave:/var/lib/postgresql/walshipping + +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +# syncdaemon update frequency +loop_delay = 10.0 + diff --git a/python/conf/wal-slave.ini b/python/conf/wal-slave.ini new file mode 100644 index 00000000..912bf756 --- /dev/null +++ b/python/conf/wal-slave.ini @@ -0,0 +1,15 @@ +[wal-slave] +logfile = slave.log +use_skylog = 0 + +slave_data = /var/lib/postgresql/8.0/main +slave_stop_cmd = /etc/init.d/postgresql-8.0 stop +slave_start_cmd = /etc/init.d/postgresql-8.0 start + +slave = /var/lib/postgresql/walshipping +completed_wals = %(slave)s/logs.complete +partial_wals = %(slave)s/logs.partial +full_backup = %(slave)s/data.master + +keep_old_logs = 0 + diff --git a/python/londiste.py b/python/londiste.py new file mode 100755 index 00000000..9e5684ea --- /dev/null +++ b/python/londiste.py @@ -0,0 +1,130 @@ +#! /usr/bin/env python + +"""Londiste launcher. +""" + +import sys, os, optparse, skytools + +# python 2.3 will try londiste.py first... +import sys, os.path +if os.path.exists(os.path.join(sys.path[0], 'londiste.py')) \ + and not os.path.exists(os.path.join(sys.path[0], 'londiste')): + del sys.path[0] + +from londiste import * + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + provider install installs modules, creates queue + provider add TBL ... add table to queue + provider remove TBL ... remove table from queue + provider tables show all tables linked to queue + provider seqs show all sequences on provider + + subscriber install installs schema + subscriber add TBL ... add table to subscriber + subscriber remove TBL ... remove table from subscriber + subscriber tables list tables subscriber has attached to + subscriber seqs list sequences subscriber is interested + subscriber missing list tables subscriber has not yet attached to + subscriber link QUE create mirror queue + subscriber unlink dont mirror queue + + replay replay events to subscriber + + switchover switch the roles between provider & subscriber + compare [TBL ...] compare table contents on both sides + repair [TBL ...] repair data on subscriber + copy full copy of table, internal cmd + subscriber check compare table structure on both sides + subscriber fkeys print out fkey drop/create commands + subscriber resync TBL ... do full copy again + subscriber register attaches subscriber to queue (also done by replay) + subscriber unregister detach subscriber from queue +""" + +"""switchover: +goal is to launch a replay with reverse config + +- should link be required? (link should guarantee all tables/seqs same?) +- should link auto-add tables for subscriber + +1. lock all tables on provider, in order specified by 'nr' +2. wait until old replay is past the point +3. sync seq +4. replace queue triggers on provider with deny triggers +5. replace deny triggers on subscriber with queue triggers +6. sync pgq tick seqs? change pgq config? + +""" + +class Londiste(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + + if self.options.rewind or self.options.reset: + self.script = Replicator(args) + return + + if len(self.args) < 2: + print "need command" + sys.exit(1) + cmd = self.args[1] + + if cmd =="provider": + script = ProviderSetup(args) + elif cmd == "subscriber": + script = SubscriberSetup(args) + elif cmd == "replay": + method = self.cf.get('method', 'direct') + if method == 'direct': + script = Replicator(args) + elif method == 'file_write': + script = FileWrite(args) + elif method == 'file_write': + script = FileWrite(args) + else: + print "unknown method, quitting" + sys.exit(1) + elif cmd == "copy": + script = CopyTable(args) + elif cmd == "compare": + script = Comparator(args) + elif cmd == "repair": + script = Repairer(args) + elif cmd == "upgrade": + script = UpgradeV2(args) + else: + print "Unknown command '%s', use --help for help" % cmd + sys.exit(1) + + self.script = script + + def start(self): + self.script.start() + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.set_usage(command_usage.strip()) + + g = optparse.OptionGroup(p, "expert options") + g.add_option("--force", action="store_true", + help = "add: ignore table differences, repair: ignore lag") + g.add_option("--expect-sync", action="store_true", dest="expect_sync", + help = "add: no copy needed", default=False) + g.add_option("--skip-truncate", action="store_true", dest="skip_truncate", + help = "copy: keep old data", default=False) + g.add_option("--rewind", action="store_true", + help = "replay: sync queue pos with subscriber") + g.add_option("--reset", action="store_true", + help = "replay: forget queue pos on subscriber") + p.add_option_group(g) + + return p + +if __name__ == '__main__': + script = Londiste(sys.argv[1:]) + script.start() + diff --git a/python/londiste/__init__.py b/python/londiste/__init__.py new file mode 100644 index 00000000..97d67433 --- /dev/null +++ b/python/londiste/__init__.py @@ -0,0 +1,12 @@ + +"""Replication on top of PgQ.""" + +from playback import * +from compare import * +from file_read import * +from file_write import * +from setup import * +from table_copy import * +from installer import * +from repair import * + diff --git a/python/londiste/compare.py b/python/londiste/compare.py new file mode 100644 index 00000000..0029665b --- /dev/null +++ b/python/londiste/compare.py @@ -0,0 +1,45 @@ +#! /usr/bin/env python + +"""Compares tables in replication set. + +Currently just does count(1) on both sides. +""" + +import sys, os, time, skytools + +__all__ = ['Comparator'] + +from syncer import Syncer + +class Comparator(Syncer): + def process_sync(self, tbl, src_db, dst_db): + """Actual comparision.""" + + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info('Counting %s' % tbl) + + q = "select count(1) from only _TABLE_" + q = self.cf.get('compare_sql', q) + q = q.replace('_TABLE_', tbl) + + self.log.debug("srcdb: " + q) + src_curs.execute(q) + src_row = src_curs.fetchone() + src_str = ", ".join(map(str, src_row)) + self.log.info("srcdb: res = %s" % src_str) + + self.log.debug("dstdb: " + q) + dst_curs.execute(q) + dst_row = dst_curs.fetchone() + dst_str = ", ".join(map(str, dst_row)) + self.log.info("dstdb: res = %s" % dst_str) + + if src_str != dst_str: + self.log.warning("%s: Results do not match!" % tbl) + +if __name__ == '__main__': + script = Comparator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/file_read.py b/python/londiste/file_read.py new file mode 100644 index 00000000..2902bda5 --- /dev/null +++ b/python/londiste/file_read.py @@ -0,0 +1,52 @@ + +"""Reads events from file instead of db queue.""" + +import sys, os, re, skytools + +from playback import * +from table_copy import * + +__all__ = ['FileRead'] + +file_regex = r"^tick_0*([0-9]+)\.sql$" +file_rc = re.compile(file_regex) + + +class FileRead(CopyTable): + """Reads events from file instead of db queue. + + Incomplete implementation. + """ + + def __init__(self, args, log = None): + CopyTable.__init__(self, args, log, copy_thread = 0) + + def launch_copy(self, tbl): + # copy immidiately + self.do_copy(t) + + def work(self): + last_batch = self.get_last_batch(curs) + list = self.get_file_list() + + def get_list(self): + """Return list of (first_batch, full_filename) pairs.""" + + src_dir = self.cf.get('file_src') + list = os.listdir(src_dir) + list.sort() + res = [] + for fn in list: + m = file_rc.match(fn) + if not m: + self.log.debug("Ignoring file: %s" % fn) + continue + full = os.path.join(src_dir, fn) + batch_id = int(m.group(1)) + res.append((batch_id, full)) + return res + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/file_write.py b/python/londiste/file_write.py new file mode 100644 index 00000000..86e16aae --- /dev/null +++ b/python/londiste/file_write.py @@ -0,0 +1,67 @@ + +"""Writes events into file.""" + +import sys, os, skytools +from cStringIO import StringIO +from playback import * + +__all__ = ['FileWrite'] + +class FileWrite(Replicator): + """Writes events into file. + + Incomplete implementation. + """ + + last_successful_batch = None + + def load_state(self, batch_id): + # maybe check if batch exists on filesystem? + self.cur_tick = self.cur_batch_info['tick_id'] + self.prev_tick = self.cur_batch_info['prev_tick_id'] + return 1 + + def process_batch(self, db, batch_id, ev_list): + pass + + def save_state(self, do_commit): + # nothing to save + pass + + def sync_tables(self, dst_db): + # nothing to sync + return 1 + + def interesting(self, ev): + # wants all of them + return 1 + + def handle_data_event(self, ev): + fmt = self.sql_command[ev.type] + sql = fmt % (ev.ev_extra1, ev.data) + row = "%s -- txid:%d" % (sql, ev.txid) + self.sql_list.append(row) + ev.tag_done() + + def handle_system_event(self, ev): + row = "-- sysevent:%s txid:%d data:%s" % ( + ev.type, ev.txid, ev.data) + self.sql_list.append(row) + ev.tag_done() + + def flush_sql(self): + self.sql_list.insert(0, "-- tick:%d prev:%s" % ( + self.cur_tick, self.prev_tick)) + self.sql_list.append("-- end_tick:%d\n" % self.cur_tick) + # store result + dir = self.cf.get("file_dst") + fn = os.path.join(dir, "tick_%010d.sql" % self.cur_tick) + f = open(fn, "w") + buf = "\n".join(self.sql_list) + f.write(buf) + f.close() + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/installer.py b/python/londiste/installer.py new file mode 100644 index 00000000..6f190ab2 --- /dev/null +++ b/python/londiste/installer.py @@ -0,0 +1,26 @@ + +"""Functions to install londiste and its depentencies into database.""" + +import os, skytools + +__all__ = ['install_provider', 'install_subscriber'] + +provider_object_list = [ + skytools.DBFunction('logtriga', 0, sql_file = "logtriga.sql"), + skytools.DBFunction('get_current_snapshot', 0, sql_file = "txid.sql"), + skytools.DBSchema('pgq', sql_file = "pgq.sql"), + skytools.DBSchema('londiste', sql_file = "londiste.sql") +] + +subscriber_object_list = [ + skytools.DBSchema('londiste', sql_file = "londiste.sql") +] + +def install_provider(curs, log): + """Installs needed code into provider db.""" + skytools.db_install(curs, provider_object_list, log) + +def install_subscriber(curs, log): + """Installs needed code into subscriber db.""" + skytools.db_install(curs, subscriber_object_list, log) + diff --git a/python/londiste/playback.py b/python/londiste/playback.py new file mode 100644 index 00000000..2bcb1bc7 --- /dev/null +++ b/python/londiste/playback.py @@ -0,0 +1,558 @@ +#! /usr/bin/env python + +"""Basic replication core.""" + +import sys, os, time +import skytools, pgq + +__all__ = ['Replicator', 'TableState', + 'TABLE_MISSING', 'TABLE_IN_COPY', 'TABLE_CATCHING_UP', + 'TABLE_WANNA_SYNC', 'TABLE_DO_SYNC', 'TABLE_OK'] + +# state # owner - who is allowed to change +TABLE_MISSING = 0 # main +TABLE_IN_COPY = 1 # copy +TABLE_CATCHING_UP = 2 # copy +TABLE_WANNA_SYNC = 3 # main +TABLE_DO_SYNC = 4 # copy +TABLE_OK = 5 # setup + +SYNC_OK = 0 # continue with batch +SYNC_LOOP = 1 # sleep, try again +SYNC_EXIT = 2 # nothing to do, exit skript + +class Counter(object): + """Counts table statuses.""" + + missing = 0 + copy = 0 + catching_up = 0 + wanna_sync = 0 + do_sync = 0 + ok = 0 + + def __init__(self, tables): + """Counts and sanity checks.""" + for t in tables: + if t.state == TABLE_MISSING: + self.missing += 1 + elif t.state == TABLE_IN_COPY: + self.copy += 1 + elif t.state == TABLE_CATCHING_UP: + self.catching_up += 1 + elif t.state == TABLE_WANNA_SYNC: + self.wanna_sync += 1 + elif t.state == TABLE_DO_SYNC: + self.do_sync += 1 + elif t.state == TABLE_OK: + self.ok += 1 + # only one table is allowed to have in-progress copy + if self.copy + self.catching_up + self.wanna_sync + self.do_sync > 1: + raise Exception('Bad table state') + +class TableState(object): + """Keeps state about one table.""" + def __init__(self, name, log): + self.name = name + self.log = log + self.forget() + self.changed = 0 + + def forget(self): + self.state = TABLE_MISSING + self.str_snapshot = None + self.from_snapshot = None + self.sync_tick_id = None + self.ok_batch_count = 0 + self.last_tick = 0 + self.changed = 1 + + def change_snapshot(self, str_snapshot, tag_changed = 1): + if self.str_snapshot == str_snapshot: + return + self.log.debug("%s: change_snapshot to %s" % (self.name, str_snapshot)) + self.str_snapshot = str_snapshot + if str_snapshot: + self.from_snapshot = skytools.Snapshot(str_snapshot) + else: + self.from_snapshot = None + + if tag_changed: + self.ok_batch_count = 0 + self.last_tick = None + self.changed = 1 + + def change_state(self, state, tick_id = None): + if self.state == state and self.sync_tick_id == tick_id: + return + self.state = state + self.sync_tick_id = tick_id + self.changed = 1 + self.log.debug("%s: change_state to %s" % (self.name, + self.render_state())) + + def render_state(self): + """Make a string to be stored in db.""" + + if self.state == TABLE_MISSING: + return None + elif self.state == TABLE_IN_COPY: + return 'in-copy' + elif self.state == TABLE_CATCHING_UP: + return 'catching-up' + elif self.state == TABLE_WANNA_SYNC: + return 'wanna-sync:%d' % self.sync_tick_id + elif self.state == TABLE_DO_SYNC: + return 'do-sync:%d' % self.sync_tick_id + elif self.state == TABLE_OK: + return 'ok' + + def parse_state(self, merge_state): + """Read state from string.""" + + state = -1 + if merge_state == None: + state = TABLE_MISSING + elif merge_state == "in-copy": + state = TABLE_IN_COPY + elif merge_state == "catching-up": + state = TABLE_CATCHING_UP + elif merge_state == "ok": + state = TABLE_OK + elif merge_state == "?": + state = TABLE_OK + else: + tmp = merge_state.split(':') + if len(tmp) == 2: + self.sync_tick_id = int(tmp[1]) + if tmp[0] == 'wanna-sync': + state = TABLE_WANNA_SYNC + elif tmp[0] == 'do-sync': + state = TABLE_DO_SYNC + + if state < 0: + raise Exception("Bad table state: %s" % merge_state) + + return state + + def loaded_state(self, merge_state, str_snapshot): + self.log.debug("loaded_state: %s: %s / %s" % ( + self.name, merge_state, str_snapshot)) + self.change_snapshot(str_snapshot, 0) + self.state = self.parse_state(merge_state) + self.changed = 0 + if merge_state == "?": + self.changed = 1 + + def interesting(self, ev, tick_id, copy_thread): + """Check if table wants this event.""" + + if copy_thread: + if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC): + return False + else: + if self.state != TABLE_OK: + return False + + # if no snapshot tracking, then accept always + if not self.from_snapshot: + return True + + # uninteresting? + if self.from_snapshot.contains(ev.txid): + return False + + # after couple interesting batches there no need to check snapshot + # as there can be only one partially interesting batch + if tick_id != self.last_tick: + self.last_tick = tick_id + self.ok_batch_count += 1 + + # disable batch tracking + if self.ok_batch_count > 3: + self.change_snapshot(None) + return True + +class SeqCache(object): + def __init__(self): + self.seq_list = [] + self.val_cache = {} + + def set_seq_list(self, seq_list): + self.seq_list = seq_list + new_cache = {} + for seq in seq_list: + val = self.val_cache.get(seq) + if val: + new_cache[seq] = val + self.val_cache = new_cache + + def resync(self, src_curs, dst_curs): + if len(self.seq_list) == 0: + return + dat = ".last_value, ".join(self.seq_list) + dat += ".last_value" + q = "select %s from %s" % (dat, ",".join(self.seq_list)) + src_curs.execute(q) + row = src_curs.fetchone() + for i in range(len(self.seq_list)): + seq = self.seq_list[i] + cur = row[i] + old = self.val_cache.get(seq) + if old != cur: + q = "select setval(%s, %s)" + dst_curs.execute(q, [seq, cur]) + self.val_cache[seq] = cur + +class Replicator(pgq.SerialConsumer): + """Replication core.""" + + sql_command = { + 'I': "insert into %s %s;", + 'U': "update only %s set %s;", + 'D': "delete from only %s where %s;", + } + + # batch info + cur_tick = 0 + prev_tick = 0 + + def __init__(self, args): + pgq.SerialConsumer.__init__(self, 'londiste', 'provider_db', 'subscriber_db', args) + + # tick table in dst for SerialConsumer(). keep londiste stuff under one schema + self.dst_completed_table = "londiste.completed" + + self.table_list = [] + self.table_map = {} + + self.copy_thread = 0 + self.maint_time = 0 + self.seq_cache = SeqCache() + self.maint_delay = self.cf.getint('maint_delay', 600) + self.mirror_queue = self.cf.get('mirror_queue', '') + + def process_remote_batch(self, src_db, batch_id, ev_list, dst_db): + "All work for a batch. Entry point from SerialConsumer." + + # this part can play freely with transactions + + dst_curs = dst_db.cursor() + + self.cur_tick = self.cur_batch_info['tick_id'] + self.prev_tick = self.cur_batch_info['prev_tick_id'] + + self.load_table_state(dst_curs) + self.sync_tables(dst_db) + + # now the actual event processing happens. + # they must be done all in one tx in dst side + # and the transaction must be kept open so that + # the SerialConsumer can save last tick and commit. + + self.handle_seqs(dst_curs) + self.handle_events(dst_curs, ev_list) + self.save_table_state(dst_curs) + + def handle_seqs(self, dst_curs): + if self.copy_thread: + return + + q = "select * from londiste.subscriber_get_seq_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + seq_list = [] + for row in dst_curs.fetchall(): + seq_list.append(row[0]) + + self.seq_cache.set_seq_list(seq_list) + + src_curs = self.get_database('provider_db').cursor() + self.seq_cache.resync(src_curs, dst_curs) + + def sync_tables(self, dst_db): + """Table sync loop. + + Calls appropriate handles, which is expected to + return one of SYNC_* constants.""" + + self.log.debug('Sync tables') + while 1: + cnt = Counter(self.table_list) + if self.copy_thread: + res = self.sync_from_copy_thread(cnt, dst_db) + else: + res = self.sync_from_main_thread(cnt, dst_db) + + if res == SYNC_EXIT: + self.log.debug('Sync tables: exit') + self.detach() + sys.exit(0) + elif res == SYNC_OK: + return + elif res != SYNC_LOOP: + raise Exception('Program error') + + self.log.debug('Sync tables: sleeping') + time.sleep(3) + dst_db.commit() + self.load_table_state(dst_db.cursor()) + dst_db.commit() + + def sync_from_main_thread(self, cnt, dst_db): + "Main thread sync logic." + + # + # decide what to do - order is imortant + # + if cnt.do_sync: + # wait for copy thread to catch up + return SYNC_LOOP + elif cnt.wanna_sync: + # copy thread wants sync, if not behind, do it + t = self.get_table_by_state(TABLE_WANNA_SYNC) + if self.cur_tick >= t.sync_tick_id: + self.change_table_state(dst_db, t, TABLE_DO_SYNC, self.cur_tick) + return SYNC_LOOP + else: + return SYNC_OK + elif cnt.catching_up: + # active copy, dont worry + return SYNC_OK + elif cnt.copy: + # active copy, dont worry + return SYNC_OK + elif cnt.missing: + # seems there is no active copy thread, launch new + t = self.get_table_by_state(TABLE_MISSING) + self.change_table_state(dst_db, t, TABLE_IN_COPY) + + # the copy _may_ happen immidiately + self.launch_copy(t) + + # there cannot be interesting events in current batch + # but maybe there's several tables, lets do them in one go + return SYNC_LOOP + else: + # seems everything is in sync + return SYNC_OK + + def sync_from_copy_thread(self, cnt, dst_db): + "Copy thread sync logic." + + # + # decide what to do - order is imortant + # + if cnt.do_sync: + # main thread is waiting, catch up, then handle over + t = self.get_table_by_state(TABLE_DO_SYNC) + if self.cur_tick == t.sync_tick_id: + self.change_table_state(dst_db, t, TABLE_OK) + return SYNC_EXIT + elif self.cur_tick < t.sync_tick_id: + return SYNC_OK + else: + self.log.error("copy_sync: cur_tick=%d sync_tick=%d" % ( + self.cur_tick, t.sync_tick_id)) + raise Exception('Invalid table state') + elif cnt.wanna_sync: + # wait for main thread to react + return SYNC_LOOP + elif cnt.catching_up: + # is there more work? + if self.work_state: + return SYNC_OK + + # seems we have catched up + t = self.get_table_by_state(TABLE_CATCHING_UP) + self.change_table_state(dst_db, t, TABLE_WANNA_SYNC, self.cur_tick) + return SYNC_LOOP + elif cnt.copy: + # table is not copied yet, do it + t = self.get_table_by_state(TABLE_IN_COPY) + self.do_copy(t) + + # forget previous value + self.work_state = 1 + + return SYNC_LOOP + else: + # nothing to do + return SYNC_EXIT + + def handle_events(self, dst_curs, ev_list): + "Actual event processing happens here." + + ignored_events = 0 + self.sql_list = [] + mirror_list = [] + for ev in ev_list: + if not self.interesting(ev): + ignored_events += 1 + ev.tag_done() + continue + + if ev.type in ('I', 'U', 'D'): + self.handle_data_event(ev, dst_curs) + else: + self.handle_system_event(ev, dst_curs) + + if self.mirror_queue: + mirror_list.append(ev) + + # finalize table changes + self.flush_sql(dst_curs) + self.stat_add('ignored', ignored_events) + + # put events into mirror queue if requested + if self.mirror_queue: + self.fill_mirror_queue(mirror_list, dst_curs) + + def handle_data_event(self, ev, dst_curs): + fmt = self.sql_command[ev.type] + sql = fmt % (ev.extra1, ev.data) + self.sql_list.append(sql) + if len(self.sql_list) > 200: + self.flush_sql(dst_curs) + ev.tag_done() + + def flush_sql(self, dst_curs): + if len(self.sql_list) == 0: + return + + buf = "\n".join(self.sql_list) + self.sql_list = [] + + dst_curs.execute(buf) + + def interesting(self, ev): + if ev.type not in ('I', 'U', 'D'): + return 1 + t = self.get_table_by_name(ev.extra1) + if t: + return t.interesting(ev, self.cur_tick, self.copy_thread) + else: + return 0 + + def handle_system_event(self, ev, dst_curs): + "System event." + + if ev.type == "T": + self.log.info("got new table event: "+ev.data) + # check tables to be dropped + name_list = [] + for name in ev.data.split(','): + name_list.append(name.strip()) + + del_list = [] + for tbl in self.table_list: + if tbl.name in name_list: + continue + del_list.append(tbl) + + # separate loop to avoid changing while iterating + for tbl in del_list: + self.log.info("Removing table %s from set" % tbl.name) + self.remove_table(tbl, dst_curs) + + ev.tag_done() + else: + self.log.warning("Unknows op %s" % ev.type) + ev.tag_failed("Unknown operation") + + def remove_table(self, tbl, dst_curs): + del self.table_map[tbl.name] + self.table_list.remove(tbl) + q = "select londiste.subscriber_remove_table(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, tbl.name]) + + def load_table_state(self, curs): + """Load table state from database. + + Todo: if all tables are OK, there is no need + to load state on every batch. + """ + + q = """select table_name, snapshot, merge_state + from londiste.subscriber_get_table_list(%s) + """ + curs.execute(q, [self.pgq_queue_name]) + + new_list = [] + new_map = {} + for row in curs.dictfetchall(): + t = self.get_table_by_name(row['table_name']) + if not t: + t = TableState(row['table_name'], self.log) + t.loaded_state(row['merge_state'], row['snapshot']) + new_list.append(t) + new_map[t.name] = t + + self.table_list = new_list + self.table_map = new_map + + def save_table_state(self, curs): + """Store changed table state in database.""" + + for t in self.table_list: + if not t.changed: + continue + merge_state = t.render_state() + self.log.info("storing state of %s: copy:%d new_state:%s" % ( + t.name, self.copy_thread, merge_state)) + q = "select londiste.subscriber_set_table_state(%s, %s, %s, %s)" + curs.execute(q, [self.pgq_queue_name, + t.name, t.str_snapshot, merge_state]) + t.changed = 0 + + def change_table_state(self, dst_db, tbl, state, tick_id = None): + tbl.change_state(state, tick_id) + self.save_table_state(dst_db.cursor()) + dst_db.commit() + + self.log.info("Table %s status changed to '%s'" % ( + tbl.name, tbl.render_state())) + + def get_table_by_state(self, state): + "get first table with specific state" + + for t in self.table_list: + if t.state == state: + return t + raise Exception('No table was found with state: %d' % state) + + def get_table_by_name(self, name): + if name.find('.') < 0: + name = "public.%s" % name + if name in self.table_map: + return self.table_map[name] + return None + + def fill_mirror_queue(self, ev_list, dst_curs): + # insert events + rows = [] + fields = ['ev_type', 'ev_data', 'ev_extra1'] + for ev in mirror_list: + rows.append((ev.type, ev.data, ev.extra1)) + pgq.bulk_insert_events(dst_curs, rows, fields, self.mirror_queue) + + # create tick + q = "select pgq.ticker(%s, %s)" + dst_curs.execute(q, [self.mirror_queue, self.cur_tick]) + + def launch_copy(self, tbl_stat): + self.log.info("Launching copy process") + script = sys.argv[0] + conf = self.cf.filename + if self.options.verbose: + cmd = "%s -d -v %s copy" + else: + cmd = "%s -d %s copy" + cmd = cmd % (script, conf) + self.log.debug("Launch args: "+repr(cmd)) + res = os.system(cmd) + self.log.debug("Launch result: "+repr(res)) + +if __name__ == '__main__': + script = Replicator(sys.argv[1:]) + script.start() + diff --git a/python/londiste/repair.py b/python/londiste/repair.py new file mode 100644 index 00000000..ec4bd404 --- /dev/null +++ b/python/londiste/repair.py @@ -0,0 +1,284 @@ + +"""Repair data on subscriber. + +Walks tables by primary key and searcher +missing inserts/updates/deletes. +""" + +import sys, os, time, psycopg, skytools + +from syncer import Syncer + +__all__ = ['Repairer'] + +def unescape(s): + return skytools.unescape_copy(s) + +def get_pkey_list(curs, tbl): + """Get list of pkey fields in right order.""" + + oid = skytools.get_table_oid(curs, tbl) + q = """SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = %s AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + ORDER BY k.attnum""" + curs.execute(q, [oid]) + list = [] + for row in curs.fetchall(): + list.append(row[0]) + return list + +def get_column_list(curs, tbl): + """Get list of columns in right order.""" + + oid = skytools.get_table_oid(curs, tbl) + q = """SELECT a.attname FROM pg_attribute a + WHERE a.attrelid = %s + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum""" + curs.execute(q, [oid]) + list = [] + for row in curs.fetchall(): + list.append(row[0]) + return list + +class Repairer(Syncer): + """Walks tables in primary key order and checks if data matches.""" + + + def process_sync(self, tbl, src_db, dst_db): + """Actual comparision.""" + + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info('Checking %s' % tbl) + + self.common_fields = [] + self.pkey_list = [] + copy_tbl = self.gen_copy_tbl(tbl, src_curs, dst_curs) + + dump_src = tbl + ".src" + dump_dst = tbl + ".dst" + + self.log.info("Dumping src table: %s" % tbl) + self.dump_table(tbl, copy_tbl, src_curs, dump_src) + src_db.commit() + self.log.info("Dumping dst table: %s" % tbl) + self.dump_table(tbl, copy_tbl, dst_curs, dump_dst) + dst_db.commit() + + self.log.info("Sorting src table: %s" % tbl) + + s_in, s_out = os.popen4("sort --version") + s_ver = s_out.read() + del s_in, s_out + if s_ver.find("coreutils") > 0: + args = "-S 30%" + else: + args = "" + os.system("sort %s -T . -o %s.sorted %s" % (args, dump_src, dump_src)) + self.log.info("Sorting dst table: %s" % tbl) + os.system("sort %s -T . -o %s.sorted %s" % (args, dump_dst, dump_dst)) + + self.dump_compare(tbl, dump_src + ".sorted", dump_dst + ".sorted") + + os.unlink(dump_src) + os.unlink(dump_dst) + os.unlink(dump_src + ".sorted") + os.unlink(dump_dst + ".sorted") + + def gen_copy_tbl(self, tbl, src_curs, dst_curs): + self.pkey_list = get_pkey_list(src_curs, tbl) + dst_pkey = get_pkey_list(dst_curs, tbl) + if dst_pkey != self.pkey_list: + self.log.error('pkeys do not match') + sys.exit(1) + + src_cols = get_column_list(src_curs, tbl) + dst_cols = get_column_list(dst_curs, tbl) + field_list = [] + for f in self.pkey_list: + field_list.append(f) + for f in src_cols: + if f in self.pkey_list: + continue + if f in dst_cols: + field_list.append(f) + + self.common_fields = field_list + + tbl_expr = "%s (%s)" % (tbl, ",".join(field_list)) + + self.log.debug("using copy expr: %s" % tbl_expr) + + return tbl_expr + + def dump_table(self, tbl, copy_tbl, curs, fn): + f = open(fn, "w", 64*1024) + curs.copy_to(f, copy_tbl) + size = f.tell() + f.close() + self.log.info('Got %d bytes' % size) + + def get_row(self, ln): + t = ln[:-1].split('\t') + row = {} + for i in range(len(self.common_fields)): + row[self.common_fields[i]] = t[i] + return row + + def dump_compare(self, tbl, src_fn, dst_fn): + self.log.info("Comparing dumps: %s" % tbl) + self.cnt_insert = 0 + self.cnt_update = 0 + self.cnt_delete = 0 + self.total_src = 0 + self.total_dst = 0 + f1 = open(src_fn, "r", 64*1024) + f2 = open(dst_fn, "r", 64*1024) + src_ln = f1.readline() + dst_ln = f2.readline() + if src_ln: self.total_src += 1 + if dst_ln: self.total_dst += 1 + + fix = "fix.%s.sql" % tbl + if os.path.isfile(fix): + os.unlink(fix) + + while src_ln or dst_ln: + keep_src = keep_dst = 0 + if src_ln != dst_ln: + src_row = self.get_row(src_ln) + dst_row = self.get_row(dst_ln) + + cmp = self.cmp_keys(src_row, dst_row) + if cmp > 0: + # src > dst + self.got_missed_delete(tbl, dst_row) + keep_src = 1 + elif cmp < 0: + # src < dst + self.got_missed_insert(tbl, src_row) + keep_dst = 1 + else: + if self.cmp_data(src_row, dst_row) != 0: + self.got_missed_update(tbl, src_row, dst_row) + + if not keep_src: + src_ln = f1.readline() + if src_ln: self.total_src += 1 + if not keep_dst: + dst_ln = f2.readline() + if dst_ln: self.total_dst += 1 + + self.log.info("finished %s: src: %d rows, dst: %d rows,"\ + " missed: %d inserts, %d updates, %d deletes" % ( + tbl, self.total_src, self.total_dst, + self.cnt_insert, self.cnt_update, self.cnt_delete)) + + def got_missed_insert(self, tbl, src_row): + self.cnt_insert += 1 + fld_list = self.common_fields + val_list = [] + for f in fld_list: + v = unescape(src_row[f]) + val_list.append(skytools.quote_literal(v)) + q = "insert into %s (%s) values (%s);" % ( + tbl, ", ".join(fld_list), ", ".join(val_list)) + self.show_fix(tbl, q, 'insert') + + def got_missed_update(self, tbl, src_row, dst_row): + self.cnt_update += 1 + fld_list = self.common_fields + set_list = [] + whe_list = [] + for f in self.pkey_list: + self.addcmp(whe_list, f, unescape(src_row[f])) + for f in fld_list: + v1 = src_row[f] + v2 = dst_row[f] + if self.cmp_value(v1, v2) == 0: + continue + + self.addeq(set_list, f, unescape(v1)) + self.addcmp(whe_list, f, unescape(v2)) + + q = "update only %s set %s where %s;" % ( + tbl, ", ".join(set_list), " and ".join(whe_list)) + self.show_fix(tbl, q, 'update') + + def got_missed_delete(self, tbl, dst_row, pkey_list): + self.cnt_delete += 1 + whe_list = [] + for f in self.pkey_list: + self.addcmp(whe_list, f, unescape(dst_row[f])) + q = "delete from only %s where %s;" % (tbl, " and ".join(whe_list)) + self.show_fix(tbl, q, 'delete') + + def show_fix(self, tbl, q, desc): + #self.log.warning("missed %s: %s" % (desc, q)) + fn = "fix.%s.sql" % tbl + open(fn, "a").write("%s\n" % q) + + def addeq(self, list, f, v): + vq = skytools.quote_literal(v) + s = "%s = %s" % (f, vq) + list.append(s) + + def addcmp(self, list, f, v): + if v is None: + s = "%s is null" % f + else: + vq = skytools.quote_literal(v) + s = "%s = %s" % (f, vq) + list.append(s) + + def cmp_data(self, src_row, dst_row): + for k in self.common_fields: + v1 = src_row[k] + v2 = dst_row[k] + if self.cmp_value(v1, v2) != 0: + return -1 + return 0 + + def cmp_value(self, v1, v2): + if v1 == v2: + return 0 + + # try to work around tz vs. notz + z1 = len(v1) + z2 = len(v2) + if z1 == z2 + 3 and z2 >= 19 and v1[z2] == '+': + v1 = v1[:-3] + if v1 == v2: + return 0 + elif z1 + 3 == z2 and z1 >= 19 and v2[z1] == '+': + v2 = v2[:-3] + if v1 == v2: + return 0 + + return -1 + + def cmp_keys(self, src_row, dst_row): + """Compare primary keys of the rows. + + Returns 1 if src > dst, -1 if src < dst and 0 if src == dst""" + + # None means table is done. tag it larger than any existing row. + if src_row is None: + if dst_row is None: + return 0 + return 1 + elif dst_row is None: + return -1 + + for k in self.pkey_list: + v1 = src_row[k] + v2 = dst_row[k] + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + return 0 + diff --git a/python/londiste/setup.py b/python/londiste/setup.py new file mode 100644 index 00000000..ed44b093 --- /dev/null +++ b/python/londiste/setup.py @@ -0,0 +1,580 @@ +#! /usr/bin/env python + +"""Londiste setup and sanity checker. + +""" +import sys, os, skytools +from installer import * + +__all__ = ['ProviderSetup', 'SubscriberSetup'] + +def find_column_types(curs, table): + table_oid = skytools.get_table_oid(curs, table) + if table_oid == None: + return None + + key_sql = """ + SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = %d AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + """ % table_oid + + # find columns + q = """ + SELECT a.attname as name, + CASE WHEN k.attname IS NOT NULL + THEN 'k' ELSE 'v' END AS type + FROM pg_attribute a LEFT JOIN (%s) k ON (k.attname = a.attname) + WHERE a.attrelid = %d AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + """ % (key_sql, table_oid) + curs.execute(q) + rows = curs.dictfetchall() + return rows + +def make_type_string(col_rows): + res = map(lambda x: x['type'], col_rows) + return "".join(res) + +class CommonSetup(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.set_single_loop(1) + self.pidfile = self.pidfile + ".setup" + + self.pgq_queue_name = self.cf.get("pgq_queue_name") + self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name) + self.fake = self.cf.getint('fake', 0) + + if len(self.args) < 3: + self.log.error("need subcommand") + sys.exit(1) + + def run(self): + self.admin() + + def fetch_provider_table_list(self, curs): + q = """select table_name, trigger_name + from londiste.provider_get_table_list(%s)""" + curs.execute(q, [self.pgq_queue_name]) + return curs.dictfetchall() + + def get_provider_table_list(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + list = self.fetch_provider_table_list(src_curs) + src_db.commit() + res = [] + for row in list: + res.append(row['table_name']) + return res + + def get_provider_seqs(self, curs): + q = """SELECT * from londiste.provider_get_seq_list(%s)""" + curs.execute(q, [self.pgq_queue_name]) + res = [] + for row in curs.fetchall(): + res.append(row[0]) + return res + + def get_all_seqs(self, curs): + q = """SELECT n.nspname || '.'|| c.relname + from pg_class c, pg_namespace n + where n.oid = c.relnamespace + and c.relkind = 'S' + order by 1""" + curs.execute(q) + res = [] + for row in curs.fetchall(): + res.append(row[0]) + return res + + def check_provider_queue(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + q = "select count(1) from pgq.get_queue_info(%s)" + src_curs.execute(q, [self.pgq_queue_name]) + ok = src_curs.fetchone()[0] + src_db.commit() + + if not ok: + self.log.error('Event queue does not exist yet') + sys.exit(1) + + def fetch_subscriber_tables(self, curs): + q = "select * from londiste.subscriber_get_table_list(%s)" + curs.execute(q, [self.pgq_queue_name]) + return curs.dictfetchall() + + def get_subscriber_table_list(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + list = self.fetch_subscriber_tables(dst_curs) + dst_db.commit() + res = [] + for row in list: + res.append(row['table_name']) + return res + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.add_option("--expect-sync", action="store_true", dest="expect_sync", + help = "no copy needed", default=False) + p.add_option("--force", action="store_true", + help="force", default=False) + return p + + +# +# Provider commands +# + +class ProviderSetup(CommonSetup): + + def admin(self): + cmd = self.args[2] + if cmd == "tables": + self.provider_show_tables() + elif cmd == "add": + self.provider_add_tables(self.args[3:]) + elif cmd == "remove": + self.provider_remove_tables(self.args[3:]) + elif cmd == "add-seq": + for seq in self.args[3:]: + self.provider_add_seq(seq) + self.provider_notify_change() + elif cmd == "remove-seq": + for seq in self.args[3:]: + self.provider_remove_seq(seq) + self.provider_notify_change() + elif cmd == "install": + self.provider_install() + elif cmd == "seqs": + self.provider_list_seqs() + else: + self.log.error('bad subcommand') + sys.exit(1) + + def provider_list_seqs(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + list = self.get_provider_seqs(src_curs) + src_db.commit() + + for seq in list: + print seq + + def provider_install(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + install_provider(src_curs, self.log) + + # create event queue + q = "select pgq.create_queue(%s)" + self.exec_provider(q, [self.pgq_queue_name]) + + def provider_add_tables(self, table_list): + self.check_provider_queue() + + cur_list = self.get_provider_table_list() + for tbl in table_list: + if tbl.find('.') < 0: + tbl = "public." + tbl + if tbl not in cur_list: + self.log.info('Adding %s' % tbl) + self.provider_add_table(tbl) + else: + self.log.info("Table %s already added" % tbl) + self.provider_notify_change() + + def provider_remove_tables(self, table_list): + self.check_provider_queue() + + cur_list = self.get_provider_table_list() + for tbl in table_list: + if tbl.find('.') < 0: + tbl = "public." + tbl + if tbl not in cur_list: + self.log.info('%s already removed' % tbl) + else: + self.log.info("Removing %s" % tbl) + self.provider_remove_table(tbl) + self.provider_notify_change() + + def provider_add_table(self, tbl): + q = "select londiste.provider_add_table(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, tbl]) + + def provider_remove_table(self, tbl): + q = "select londiste.provider_remove_table(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, tbl]) + + def provider_show_tables(self): + self.check_provider_queue() + list = self.get_provider_table_list() + for tbl in list: + print tbl + + def provider_notify_change(self): + q = "select londiste.provider_notify_change(%s)" + self.exec_provider(q, [self.pgq_queue_name]) + + def provider_add_seq(self, seq): + seq = skytools.fq_name(seq) + q = "select londiste.provider_add_seq(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, seq]) + + def provider_remove_seq(self, seq): + seq = skytools.fq_name(seq) + q = "select londiste.provider_remove_seq(%s, %s)" + self.exec_provider(q, [self.pgq_queue_name, seq]) + + def exec_provider(self, sql, args): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + + src_curs.execute(sql, args) + + if self.fake: + src_db.rollback() + else: + src_db.commit() + +# +# Subscriber commands +# + +class SubscriberSetup(CommonSetup): + + def admin(self): + cmd = self.args[2] + if cmd == "tables": + self.subscriber_show_tables() + elif cmd == "missing": + self.subscriber_missing_tables() + elif cmd == "add": + self.subscriber_add_tables(self.args[3:]) + elif cmd == "remove": + self.subscriber_remove_tables(self.args[3:]) + elif cmd == "resync": + self.subscriber_resync_tables(self.args[3:]) + elif cmd == "register": + self.subscriber_register() + elif cmd == "unregister": + self.subscriber_unregister() + elif cmd == "install": + self.subscriber_install() + elif cmd == "check": + self.check_tables(self.get_provider_table_list()) + elif cmd == "fkeys": + self.collect_fkeys(self.get_provider_table_list()) + elif cmd == "seqs": + self.subscriber_list_seqs() + elif cmd == "add-seq": + self.subscriber_add_seq(self.args[3:]) + elif cmd == "remove-seq": + self.subscriber_remove_seq(self.args[3:]) + else: + self.log.error('bad subcommand: ' + cmd) + sys.exit(1) + + def collect_fkeys(self, table_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + oid_list = [] + for tbl in table_list: + try: + oid = skytools.get_table_oid(dst_curs, tbl) + if oid: + oid_list.append(str(oid)) + except: + pass + if len(oid_list) == 0: + print "No tables" + return + oid_str = ",".join(oid_list) + + q = "SELECT n.nspname || '.' || t.relname as tbl, c.conname as con,"\ + " pg_get_constraintdef(c.oid) as def"\ + " FROM pg_constraint c, pg_class t, pg_namespace n"\ + " WHERE c.contype = 'f' and c.conrelid in (%s)"\ + " AND t.oid = c.conrelid AND n.oid = t.relnamespace" % oid_str + dst_curs.execute(q) + res = dst_curs.dictfetchall() + dst_db.commit() + + print "-- dropping" + for row in res: + q = "ALTER TABLE ONLY %(tbl)s DROP CONSTRAINT %(con)s;" + print q % row + print "-- creating" + for row in res: + q = "ALTER TABLE ONLY %(tbl)s ADD CONSTRAINT %(con)s %(def)s;" + print q % row + + def check_tables(self, table_list): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + failed = 0 + for tbl in table_list: + self.log.info('Checking %s' % tbl) + if not skytools.exists_table(src_curs, tbl): + self.log.error('Table %s missing from provider side' % tbl) + failed += 1 + elif not skytools.exists_table(dst_curs, tbl): + self.log.error('Table %s missing from subscriber side' % tbl) + failed += 1 + else: + failed += self.check_table_columns(src_curs, dst_curs, tbl) + failed += self.check_table_triggers(dst_curs, tbl) + + src_db.commit() + dst_db.commit() + + return failed + + def check_table_triggers(self, dst_curs, tbl): + oid = skytools.get_table_oid(dst_curs, tbl) + if not oid: + self.log.error('Table %s not found' % tbl) + return 1 + q = "select count(1) from pg_trigger where tgrelid = %s" + dst_curs.execute(q, [oid]) + got = dst_curs.fetchone()[0] + if got: + self.log.error('found trigger on table %s (%s)' % (tbl, str(oid))) + return 1 + else: + return 0 + + def check_table_columns(self, src_curs, dst_curs, tbl): + src_colrows = find_column_types(src_curs, tbl) + dst_colrows = find_column_types(dst_curs, tbl) + + src_cols = make_type_string(src_colrows) + dst_cols = make_type_string(dst_colrows) + if src_cols.find('k') < 0: + self.log.error('provider table %s has no primary key (%s)' % ( + tbl, src_cols)) + return 1 + if dst_cols.find('k') < 0: + self.log.error('subscriber table %s has no primary key (%s)' % ( + tbl, dst_cols)) + return 1 + + if src_cols != dst_cols: + self.log.warning('table %s structure is not same (%s/%s)'\ + ', trying to continue' % (tbl, src_cols, dst_cols)) + + err = 0 + for row in src_colrows: + found = 0 + for row2 in dst_colrows: + if row2['name'] == row['name']: + found = 1 + break + if not found: + err = 1 + self.log.error('%s: column %s on provider not on subscriber' + % (tbl, row['name'])) + elif row['type'] != row2['type']: + err = 1 + self.log.error('%s: pk different on column %s' + % (tbl, row['name'])) + + return err + + def subscriber_install(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + install_subscriber(dst_curs, self.log) + + if self.fake: + self.log.debug('rollback') + dst_db.rollback() + else: + self.log.debug('commit') + dst_db.commit() + + def subscriber_register(self): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + src_curs.execute("select pgq.register_consumer(%s, %s)", + [self.pgq_queue_name, self.consumer_id]) + src_db.commit() + + def subscriber_unregister(self): + q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + tbl_rows = self.fetch_subscriber_tables(dst_curs) + for row in tbl_rows: + dst_curs.execute(q, [self.pgq_queue_name, row['table_name']]) + dst_db.commit() + + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + src_curs.execute("select pgq.unregister_consumer(%s, %s)", + [self.pgq_queue_name, self.consumer_id]) + src_db.commit() + + def subscriber_show_tables(self): + list = self.get_subscriber_table_list() + for tbl in list: + print tbl + + def subscriber_missing_tables(self): + provider_tables = self.get_provider_table_list() + subscriber_tables = self.get_subscriber_table_list() + for tbl in provider_tables: + if tbl not in subscriber_tables: + print tbl + + def subscriber_add_tables(self, table_list): + provider_tables = self.get_provider_table_list() + subscriber_tables = self.get_subscriber_table_list() + + err = 0 + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl not in provider_tables: + err = 1 + self.log.error("Table %s not attached to queue" % tbl) + if err: + if self.options.force: + self.log.warning('--force used, ignoring errors') + else: + sys.exit(1) + + err = self.check_tables(table_list) + if err: + if self.options.force: + self.log.warning('--force used, ignoring errors') + else: + sys.exit(1) + + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl in subscriber_tables: + self.log.info("Table %s already added" % tbl) + else: + self.log.info("Adding %s" % tbl) + self.subscriber_add_one_table(tbl) + + def subscriber_remove_tables(self, table_list): + subscriber_tables = self.get_subscriber_table_list() + for tbl in table_list: + tbl = skytools.fq_name(tbl) + if tbl in subscriber_tables: + self.subscriber_remove_one_table(tbl) + else: + self.log.info("Table %s already removed" % tbl) + + def subscriber_resync_tables(self, table_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + list = self.fetch_subscriber_tables(dst_curs) + for tbl in table_list: + tbl = skytools.fq_name(tbl) + tbl_row = None + for row in list: + if row['table_name'] == tbl: + tbl_row = row + break + if not tbl_row: + self.log.warning("Table %s not found" % tbl) + elif tbl_row['merge_state'] != 'ok': + self.log.warning("Table %s is not in stable state" % tbl) + else: + self.log.info("Resyncing %s" % tbl) + q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)" + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def subscriber_add_one_table(self, tbl): + q = "select londiste.subscriber_add_table(%s, %s)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + if self.options.expect_sync: + q = "select londiste.subscriber_set_table_state(%s, %s, null, 'ok')" + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def subscriber_remove_one_table(self, tbl): + q = "select londiste.subscriber_remove_table(%s, %s)" + + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + dst_curs.execute(q, [self.pgq_queue_name, tbl]) + dst_db.commit() + + def get_subscriber_seq_list(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + q = "SELECT * from londiste.subscriber_get_seq_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + list = dst_curs.fetchall() + dst_db.commit() + res = [] + for row in list: + res.append(row[0]) + return res + + def subscriber_list_seqs(self): + list = self.get_subscriber_seq_list() + for seq in list: + print seq + + def subscriber_add_seq(self, seq_list): + src_db = self.get_database('provider_db') + src_curs = src_db.cursor() + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + + prov_list = self.get_provider_seqs(src_curs) + src_db.commit() + + full_list = self.get_all_seqs(dst_curs) + cur_list = self.get_subscriber_seq_list() + + for seq in seq_list: + seq = skytools.fq_name(seq) + if seq not in prov_list: + self.log.error('Seq %s does not exist on provider side' % seq) + continue + if seq not in full_list: + self.log.error('Seq %s does not exist on subscriber side' % seq) + continue + if seq in cur_list: + self.log.info('Seq %s already subscribed' % seq) + continue + + self.log.info('Adding sequence: %s' % seq) + q = "select londiste.subscriber_add_seq(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, seq]) + + dst_db.commit() + + def subscriber_remove_seq(self, seq_list): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + cur_list = self.get_subscriber_seq_list() + + for seq in seq_list: + seq = skytools.fq_name(seq) + if seq not in cur_list: + self.log.warning('Seq %s not subscribed') + else: + self.log.info('Removing sequence: %s' % seq) + q = "select londiste.subscriber_remove_seq(%s, %s)" + dst_curs.execute(q, [self.pgq_queue_name, seq]) + dst_db.commit() + diff --git a/python/londiste/syncer.py b/python/londiste/syncer.py new file mode 100644 index 00000000..eaee3468 --- /dev/null +++ b/python/londiste/syncer.py @@ -0,0 +1,177 @@ + +"""Catch moment when tables are in sync on master and slave. +""" + +import sys, time, skytools + +class Syncer(skytools.DBScript): + """Walks tables in primary key order and checks if data matches.""" + + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.set_single_loop(1) + + self.pgq_queue_name = self.cf.get("pgq_queue_name") + self.pgq_consumer_id = self.cf.get('pgq_consumer_id', self.job_name) + + if self.pidfile: + self.pidfile += ".repair" + + def init_optparse(self, p=None): + p = skytools.DBScript.init_optparse(self, p) + p.add_option("--force", action="store_true", help="ignore lag") + return p + + def check_consumer(self, src_db): + src_curs = src_db.cursor() + + # before locking anything check if consumer is working ok + q = "select extract(epoch from ticker_lag) from pgq.get_queue_list()"\ + " where queue_name = %s" + src_curs.execute(q, [self.pgq_queue_name]) + ticker_lag = src_curs.fetchone()[0] + q = "select extract(epoch from lag)"\ + " from pgq.get_consumer_list()"\ + " where queue_name = %s"\ + " and consumer_name = %s" + src_curs.execute(q, [self.pgq_queue_name, self.pgq_consumer_id]) + res = src_curs.fetchall() + src_db.commit() + + if len(res) == 0: + self.log.error('No such consumer') + sys.exit(1) + consumer_lag = res[0][0] + + if consumer_lag > ticker_lag + 10 and not self.options.force: + self.log.error('Consumer lagging too much, cannot proceed') + sys.exit(1) + + def get_subscriber_table_state(self): + dst_db = self.get_database('subscriber_db') + dst_curs = dst_db.cursor() + q = "select * from londiste.subscriber_get_table_list(%s)" + dst_curs.execute(q, [self.pgq_queue_name]) + res = dst_curs.dictfetchall() + dst_db.commit() + return res + + def work(self): + src_loc = self.cf.get('provider_db') + lock_db = self.get_database('provider_db', cache='lock_db') + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + + self.check_consumer(src_db) + + state_list = self.get_subscriber_table_state() + state_map = {} + full_list = [] + for ts in state_list: + name = ts['table_name'] + full_list.append(name) + state_map[name] = ts + + if len(self.args) > 2: + tlist = self.args[2:] + else: + tlist = full_list + + for tbl in tlist: + if not tbl in state_map: + self.log.warning('Table not subscribed: %s' % tbl) + continue + st = state_map[tbl] + if st['merge_state'] != 'ok': + self.log.info('Table %s not synced yet, no point' % tbl) + continue + self.check_table(tbl, lock_db, src_db, dst_db) + lock_db.commit() + src_db.commit() + dst_db.commit() + + def check_table(self, tbl, lock_db, src_db, dst_db): + """Get transaction to same state, then process.""" + + + lock_curs = lock_db.cursor() + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + if not skytools.exists_table(src_curs, tbl): + self.log.warning("Table %s does not exist on provider side" % tbl) + return + if not skytools.exists_table(dst_curs, tbl): + self.log.warning("Table %s does not exist on subscriber side" % tbl) + return + + # lock table in separate connection + self.log.info('Locking %s' % tbl) + lock_db.commit() + lock_curs.execute("LOCK TABLE %s IN SHARE MODE" % tbl) + lock_time = time.time() + + # now wait until consumer has updated target table until locking + self.log.info('Syncing %s' % tbl) + + # consumer must get futher than this tick + src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name]) + tick_id = src_curs.fetchone()[0] + src_db.commit() + # avoid depending on ticker by inserting second tick also + time.sleep(0.1) + src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name]) + src_db.commit() + src_curs.execute("select to_char(now(), 'YYYY-MM-DD HH24:MI:SS.MS')") + tpos = src_curs.fetchone()[0] + src_db.commit() + # now wait + while 1: + time.sleep(0.2) + + q = """select now() - lag > %s, now(), lag + from pgq.get_consumer_list() + where consumer_name = %s + and queue_name = %s""" + src_curs.execute(q, [tpos, self.pgq_consumer_id, self.pgq_queue_name]) + res = src_curs.fetchall() + src_db.commit() + + if len(res) == 0: + raise Exception('No such consumer') + + row = res[0] + self.log.debug("tpos=%s now=%s lag=%s ok=%s" % (tpos, row[1], row[2], row[0])) + if row[0]: + break + + # loop max 10 secs + if time.time() > lock_time + 10 and not self.options.force: + self.log.error('Consumer lagging too much, exiting') + lock_db.rollback() + sys.exit(1) + + # take snapshot on provider side + src_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + src_curs.execute("SELECT 1") + + # take snapshot on subscriber side + dst_db.commit() + dst_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE") + dst_curs.execute("SELECT 1") + + # release lock + lock_db.commit() + + # do work + self.process_sync(tbl, src_db, dst_db) + + # done + src_db.commit() + dst_db.commit() + + def process_sync(self, tbl, src_db, dst_db): + """It gets 2 connections in state where tbl should be in same state. + """ + raise Exception('process_sync not implemented') + diff --git a/python/londiste/table_copy.py b/python/londiste/table_copy.py new file mode 100644 index 00000000..1754baaf --- /dev/null +++ b/python/londiste/table_copy.py @@ -0,0 +1,107 @@ +#! /usr/bin/env python + +"""Do a full table copy. + +For internal usage. +""" + +import sys, os, skytools + +from skytools.dbstruct import * +from playback import * + +__all__ = ['CopyTable'] + +class CopyTable(Replicator): + def __init__(self, args, copy_thread = 1): + Replicator.__init__(self, args) + + if copy_thread: + self.pidfile += ".copy" + self.consumer_id += "_copy" + self.copy_thread = 1 + + def init_optparse(self, parser=None): + p = Replicator.init_optparse(self, parser) + p.add_option("--skip-truncate", action="store_true", dest="skip_truncate", + help = "avoid truncate", default=False) + return p + + def do_copy(self, tbl_stat): + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + + # it should not matter to pgq + src_db.commit() + dst_db.commit() + + # change to SERIALIZABLE isolation level + src_db.set_isolation_level(2) + src_db.commit() + + # initial sync copy + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + self.log.info("Starting full copy of %s" % tbl_stat.name) + + # find dst struct + src_struct = TableStruct(src_curs, tbl_stat.name) + dst_struct = TableStruct(dst_curs, tbl_stat.name) + + # check if columns match + dlist = dst_struct.get_column_list() + for c in src_struct.get_column_list(): + if c not in dlist: + raise Exception('Column %s does not exist on dest side' % c) + + # drop unnecessary stuff + objs = T_CONSTRAINT | T_INDEX | T_TRIGGER | T_RULE + dst_struct.drop(dst_curs, objs, log = self.log) + + # do truncate & copy + self.real_copy(src_curs, dst_curs, tbl_stat.name) + + # get snapshot + src_curs.execute("select get_current_snapshot()") + snapshot = src_curs.fetchone()[0] + src_db.commit() + + # restore READ COMMITTED behaviour + src_db.set_isolation_level(1) + src_db.commit() + + # create previously dropped objects + dst_struct.create(dst_curs, objs, log = self.log) + + # set state + tbl_stat.change_snapshot(snapshot) + if self.copy_thread: + tbl_stat.change_state(TABLE_CATCHING_UP) + else: + tbl_stat.change_state(TABLE_OK) + self.save_table_state(dst_curs) + dst_db.commit() + + def real_copy(self, srccurs, dstcurs, tablename): + "Main copy logic." + + # drop data + if self.options.skip_truncate: + self.log.info("%s: skipping truncate" % tablename) + else: + self.log.info("%s: truncating" % tablename) + dstcurs.execute("truncate " + tablename) + + # do copy + self.log.info("%s: start copy" % tablename) + col_list = skytools.get_table_columns(srccurs, tablename) + stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list) + if stats: + self.log.info("%s: copy finished: %d bytes, %d rows" % ( + tablename, stats[0], stats[1])) + +if __name__ == '__main__': + script = CopyTable(sys.argv[1:]) + script.start() + diff --git a/python/pgq/__init__.py b/python/pgq/__init__.py new file mode 100644 index 00000000..f0e9c1a6 --- /dev/null +++ b/python/pgq/__init__.py @@ -0,0 +1,6 @@ +"""PgQ framework for Python.""" + +from pgq.event import * +from pgq.consumer import * +from pgq.producer import * + diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py new file mode 100644 index 00000000..bd49dccf --- /dev/null +++ b/python/pgq/consumer.py @@ -0,0 +1,410 @@ + +"""PgQ consumer framework for Python. + +API problems(?): + - process_event() and process_batch() should have db as argument. + - should ev.tag*() update db immidiately? + +""" + +import sys, time, skytools + +from pgq.event import * + +__all__ = ['Consumer', 'RemoteConsumer', 'SerialConsumer'] + +class Consumer(skytools.DBScript): + """Consumer base class. + """ + + def __init__(self, service_name, db_name, args): + """Initialize new consumer. + + @param service_name: service_name for DBScript + @param db_name: name of database for get_database() + @param args: cmdline args for DBScript + """ + + skytools.DBScript.__init__(self, service_name, args) + + self.db_name = db_name + self.reg_list = [] + self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name) + self.pgq_queue_name = self.cf.get("pgq_queue_name") + + def attach(self): + """Attach consumer to interesting queues.""" + res = self.register_consumer(self.pgq_queue_name) + return res + + def detach(self): + """Detach consumer from all queues.""" + tmp = self.reg_list[:] + for q in tmp: + self.unregister_consumer(q) + + def process_event(self, db, event): + """Process one event. + + Should be overrided by user code. + + Event should be tagged as done, retry or failed. + If not, it will be tagged as for retry. + """ + raise Exception("needs to be implemented") + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + + By default calls process_event for each. + Can be overrided by user code. + + Events should be tagged as done, retry or failed. + If not, they will be tagged as for retry. + """ + for ev in event_list: + self.process_event(db, ev) + + def work(self): + """Do the work loop, once (internal).""" + + if len(self.reg_list) == 0: + self.log.debug("Attaching") + self.attach() + + db = self.get_database(self.db_name) + curs = db.cursor() + + data_avail = 0 + for queue in self.reg_list: + self.stat_start() + + # acquire batch + batch_id = self._load_next_batch(curs, queue) + db.commit() + if batch_id == None: + continue + data_avail = 1 + + # load events + list = self._load_batch_events(curs, batch_id, queue) + db.commit() + + # process events + self._launch_process_batch(db, batch_id, list) + + # done + self._finish_batch(curs, batch_id, list) + db.commit() + self.stat_end(len(list)) + + # if false, script sleeps + return data_avail + + def register_consumer(self, queue_name): + db = self.get_database(self.db_name) + cx = db.cursor() + cx.execute("select pgq.register_consumer(%s, %s)", + [queue_name, self.consumer_id]) + res = cx.fetchone()[0] + db.commit() + + self.reg_list.append(queue_name) + + return res + + def unregister_consumer(self, queue_name): + db = self.get_database(self.db_name) + cx = db.cursor() + cx.execute("select pgq.unregister_consumer(%s, %s)", + [queue_name, self.consumer_id]) + db.commit() + + self.reg_list.remove(queue_name) + + def _launch_process_batch(self, db, batch_id, list): + self.process_batch(db, batch_id, list) + + def _load_batch_events(self, curs, batch_id, queue_name): + """Fetch all events for this batch.""" + + # load events + sql = "select * from pgq.get_batch_events(%d)" % batch_id + curs.execute(sql) + rows = curs.dictfetchall() + + # map them to python objects + list = [] + for r in rows: + ev = Event(queue_name, r) + list.append(ev) + + return list + + def _load_next_batch(self, curs, queue_name): + """Allocate next batch. (internal)""" + + q = "select pgq.next_batch(%s, %s)" + curs.execute(q, [queue_name, self.consumer_id]) + return curs.fetchone()[0] + + def _finish_batch(self, curs, batch_id, list): + """Tag events and notify that the batch is done.""" + + retry = failed = 0 + for ev in list: + if ev.status == EV_FAILED: + self._tag_failed(curs, batch_id, ev) + failed += 1 + elif ev.status == EV_RETRY: + self._tag_retry(curs, batch_id, ev) + retry += 1 + curs.execute("select pgq.finish_batch(%s)", [batch_id]) + + def _tag_failed(self, curs, batch_id, ev): + """Tag event as failed. (internal)""" + curs.execute("select pgq.event_failed(%s, %s, %s)", + [batch_id, ev.id, ev.fail_reason]) + + def _tag_retry(self, cx, batch_id, ev): + """Tag event for retry. (internal)""" + cx.execute("select pgq.event_retry(%s, %s, %s)", + [batch_id, ev.id, ev.retry_time]) + + def get_batch_info(self, batch_id): + """Get info about batch. + + @return: Return value is a dict of: + + - queue_name: queue name + - consumer_name: consumers name + - batch_start: batch start time + - batch_end: batch end time + - tick_id: end tick id + - prev_tick_id: start tick id + - lag: how far is batch_end from current moment. + """ + db = self.get_database(self.db_name) + cx = db.cursor() + q = "select queue_name, consumer_name, batch_start, batch_end,"\ + " prev_tick_id, tick_id, lag"\ + " from pgq.get_batch_info(%s)" + cx.execute(q, [batch_id]) + row = cx.dictfetchone() + db.commit() + return row + + def stat_start(self): + self.stat_batch_start = time.time() + + def stat_end(self, count): + t = time.time() + self.stat_add('count', count) + self.stat_add('duration', t - self.stat_batch_start) + + +class RemoteConsumer(Consumer): + """Helper for doing event processing in another database. + + Requires that whole batch is processed in one TX. + """ + + def __init__(self, service_name, db_name, remote_db, args): + Consumer.__init__(self, service_name, db_name, args) + self.remote_db = remote_db + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + + By default calls process_event for each. + """ + dst_db = self.get_database(self.remote_db) + curs = dst_db.cursor() + + if self.is_last_batch(curs, batch_id): + for ev in event_list: + ev.tag_done() + return + + self.process_remote_batch(db, batch_id, event_list, dst_db) + + self.set_last_batch(curs, batch_id) + dst_db.commit() + + def is_last_batch(self, dst_curs, batch_id): + """Helper function to keep track of last successful batch + in external database. + """ + q = "select pgq_ext.is_batch_done(%s, %s)" + dst_curs.execute(q, [ self.consumer_id, batch_id ]) + return dst_curs.fetchone()[0] + + def set_last_batch(self, dst_curs, batch_id): + """Helper function to set last successful batch + in external database. + """ + q = "select pgq_ext.set_batch_done(%s, %s)" + dst_curs.execute(q, [ self.consumer_id, batch_id ]) + + def process_remote_batch(self, db, batch_id, event_list, dst_db): + raise Exception('process_remote_batch not implemented') + +class SerialConsumer(Consumer): + """Consumer that applies batches sequentially in second database. + + Requirements: + - Whole batch in one TX. + - Must not use retry queue. + + Features: + - Can detect if several batches are already applied to dest db. + - If some ticks are lost. allows to seek back on queue. + Whether it succeeds, depends on pgq configuration. + """ + + def __init__(self, service_name, db_name, remote_db, args): + Consumer.__init__(self, service_name, db_name, args) + self.remote_db = remote_db + self.dst_completed_table = "pgq_ext.completed_tick" + self.cur_batch_info = None + + def startup(self): + if self.options.rewind: + self.rewind() + sys.exit(0) + if self.options.reset: + self.dst_reset() + sys.exit(0) + return Consumer.startup(self) + + def init_optparse(self, parser = None): + p = Consumer.init_optparse(self, parser) + p.add_option("--rewind", action = "store_true", + help = "change queue position according to destination") + p.add_option("--reset", action = "store_true", + help = "reset queue pos on destination side") + return p + + def process_batch(self, db, batch_id, event_list): + """Process all events in batch. + """ + + dst_db = self.get_database(self.remote_db) + curs = dst_db.cursor() + + self.cur_batch_info = self.get_batch_info(batch_id) + + # check if done + if self.is_batch_done(curs): + for ev in event_list: + ev.tag_done() + return + + # actual work + self.process_remote_batch(db, batch_id, event_list, dst_db) + + # make sure no retry events + for ev in event_list: + if ev.status == EV_RETRY: + raise Exception("SerialConsumer must not use retry queue") + + # finish work + self.set_batch_done(curs) + dst_db.commit() + + def is_batch_done(self, dst_curs): + """Helper function to keep track of last successful batch + in external database. + """ + + prev_tick = self.cur_batch_info['prev_tick_id'] + + q = "select last_tick_id from %s where consumer_id = %%s" % ( + self.dst_completed_table ,) + dst_curs.execute(q, [self.consumer_id]) + res = dst_curs.fetchone() + + if not res or not res[0]: + # seems this consumer has not run yet against dst_db + return False + dst_tick = res[0] + + if prev_tick == dst_tick: + # on track + return False + + if prev_tick < dst_tick: + self.log.warning('Got tick %d, dst has %d - skipping' % (prev_tick, dst_tick)) + return True + else: + self.log.error('Got tick %d, dst has %d - ticks lost' % (prev_tick, dst_tick)) + raise Exception('Lost ticks') + + def set_batch_done(self, dst_curs): + """Helper function to set last successful batch + in external database. + """ + tick_id = self.cur_batch_info['tick_id'] + q = "delete from %s where consumer_id = %%s; "\ + "insert into %s (consumer_id, last_tick_id) values (%%s, %%s)" % ( + self.dst_completed_table, + self.dst_completed_table) + dst_curs.execute(q, [ self.consumer_id, + self.consumer_id, tick_id ]) + + def attach(self): + new = Consumer.attach(self) + if new: + self.clean_completed_tick() + + def detach(self): + """If detaching, also clean completed tick table on dest.""" + + Consumer.detach(self) + self.clean_completed_tick() + + def clean_completed_tick(self): + self.log.info("removing completed tick from dst") + dst_db = self.get_database(self.remote_db) + dst_curs = dst_db.cursor() + + q = "delete from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + dst_db.commit() + + def process_remote_batch(self, db, batch_id, event_list, dst_db): + raise Exception('process_remote_batch not implemented') + + def rewind(self): + self.log.info("Rewinding queue") + src_db = self.get_database(self.db_name) + dst_db = self.get_database(self.remote_db) + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + + q = "select last_tick_id from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + row = dst_curs.fetchone() + if row: + dst_tick = row[0] + q = "select pgq.register_consumer(%s, %s, %s)" + src_curs.execute(q, [self.pgq_queue_name, self.consumer_id, dst_tick]) + else: + self.log.warning('No tick found on dst side') + + dst_db.commit() + src_db.commit() + + def dst_reset(self): + self.log.info("Resetting queue tracking on dst side") + dst_db = self.get_database(self.remote_db) + dst_curs = dst_db.cursor() + + q = "delete from %s where consumer_id = %%s" % ( + self.dst_completed_table,) + dst_curs.execute(q, [self.consumer_id]) + dst_db.commit() + + diff --git a/python/pgq/event.py b/python/pgq/event.py new file mode 100644 index 00000000..d7b2d7ee --- /dev/null +++ b/python/pgq/event.py @@ -0,0 +1,60 @@ + +"""PgQ event container. +""" + +__all__ = ('EV_RETRY', 'EV_DONE', 'EV_FAILED', 'Event') + +# Event status codes +EV_RETRY = 0 +EV_DONE = 1 +EV_FAILED = 2 + +_fldmap = { + 'ev_id': 'ev_id', + 'ev_txid': 'ev_txid', + 'ev_time': 'ev_time', + 'ev_type': 'ev_type', + 'ev_data': 'ev_data', + 'ev_extra1': 'ev_extra1', + 'ev_extra2': 'ev_extra2', + 'ev_extra3': 'ev_extra3', + 'ev_extra4': 'ev_extra4', + + 'id': 'ev_id', + 'txid': 'ev_txid', + 'time': 'ev_time', + 'type': 'ev_type', + 'data': 'ev_data', + 'extra1': 'ev_extra1', + 'extra2': 'ev_extra2', + 'extra3': 'ev_extra3', + 'extra4': 'ev_extra4', +} + +class Event(object): + """Event data for consumers. + + Consumer is supposed to tag them after processing. + If not, events will stay in retry queue. + """ + def __init__(self, queue_name, row): + self._event_row = row + self.status = EV_RETRY + self.retry_time = 60 + self.fail_reason = "Buggy consumer" + self.queue_name = queue_name + + def __getattr__(self, key): + return self._event_row[_fldmap[key]] + + def tag_done(self): + self.status = EV_DONE + + def tag_retry(self, retry_time = 60): + self.status = EV_RETRY + self.retry_time = retry_time + + def tag_failed(self, reason): + self.status = EV_FAILED + self.fail_reason = reason + diff --git a/python/pgq/maint.py b/python/pgq/maint.py new file mode 100644 index 00000000..4636f74f --- /dev/null +++ b/python/pgq/maint.py @@ -0,0 +1,99 @@ +"""PgQ maintenance functions.""" + +import skytools, time + +def get_pgq_api_version(curs): + q = "select count(1) from pg_proc p, pg_namespace n"\ + " where n.oid = p.pronamespace and n.nspname='pgq'"\ + " and p.proname='version';" + curs.execute(q) + if not curs.fetchone()[0]: + return '1.0.0' + + curs.execute("select pgq.version()") + return curs.fetchone()[0] + +def version_ge(curs, want_ver): + """Check is db version of pgq is greater than want_ver.""" + db_ver = get_pgq_api_version(curs) + want_tuple = map(int, want_ver.split('.')) + db_tuple = map(int, db_ver.split('.')) + if db_tuple[0] != want_tuple[0]: + raise Exception('Wrong major version') + if db_tuple[1] >= want_tuple[1]: + return 1 + return 0 + +class MaintenanceJob(skytools.DBScript): + """Periodic maintenance.""" + def __init__(self, ticker, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + self.ticker = ticker + self.last_time = 0 # start immidiately + self.last_ticks = 0 + self.clean_ticks = 1 + self.maint_delay = 5*60 + + def startup(self): + # disable regular DBScript startup() + pass + + def reload(self): + skytools.DBScript.reload(self) + + # force loop_delay + self.loop_delay = 5 + + self.maint_delay = 60 * self.cf.getfloat('maint_delay_min', 5) + self.maint_delay = self.cf.getfloat('maint_delay', self.maint_delay) + + def work(self): + t = time.time() + if self.last_time + self.maint_delay > t: + return + + self.do_maintenance() + + self.last_time = t + duration = time.time() - t + self.stat_add('maint_duration', duration) + + def do_maintenance(self): + """Helper function for running maintenance.""" + + db = self.get_database('db', autocommit=1) + cx = db.cursor() + + if skytools.exists_function(cx, "pgq.maint_rotate_tables_step1", 1): + # rotate each queue in own TX + q = "select queue_name from pgq.get_queue_info()" + cx.execute(q) + for row in cx.fetchall(): + cx.execute("select pgq.maint_rotate_tables_step1(%s)", [row[0]]) + res = cx.fetchone()[0] + if res: + self.log.info('Rotating %s' % row[0]) + else: + cx.execute("select pgq.maint_rotate_tables_step1();") + + # finish rotation + cx.execute("select pgq.maint_rotate_tables_step2();") + + # move retry events to main queue in small blocks + rcount = 0 + while 1: + cx.execute('select pgq.maint_retry_events();') + res = cx.fetchone()[0] + rcount += res + if res == 0: + break + if rcount: + self.log.info('Got %d events for retry' % rcount) + + # vacuum tables that are needed + cx.execute('set maintenance_work_mem = 32768') + cx.execute('select * from pgq.maint_tables_to_vacuum()') + for row in cx.fetchall(): + cx.execute('vacuum %s;' % row[0]) + + diff --git a/python/pgq/producer.py b/python/pgq/producer.py new file mode 100644 index 00000000..81e1ca4f --- /dev/null +++ b/python/pgq/producer.py @@ -0,0 +1,41 @@ + +"""PgQ producer helpers for Python. +""" + +import skytools + +_fldmap = { + 'id': 'ev_id', + 'time': 'ev_time', + 'type': 'ev_type', + 'data': 'ev_data', + 'extra1': 'ev_extra1', + 'extra2': 'ev_extra2', + 'extra3': 'ev_extra3', + 'extra4': 'ev_extra4', + + 'ev_id': 'ev_id', + 'ev_time': 'ev_time', + 'ev_type': 'ev_type', + 'ev_data': 'ev_data', + 'ev_extra1': 'ev_extra1', + 'ev_extra2': 'ev_extra2', + 'ev_extra3': 'ev_extra3', + 'ev_extra4': 'ev_extra4', +} + +def bulk_insert_events(curs, rows, fields, queue_name): + q = "select pgq.current_event_table(%s)" + curs.execute(q, [queue_name]) + tbl = curs.fetchone()[0] + db_fields = map(_fldmap.get, fields) + skytools.magic_insert(curs, tbl, rows, db_fields) + +def insert_event(curs, queue, ev_type, ev_data, + extra1=None, extra2=None, + extra3=None, extra4=None): + q = "select pgq.insert_event(%s, %s, %s, %s, %s, %s, %s)" + curs.execute(q, [queue, ev_type, ev_data, + extra1, extra2, extra3, extra4]) + return curs.fetchone()[0] + diff --git a/python/pgq/status.py b/python/pgq/status.py new file mode 100644 index 00000000..2214045f --- /dev/null +++ b/python/pgq/status.py @@ -0,0 +1,93 @@ + +"""Status display. +""" + +import sys, os, skytools + +def ival(data, as = None): + "Format interval for output" + if not as: + as = data.split('.')[-1] + numfmt = 'FM9999999' + expr = "coalesce(to_char(extract(epoch from %s), '%s') || 's', 'NULL') as %s" + return expr % (data, numfmt, as) + +class PGQStatus(skytools.DBScript): + def __init__(self, args, check = 0): + skytools.DBScript.__init__(self, 'pgqadm', args) + + self.show_status() + + sys.exit(0) + + def show_status(self): + db = self.get_database("db", autocommit=1) + cx = db.cursor() + + cx.execute("show server_version") + pgver = cx.fetchone()[0] + cx.execute("select pgq.version()") + qver = cx.fetchone()[0] + print "Postgres version: %s PgQ version: %s" % (pgver, qver) + + q = """select f.queue_name, f.num_tables, %s, %s, %s, + q.queue_ticker_max_lag, q.queue_ticker_max_amount, + q.queue_ticker_idle_interval + from pgq.get_queue_info() f, pgq.queue q + where q.queue_name = f.queue_name""" % ( + ival('f.rotation_delay'), + ival('f.ticker_lag'), + ) + cx.execute(q) + event_rows = cx.dictfetchall() + + q = """select queue_name, consumer_name, %s, %s, %s + from pgq.get_consumer_info()""" % ( + ival('lag'), + ival('last_seen'), + ) + cx.execute(q) + consumer_rows = cx.dictfetchall() + + print "\n%-32s %s %9s %13s %6s" % ('Event queue', + 'Rotation', 'Ticker', 'TLag') + print '-' * 78 + for ev_row in event_rows: + tck = "%s/%ss/%ss" % (ev_row['queue_ticker_max_amount'], + ev_row['queue_ticker_max_lag'], + ev_row['queue_ticker_idle_interval']) + rot = "%s/%s" % (ev_row['queue_ntables'], ev_row['queue_rotation_period']) + print "%-39s%7s %9s %13s %6s" % ( + ev_row['queue_name'], + rot, + tck, + ev_row['ticker_lag'], + ) + print '-' * 78 + print "\n%-42s %9s %9s" % ( + 'Consumer', 'Lag', 'LastSeen') + print '-' * 78 + for ev_row in event_rows: + cons = self.pick_consumers(ev_row, consumer_rows) + self.show_queue(ev_row, cons) + print '-' * 78 + db.commit() + + def show_consumer(self, cons): + print " %-48s %9s %9s" % ( + cons['consumer_name'], + cons['lag'], cons['last_seen']) + def show_queue(self, ev_row, consumer_rows): + print "%(queue_name)s:" % ev_row + for cons in consumer_rows: + self.show_consumer(cons) + + + def pick_consumers(self, ev_row, consumer_rows): + res = [] + for con in consumer_rows: + if con['queue_name'] != ev_row['queue_name']: + continue + res.append(con) + return res + diff --git a/python/pgq/ticker.py b/python/pgq/ticker.py new file mode 100644 index 00000000..c218eaf1 --- /dev/null +++ b/python/pgq/ticker.py @@ -0,0 +1,172 @@ +"""PgQ ticker. + +It will also launch maintenance job. +""" + +import sys, os, time, threading +import skytools + +from maint import MaintenanceJob + +__all__ = ['SmartTicker'] + +def is_txid_sane(curs): + curs.execute("select get_current_txid()") + txid = curs.fetchone()[0] + + # on 8.2 theres no such table + if not skytools.exists_table(curs, 'txid.epoch'): + return 1 + + curs.execute("select epoch, last_value from txid.epoch") + epoch, last_val = curs.fetchone() + stored_val = (epoch << 32) | last_val + + if stored_val <= txid: + return 1 + else: + return 0 + +class QueueStatus(object): + def __init__(self, name): + self.queue_name = name + self.seq_name = None + self.idle_period = 60 + self.max_lag = 3 + self.max_count = 200 + self.last_tick_time = 0 + self.last_count = 0 + self.quiet_count = 0 + + def set_data(self, row): + self.seq_name = row['queue_event_seq'] + self.idle_period = row['queue_ticker_idle_period'] + self.max_lag = row['queue_ticker_max_lag'] + self.max_count = row['queue_ticker_max_count'] + + def need_tick(self, cur_count, cur_time): + # check if tick is needed + need_tick = 0 + lag = cur_time - self.last_tick_time + + if cur_count == self.last_count: + # totally idle database + + # don't go immidiately to big delays, as seq grows before commit + if self.quiet_count < 5: + if lag >= self.max_lag: + need_tick = 1 + self.quiet_count += 1 + else: + if lag >= self.idle_period: + need_tick = 1 + else: + self.quiet_count = 0 + # somewhat loaded machine + if cur_count - self.last_count >= self.max_count: + need_tick = 1 + elif lag >= self.max_lag: + need_tick = 1 + if need_tick: + self.last_tick_time = cur_time + self.last_count = cur_count + return need_tick + +class SmartTicker(skytools.DBScript): + last_tick_event = 0 + last_tick_time = 0 + quiet_count = 0 + tick_count = 0 + maint_thread = None + + def __init__(self, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + + self.ticker_log_time = 0 + self.ticker_log_delay = 5*60 + self.queue_map = {} + self.refresh_time = 0 + + def reload(self): + skytools.DBScript.reload(self) + self.ticker_log_delay = self.cf.getfloat("ticker_log_delay", 5*60) + + def startup(self): + if self.maint_thread: + return + + db = self.get_database("db", autocommit = 1) + cx = db.cursor() + ok = is_txid_sane(cx) + if not ok: + self.log.error('txid in bad state') + sys.exit(1) + + self.maint_thread = MaintenanceJob(self, [self.cf.filename]) + t = threading.Thread(name = 'maint_thread', + target = self.maint_thread.run) + t.setDaemon(1) + t.start() + + def refresh_queues(self, cx): + q = "select queue_name, queue_event_seq, queue_ticker_idle_period,"\ + " queue_ticker_max_lag, queue_ticker_max_count"\ + " from pgq.queue"\ + " where not queue_external_ticker" + cx.execute(q) + new_map = {} + data_list = [] + from_list = [] + for row in cx.dictfetchall(): + queue_name = row['queue_name'] + try: + que = self.queue_map[queue_name] + except KeyError, x: + que = QueueStatus(queue_name) + que.set_data(row) + new_map[queue_name] = que + + p1 = "'%s', %s.last_value" % (queue_name, que.seq_name) + data_list.append(p1) + from_list.append(que.seq_name) + + self.queue_map = new_map + self.seq_query = "select %s from %s" % ( + ", ".join(data_list), + ", ".join(from_list)) + + if len(from_list) == 0: + self.seq_query = None + + self.refresh_time = time.time() + + def work(self): + db = self.get_database("db", autocommit = 1) + cx = db.cursor() + + cur_time = time.time() + + if cur_time >= self.refresh_time + 30: + self.refresh_queues(cx) + + if not self.seq_query: + return + + # now check seqs + cx.execute(self.seq_query) + res = cx.fetchone() + pos = 0 + while pos < len(res): + id = res[pos] + val = res[pos + 1] + pos += 2 + que = self.queue_map[id] + if que.need_tick(val, cur_time): + cx.execute("select pgq.ticker(%s)", [que.queue_name]) + self.tick_count += 1 + + if cur_time > self.ticker_log_time + self.ticker_log_delay: + self.ticker_log_time = cur_time + self.stat_add('ticks', self.tick_count) + self.tick_count = 0 + diff --git a/python/pgqadm.py b/python/pgqadm.py new file mode 100755 index 00000000..78f513dc --- /dev/null +++ b/python/pgqadm.py @@ -0,0 +1,162 @@ +#! /usr/bin/env python + +"""PgQ ticker and maintenance. +""" + +import sys +import skytools + +from pgq.ticker import SmartTicker +from pgq.status import PGQStatus +#from pgq.admin import PGQAdmin + +"""TODO: +pgqadm ini check +""" + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + ticker start ticking & maintenance process + + status show overview of queue health + check show problematic consumers + + install install code into db + create QNAME create queue + drop QNAME drop queue + register QNAME CONS install code into db + unregister QNAME CONS install code into db + config QNAME [VAR=VAL] show or change queue config +""" + +config_allowed_list = [ + 'queue_ticker_max_lag', 'queue_ticker_max_amount', + 'queue_ticker_idle_interval', 'queue_rotation_period'] + +class PGQAdmin(skytools.DBScript): + def __init__(self, args): + skytools.DBScript.__init__(self, 'pgqadm', args) + self.set_single_loop(1) + + if len(self.args) < 2: + print "need command" + sys.exit(1) + + int_cmds = { + 'create': self.create_queue, + 'drop': self.drop_queue, + 'register': self.register, + 'unregister': self.unregister, + 'install': self.installer, + 'config': self.change_config, + } + + cmd = self.args[1] + if cmd == "ticker": + script = SmartTicker(args) + elif cmd == "status": + script = PGQStatus(args) + elif cmd == "check": + script = PGQStatus(args, check = 1) + elif cmd in int_cmds: + script = None + self.work = int_cmds[cmd] + else: + print "unknown command" + sys.exit(1) + + if self.pidfile: + self.pidfile += ".admin" + self.run_script = script + + def start(self): + if self.run_script: + self.run_script.start() + else: + skytools.DBScript.start(self) + + def init_optparse(self, parser=None): + p = skytools.DBScript.init_optparse(self, parser) + p.set_usage(command_usage.strip()) + return p + + def installer(self): + objs = [ + skytools.DBLanguage("plpgsql"), + skytools.DBLanguage("plpythonu"), + skytools.DBFunction("get_current_txid", 0, sql_file="txid.sql"), + skytools.DBSchema("pgq", sql_file="pgq.sql"), + ] + + db = self.get_database('db') + curs = db.cursor() + skytools.db_install(curs, objs, self.log) + db.commit() + + def create_queue(self): + qname = self.args[2] + self.log.info('Creating queue: %s' % qname) + self.exec_sql("select pgq.create_queue(%s)", [qname]) + + def drop_queue(self): + qname = self.args[2] + self.log.info('Dropping queue: %s' % qname) + self.exec_sql("select pgq.drop_queue(%s)", [qname]) + + def register(self): + qname = self.args[2] + cons = self.args[3] + self.log.info('Registering consumer %s on queue %s' % (cons, qname)) + self.exec_sql("select pgq.register_consumer(%s, %s)", [qname, cons]) + + def unregister(self): + qname = self.args[2] + cons = self.args[3] + self.log.info('Unregistering consumer %s from queue %s' % (cons, qname)) + self.exec_sql("select pgq.unregister_consumer(%s, %s)", [qname, cons]) + + def change_config(self): + qname = self.args[2] + if len(self.args) == 3: + self.show_config(qname) + return + alist = [] + for el in self.args[3:]: + k, v = el.split('=') + if k not in config_allowed_list: + raise Exception('unknown config var: '+k) + expr = "%s=%s" % (k, skytools.quote_literal(v)) + alist.append(expr) + self.log.info('Change queue %s config to: %s' % (qname, ", ".join(alist))) + sql = "update pgq.queue set %s where queue_name = %s" % ( + ", ".join(alist), skytools.quote_literal(qname)) + self.exec_sql(sql, []) + + def exec_sql(self, q, args): + self.log.debug(q) + db = self.get_database('db') + curs = db.cursor() + curs.execute(q, args) + db.commit() + + def show_config(self, qname): + klist = ",".join(config_allowed_list) + q = "select * from pgq.queue where queue_name = %s" + db = self.get_database('db') + curs = db.cursor() + curs.execute(q, [qname]) + res = curs.dictfetchone() + db.commit() + + print qname + for k in config_allowed_list: + print " %s=%s" % (k, res[k]) + +if __name__ == '__main__': + script = PGQAdmin(sys.argv[1:]) + script.start() + + + diff --git a/python/skytools/__init__.py b/python/skytools/__init__.py new file mode 100644 index 00000000..ed2b39bc --- /dev/null +++ b/python/skytools/__init__.py @@ -0,0 +1,10 @@ + +"""Tools for Python database scripts.""" + +from config import * +from dbstruct import * +from gzlog import * +from quoting import * +from scripting import * +from sqltools import * + diff --git a/python/skytools/config.py b/python/skytools/config.py new file mode 100644 index 00000000..de420322 --- /dev/null +++ b/python/skytools/config.py @@ -0,0 +1,139 @@ + +"""Nicer config class.""" + +import sys, os, ConfigParser, socket + +__all__ = ['Config'] + +class Config(object): + """Bit improved ConfigParser. + + Additional features: + - Remembers section. + - Acceps defaults in get() functions. + - List value support. + """ + def __init__(self, main_section, filename, sane_config = 1): + """Initialize Config and read from file. + + @param sane_config: chooses between ConfigParser/SafeConfigParser. + """ + defs = { + 'job_name': main_section, + 'service_name': main_section, + 'host_name': socket.gethostname(), + } + if not os.path.isfile(filename): + raise Exception('Config file not found: '+filename) + + self.filename = filename + self.sane_config = sane_config + if sane_config: + self.cf = ConfigParser.SafeConfigParser(defs) + else: + self.cf = ConfigParser.ConfigParser(defs) + self.cf.read(filename) + self.main_section = main_section + if not self.cf.has_section(main_section): + raise Exception("Wrong config file, no section '%s'"%main_section) + + def reload(self): + """Re-reads config file.""" + self.cf.read(self.filename) + + def get(self, key, default=None): + """Reads string value, if not set then default.""" + try: + return self.cf.get(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getint(self, key, default=None): + """Reads int value, if not set then default.""" + try: + return self.cf.getint(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getboolean(self, key, default=None): + """Reads boolean value, if not set then default.""" + try: + return self.cf.getboolean(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getfloat(self, key, default=None): + """Reads float value, if not set then default.""" + try: + return self.cf.getfloat(self.main_section, key) + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getlist(self, key, default=None): + """Reads comma-separated list from key.""" + try: + s = self.cf.get(self.main_section, key).strip() + res = [] + if not s: + return res + for v in s.split(","): + res.append(v.strip()) + return res + except ConfigParser.NoOptionError, det: + if default == None: + raise Exception("Config value not set: " + key) + return default + + def getfile(self, key, default=None): + """Reads filename from config. + + In addition to reading string value, expands ~ to user directory. + """ + fn = self.get(key, default) + if fn == "" or fn == "-": + return fn + # simulate that the cwd is script location + #path = os.path.dirname(sys.argv[0]) + # seems bad idea, cwd should be cwd + + fn = os.path.expanduser(fn) + + return fn + + def get_wildcard(self, key, values=[], default=None): + """Reads a wildcard property from conf and returns its string value, if not set then default.""" + + orig_key = key + keys = [key] + + for wild in values: + key = key.replace('*', wild, 1) + keys.append(key) + keys.reverse() + + for key in keys: + try: + return self.cf.get(self.main_section, key) + except ConfigParser.NoOptionError, det: + pass + + if default == None: + raise Exception("Config value not set: " + orig_key) + return default + + def sections(self): + """Returns list of sections in config file, excluding DEFAULT.""" + return self.cf.sections() + + def clone(self, main_section): + """Return new Config() instance with new main section on same config file.""" + return Config(main_section, self.filename, self.sane_config) + diff --git a/python/skytools/dbstruct.py b/python/skytools/dbstruct.py new file mode 100644 index 00000000..22333429 --- /dev/null +++ b/python/skytools/dbstruct.py @@ -0,0 +1,380 @@ +"""Find table structure and allow CREATE/DROP elements from it. +""" + +import sys, re + +from sqltools import fq_name_parts, get_table_oid + +__all__ = ['TableStruct', + 'T_TABLE', 'T_CONSTRAINT', 'T_INDEX', 'T_TRIGGER', + 'T_RULE', 'T_GRANT', 'T_OWNER', 'T_PKEY', 'T_ALL'] + +T_TABLE = 1 << 0 +T_CONSTRAINT = 1 << 1 +T_INDEX = 1 << 2 +T_TRIGGER = 1 << 3 +T_RULE = 1 << 4 +T_GRANT = 1 << 5 +T_OWNER = 1 << 6 +T_PKEY = 1 << 20 # special, one of constraints +T_ALL = ( T_TABLE | T_CONSTRAINT | T_INDEX + | T_TRIGGER | T_RULE | T_GRANT | T_OWNER ) + +# +# Utility functions +# + +def find_new_name(curs, name): + """Create new object name for case the old exists. + + Needed when creating a new table besides old one. + """ + # cut off previous numbers + m = re.search('_[0-9]+$', name) + if m: + name = name[:m.start()] + + # now loop + for i in range(1, 1000): + tname = "%s_%d" % (name, i) + q = "select count(1) from pg_class where relname = %s" + curs.execute(q, [tname]) + if curs.fetchone()[0] == 0: + return tname + + # failed + raise Exception('find_new_name failed') + +def rx_replace(rx, sql, new_part): + """Find a regex match and replace that part with new_part.""" + m = re.search(rx, sql, re.I) + if not m: + raise Exception('rx_replace failed') + p1 = sql[:m.start()] + p2 = sql[m.end():] + return p1 + new_part + p2 + +# +# Schema objects +# + +class TElem(object): + """Keeps info about one metadata object.""" + SQL = "" + type = 0 + def get_create_sql(self, curs): + """Return SQL statement for creating or None if not supported.""" + return None + def get_drop_sql(self, curs): + """Return SQL statement for dropping or None of not supported.""" + return None + +class TConstraint(TElem): + """Info about constraint.""" + type = T_CONSTRAINT + SQL = """ + SELECT conname as name, pg_get_constraintdef(oid) as def, contype + FROM pg_constraint WHERE conrelid = %(oid)s + """ + def __init__(self, table_name, row): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + self.contype = row['contype'] + + # tag pkeys + if self.contype == 'p': + self.type += T_PKEY + + def get_create_sql(self, curs, new_table_name=None): + fmt = "ALTER TABLE ONLY %s ADD CONSTRAINT %s %s;" + if new_table_name: + name = self.name + if self.contype in ('p', 'u'): + name = find_new_name(curs, self.name) + sql = fmt % (new_table_name, name, self.defn) + else: + sql = fmt % (self.table_name, self.name, self.defn) + return sql + + def get_drop_sql(self, curs): + fmt = "ALTER TABLE ONLY %s DROP CONSTRAINT %s;" + sql = fmt % (self.table_name, self.name) + return sql + +class TIndex(TElem): + """Info about index.""" + type = T_INDEX + SQL = """ + SELECT n.nspname || '.' || c.relname as name, + pg_get_indexdef(i.indexrelid) as defn + FROM pg_index i, pg_class c, pg_namespace n + WHERE c.oid = i.indexrelid AND i.indrelid = %(oid)s + AND n.oid = c.relnamespace + AND NOT EXISTS + (select objid from pg_depend + where classid = %(pg_class_oid)s + and objid = c.oid + and deptype = 'i') + """ + def __init__(self, table_name, row): + self.name = row['name'] + self.defn = row['defn'] + ';' + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + name = find_new_name(curs, self.name) + pnew = "INDEX %s ON %s " % (name, new_table_name) + rx = r"\bINDEX[ ][a-z0-9._]+[ ]ON[ ][a-z0-9._]+[ ]" + sql = rx_replace(rx, self.defn, pnew) + return sql + def get_drop_sql(self, curs): + return 'DROP INDEX %s;' % self.name + +class TRule(TElem): + """Info about rule.""" + type = T_RULE + SQL = """ + SELECT rulename as name, pg_get_ruledef(oid) as def + FROM pg_rewrite + WHERE ev_class = %(oid)s AND rulename <> '_RETURN'::name + """ + def __init__(self, table_name, row, new_name = None): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + rx = r"\bTO[ ][a-z0-9._]+[ ]DO[ ]" + pnew = "TO %s DO " % new_table_name + return rx_replace(rx, self.defn, pnew) + + def get_drop_sql(self, curs): + return 'DROP RULE %s ON %s' % (self.name, self.table_name) + +class TTrigger(TElem): + """Info about trigger.""" + type = T_TRIGGER + SQL = """ + SELECT tgname as name, pg_get_triggerdef(oid) as def + FROM pg_trigger + WHERE tgrelid = %(oid)s AND NOT tgisconstraint + """ + def __init__(self, table_name, row): + self.table_name = table_name + self.name = row['name'] + self.defn = row['def'] + ';' + + def get_create_sql(self, curs, new_table_name = None): + if not new_table_name: + return self.defn + + rx = r"\bON[ ][a-z0-9._]+[ ]" + pnew = "ON %s " % new_table_name + return rx_replace(rx, self.defn, pnew) + + def get_drop_sql(self, curs): + return 'DROP TRIGGER %s ON %s' % (self.name, self.table_name) + +class TOwner(TElem): + """Info about table owner.""" + type = T_OWNER + SQL = """ + SELECT pg_get_userbyid(relowner) as owner FROM pg_class + WHERE oid = %(oid)s + """ + def __init__(self, table_name, row, new_name = None): + self.table_name = table_name + self.name = 'Owner' + self.owner = row['owner'] + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.table_name + return 'ALTER TABLE %s OWNER TO %s;' % (new_name, self.owner) + +class TGrant(TElem): + """Info about permissions.""" + type = T_GRANT + SQL = "SELECT relacl FROM pg_class where oid = %(oid)s" + acl_map = { + 'r': 'SELECT', 'w': 'UPDATE', 'a': 'INSERT', 'd': 'DELETE', + 'R': 'RULE', 'x': 'REFERENCES', 't': 'TRIGGER', 'X': 'EXECUTE', + 'U': 'USAGE', 'C': 'CREATE', 'T': 'TEMPORARY' + } + def acl_to_grants(self, acl): + if acl == "arwdRxt": # ALL for tables + return "ALL" + return ", ".join([ self.acl_map[c] for c in acl ]) + + def parse_relacl(self, relacl): + if relacl is None: + return [] + if len(relacl) > 0 and relacl[0] == '{' and relacl[-1] == '}': + relacl = relacl[1:-1] + list = [] + for f in relacl.split(','): + user, tmp = f.strip('"').split('=') + acl, who = tmp.split('/') + list.append((user, acl, who)) + return list + + def __init__(self, table_name, row, new_name = None): + self.name = table_name + self.acl_list = self.parse_relacl(row['relacl']) + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.name + + list = [] + for user, acl, who in self.acl_list: + astr = self.acl_to_grants(acl) + sql = "GRANT %s ON %s TO %s;" % (astr, new_name, user) + list.append(sql) + return "\n".join(list) + + def get_drop_sql(self, curs): + list = [] + for user, acl, who in self.acl_list: + sql = "REVOKE ALL FROM %s ON %s;" % (user, self.name) + list.append(sql) + return "\n".join(list) + +class TColumn(TElem): + """Info about table column.""" + SQL = """ + select a.attname as name, + a.attname || ' ' + || format_type(a.atttypid, a.atttypmod) + || case when a.attnotnull then ' not null' else '' end + || case when a.atthasdef then ' ' || d.adsrc else '' end + as def + from pg_attribute a left join pg_attrdef d + on (d.adrelid = a.attrelid and d.adnum = a.attnum) + where a.attrelid = %(oid)s + and not a.attisdropped + and a.attnum > 0 + order by a.attnum; + """ + def __init__(self, table_name, row): + self.name = row['name'] + self.column_def = row['def'] + +class TTable(TElem): + """Info about table only (columns).""" + type = T_TABLE + def __init__(self, table_name, col_list): + self.name = table_name + self.col_list = col_list + + def get_create_sql(self, curs, new_name = None): + if not new_name: + new_name = self.name + sql = "create table %s (" % new_name + sep = "\n\t" + for c in self.col_list: + sql += sep + c.column_def + sep = ",\n\t" + sql += "\n);" + return sql + + def get_drop_sql(self, curs): + return "DROP TABLE %s;" % self.name + +# +# Main table object, loads all the others +# + +class TableStruct(object): + """Collects and manages all info about table. + + Allow to issue CREATE/DROP statements about any + group of elements. + """ + def __init__(self, curs, table_name): + """Initializes class by loading info about table_name from database.""" + + self.table_name = table_name + + # fill args + schema, name = fq_name_parts(table_name) + args = { + 'schema': schema, + 'table': name, + 'oid': get_table_oid(curs, table_name), + 'pg_class_oid': get_table_oid(curs, 'pg_catalog.pg_class'), + } + + # load table struct + self.col_list = self._load_elem(curs, args, TColumn) + self.object_list = [ TTable(table_name, self.col_list) ] + + # load additional objects + to_load = [TConstraint, TIndex, TTrigger, TRule, TGrant, TOwner] + for eclass in to_load: + self.object_list += self._load_elem(curs, args, eclass) + + def _load_elem(self, curs, args, eclass): + list = [] + curs.execute(eclass.SQL % args) + for row in curs.dictfetchall(): + list.append(eclass(self.table_name, row)) + return list + + def create(self, curs, objs, new_table_name = None, log = None): + """Issues CREATE statements for requested set of objects. + + If new_table_name is giver, creates table under that name + and also tries to rename all indexes/constraints that conflict + with existing table. + """ + + for o in self.object_list: + if o.type & objs: + sql = o.get_create_sql(curs, new_table_name) + if not sql: + continue + if log: + log.info('Creating %s' % o.name) + log.debug(sql) + curs.execute(sql) + + def drop(self, curs, objs, log = None): + """Issues DROP statements for requested set of objects.""" + for o in self.object_list: + if o.type & objs: + sql = o.get_drop_sql(curs) + if not sql: + continue + if log: + log.info('Dropping %s' % o.name) + log.debug(sql) + curs.execute(sql) + + def get_column_list(self): + """Returns list of column names the table has.""" + + res = [] + for c in self.col_list: + res.append(c.name) + return res + +def test(): + import psycopg + db = psycopg.connect("dbname=fooz") + curs = db.cursor() + + s = TableStruct(curs, "public.data1") + + s.drop(curs, T_ALL) + s.create(curs, T_ALL) + s.create(curs, T_ALL, "data1_new") + s.create(curs, T_PKEY) + +if __name__ == '__main__': + test() + diff --git a/python/skytools/gzlog.py b/python/skytools/gzlog.py new file mode 100644 index 00000000..558e2813 --- /dev/null +++ b/python/skytools/gzlog.py @@ -0,0 +1,39 @@ + +"""Atomic append of gzipped data. + +The point is - if several gzip streams are concated, they +are read back as one whose stream. +""" + +import gzip +from cStringIO import StringIO + +__all__ = ['gzip_append'] + +# +# gzip storage +# +def gzip_append(filename, data, level = 6): + """Append a block of data to file with safety checks.""" + + # compress data + buf = StringIO() + g = gzip.GzipFile(fileobj = buf, compresslevel = level, mode = "w") + g.write(data) + g.close() + zdata = buf.getvalue() + + # append, safely + f = open(filename, "a+", 0) + f.seek(0, 2) + pos = f.tell() + try: + f.write(zdata) + f.close() + except Exception, ex: + # rollback on error + f.seek(pos, 0) + f.truncate() + f.close() + raise ex + diff --git a/python/skytools/quoting.py b/python/skytools/quoting.py new file mode 100644 index 00000000..96b0b022 --- /dev/null +++ b/python/skytools/quoting.py @@ -0,0 +1,156 @@ +# quoting.py + +"""Various helpers for string quoting/unquoting.""" + +import psycopg, urllib, re + +# +# SQL quoting +# + +def quote_literal(s): + """Quote a literal value for SQL. + + Surronds it with single-quotes. + """ + + if s == None: + return "null" + s = psycopg.QuotedString(str(s)) + return str(s) + +def quote_copy(s): + """Quoting for copy command.""" + + if s == None: + return "\\N" + s = str(s) + s = s.replace("\\", "\\\\") + s = s.replace("\t", "\\t") + s = s.replace("\n", "\\n") + s = s.replace("\r", "\\r") + return s + +def quote_bytea_raw(s): + """Quoting for bytea parser.""" + + if s == None: + return None + return s.replace("\\", "\\\\").replace("\0", "\\000") + +def quote_bytea_literal(s): + """Quote bytea for regular SQL.""" + + return quote_literal(quote_bytea_raw(s)) + +def quote_bytea_copy(s): + """Quote bytea for COPY.""" + + return quote_copy(quote_bytea_raw(s)) + +def quote_statement(sql, dict): + """Quote whose statement. + + Data values are taken from dict. + """ + xdict = {} + for k, v in dict.items(): + xdict[k] = quote_literal(v) + return sql % xdict + +# +# quoting for JSON strings +# + +_jsre = re.compile(r'[\x00-\x1F\\/"]') +_jsmap = { "\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", + "\t": "\\t", "\\": "\\\\", '"': '\\"', + "/": "\\/", # to avoid html attacks +} + +def _json_quote_char(m): + c = m.group(0) + try: + return _jsmap[c] + except KeyError: + return r"\u%04x" % ord(c) + +def quote_json(s): + """JSON style quoting.""" + if s is None: + return "null" + return '"%s"' % _jsre.sub(_json_quote_char, s) + +# +# Database specific urlencode and urldecode. +# + +def db_urlencode(dict): + """Database specific urlencode. + + Encode None as key without '='. That means that in "foo&bar=", + foo is NULL and bar is empty string. + """ + + elem_list = [] + for k, v in dict.items(): + if v is None: + elem = urllib.quote_plus(str(k)) + else: + elem = urllib.quote_plus(str(k)) + '=' + urllib.quote_plus(str(v)) + elem_list.append(elem) + return '&'.join(elem_list) + +def db_urldecode(qs): + """Database specific urldecode. + + Decode key without '=' as None. + This also does not support one key several times. + """ + + res = {} + for elem in qs.split('&'): + if not elem: + continue + pair = elem.split('=', 1) + name = urllib.unquote_plus(pair[0]) + if len(pair) == 1: + res[name] = None + else: + res[name] = urllib.unquote_plus(pair[1]) + return res + +# +# Remove C-like backslash escapes +# + +_esc_re = r"\\([0-7][0-7][0-7]|.)" +_esc_rc = re.compile(_esc_re) +_esc_map = { + 't': '\t', + 'n': '\n', + 'r': '\r', + 'a': '\a', + 'b': '\b', + "'": "'", + '"': '"', + '\\': '\\', +} + +def _sub_unescape(m): + v = m.group(1) + if len(v) == 1: + return _esc_map[v] + else: + return chr(int(v, 8)) + +def unescape(val): + """Removes C-style escapes from string.""" + return _esc_rc.sub(_sub_unescape, val) + +def unescape_copy(val): + """Removes C-style escapes, also converts "\N" to None.""" + if val == r"\N": + return None + return unescape(val) + diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py new file mode 100644 index 00000000..cf976801 --- /dev/null +++ b/python/skytools/scripting.py @@ -0,0 +1,523 @@ + +"""Useful functions and classes for database scripts.""" + +import sys, os, signal, psycopg, optparse, traceback, time +import logging, logging.handlers, logging.config + +from skytools.config import * +import skytools.skylog + +__all__ = ['daemonize', 'run_single_process', 'DBScript', + 'I_AUTOCOMMIT', 'I_READ_COMMITTED', 'I_SERIALIZABLE'] + +# +# daemon mode +# + +def daemonize(): + """Turn the process into daemon. + + Goes background and disables all i/o. + """ + + # launch new process, kill parent + pid = os.fork() + if pid != 0: + os._exit(0) + + # start new session + os.setsid() + + # stop i/o + fd = os.open("/dev/null", os.O_RDWR) + os.dup2(fd, 0) + os.dup2(fd, 1) + os.dup2(fd, 2) + if fd > 2: + os.close(fd) + +# +# Pidfile locking+cleanup & daemonization combined +# + +def _write_pidfile(pidfile): + pid = os.getpid() + f = open(pidfile, 'w') + f.write(str(pid)) + f.close() + +def run_single_process(runnable, daemon, pidfile): + """Run runnable class, possibly daemonized, locked on pidfile.""" + + # check if another process is running + if pidfile and os.path.isfile(pidfile): + print "Pidfile exists, another process running?" + sys.exit(1) + + # daemonize if needed and write pidfile + if daemon: + daemonize() + if pidfile: + _write_pidfile(pidfile) + + # Catch SIGTERM to cleanup pidfile + def sigterm_hook(signum, frame): + try: + os.remove(pidfile) + except: pass + sys.exit(0) + # attach it to signal + if pidfile: + signal.signal(signal.SIGTERM, sigterm_hook) + + # run + try: + runnable.run() + finally: + # another try of cleaning up + if pidfile: + try: + os.remove(pidfile) + except: pass + +# +# logging setup +# + +_log_config_done = 0 +_log_init_done = {} + +def _init_log(job_name, cf, log_level): + """Logging setup happens here.""" + global _log_init_done, _log_config_done + + got_skylog = 0 + use_skylog = cf.getint("use_skylog", 0) + + # load logging config if needed + if use_skylog and not _log_config_done: + # python logging.config braindamage: + # cannot specify external classess without such hack + logging.skylog = skytools.skylog + + # load general config + list = ['skylog.ini', '~/.skylog.ini', '/etc/skylog.ini'] + for fn in list: + fn = os.path.expanduser(fn) + if os.path.isfile(fn): + defs = {'job_name': job_name} + logging.config.fileConfig(fn, defs) + got_skylog = 1 + break + _log_config_done = 1 + if not got_skylog: + sys.stderr.write("skylog.ini not found!\n") + sys.exit(1) + + # avoid duplicate logging init for job_name + log = logging.getLogger(job_name) + if job_name in _log_init_done: + return log + _log_init_done[job_name] = 1 + + # compatibility: specify ini file in script config + logfile = cf.getfile("logfile", "") + if logfile: + fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s') + size = cf.getint('log_size', 10*1024*1024) + num = cf.getint('log_count', 3) + hdlr = logging.handlers.RotatingFileHandler( + logfile, 'a', size, num) + hdlr.setFormatter(fmt) + log.addHandler(hdlr) + + # if skylog.ini is disabled or not available, log at least to stderr + if not got_skylog: + hdlr = logging.StreamHandler() + fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s') + hdlr.setFormatter(fmt) + log.addHandler(hdlr) + + log.setLevel(log_level) + + return log + +#: how old connections need to be closed +DEF_CONN_AGE = 20*60 # 20 min + +#: isolation level not set +I_DEFAULT = -1 + +#: isolation level constant for AUTOCOMMIT +I_AUTOCOMMIT = 0 +#: isolation level constant for READ COMMITTED +I_READ_COMMITTED = 1 +#: isolation level constant for SERIALIZABLE +I_SERIALIZABLE = 2 + +class DBCachedConn(object): + """Cache a db connection.""" + def __init__(self, name, loc, max_age = DEF_CONN_AGE): + self.name = name + self.loc = loc + self.conn = None + self.conn_time = 0 + self.max_age = max_age + self.autocommit = -1 + self.isolation_level = -1 + + def get_connection(self, autocommit = 0, isolation_level = -1): + # autocommit overrider isolation_level + if autocommit: + isolation_level = I_AUTOCOMMIT + + # default isolation_level is READ COMMITTED + if isolation_level < 0: + isolation_level = I_READ_COMMITTED + + # new conn? + if not self.conn: + self.isolation_level = isolation_level + self.conn = psycopg.connect(self.loc) + + self.conn.set_isolation_level(isolation_level) + self.conn_time = time.time() + else: + if self.isolation_level != isolation_level: + raise Exception("Conflict in isolation_level") + + # done + return self.conn + + def refresh(self): + if not self.conn: + return + #for row in self.conn.notifies(): + # if row[0].lower() == "reload": + # self.reset() + # return + if not self.max_age: + return + if time.time() - self.conn_time >= self.max_age: + self.reset() + + def reset(self): + if not self.conn: + return + + # drop reference + conn = self.conn + self.conn = None + + if self.isolation_level == I_AUTOCOMMIT: + return + + # rollback & close + try: + conn.rollback() + except: pass + try: + conn.close() + except: pass + +class DBScript(object): + """Base class for database scripts. + + Handles logging, daemonizing, config, errors. + """ + service_name = None + job_name = None + cf = None + log = None + + def __init__(self, service_name, args): + """Script setup. + + User class should override work() and optionally __init__(), startup(), + reload(), reset() and init_optparse(). + + NB: in case of daemon, the __init__() and startup()/work() will be + run in different processes. So nothing fancy should be done in __init__(). + + @param service_name: unique name for script. + It will be also default job_name, if not specified in config. + @param args: cmdline args (sys.argv[1:]), but can be overrided + """ + self.service_name = service_name + self.db_cache = {} + self.go_daemon = 0 + self.do_single_loop = 0 + self.looping = 1 + self.need_reload = 1 + self.stat_dict = {} + self.log_level = logging.INFO + self.work_state = 1 + + # parse command line + parser = self.init_optparse() + self.options, self.args = parser.parse_args(args) + + # check args + if self.options.daemon: + self.go_daemon = 1 + if self.options.quiet: + self.log_level = logging.WARNING + if self.options.verbose: + self.log_level = logging.DEBUG + if len(self.args) < 1: + print "need config file" + sys.exit(1) + conf_file = self.args[0] + + # load config + self.cf = Config(self.service_name, conf_file) + self.job_name = self.cf.get("job_name", self.service_name) + self.pidfile = self.cf.getfile("pidfile", '') + + self.reload() + + # init logging + self.log = _init_log(self.job_name, self.cf, self.log_level) + + # send signal, if needed + if self.options.cmd == "kill": + self.send_signal(signal.SIGTERM) + elif self.options.cmd == "stop": + self.send_signal(signal.SIGINT) + elif self.options.cmd == "reload": + self.send_signal(signal.SIGHUP) + + def init_optparse(self, parser = None): + """Initialize a OptionParser() instance that will be used to + parse command line arguments. + + Note that it can be overrided both directions - either DBScript + will initialize a instance and passes to user code or user can + initialize and then pass to DBScript.init_optparse(). + + @param parser: optional OptionParser() instance, + where DBScript should attachs its own arguments. + @return: initialized OptionParser() instance. + """ + if parser: + p = parser + else: + p = optparse.OptionParser() + p.set_usage("%prog [options] INI") + # generic options + p.add_option("-q", "--quiet", action="store_true", + help = "make program silent") + p.add_option("-v", "--verbose", action="store_true", + help = "make program verbose") + p.add_option("-d", "--daemon", action="store_true", + help = "go background") + + # control options + g = optparse.OptionGroup(p, 'control running process') + g.add_option("-r", "--reload", + action="store_const", const="reload", dest="cmd", + help = "reload config (send SIGHUP)") + g.add_option("-s", "--stop", + action="store_const", const="stop", dest="cmd", + help = "stop program safely (send SIGINT)") + g.add_option("-k", "--kill", + action="store_const", const="kill", dest="cmd", + help = "kill program immidiately (send SIGTERM)") + p.add_option_group(g) + + return p + + def send_signal(self, sig): + if not self.pidfile: + self.log.warning("No pidfile in config, nothing todo") + sys.exit(0) + if not os.path.isfile(self.pidfile): + self.log.warning("No pidfile, process not running") + sys.exit(0) + pid = int(open(self.pidfile, "r").read()) + os.kill(pid, sig) + sys.exit(0) + + def set_single_loop(self, do_single_loop): + """Changes whether the script will loop or not.""" + self.do_single_loop = do_single_loop + + def start(self): + """This will launch main processing thread.""" + if self.go_daemon: + if not self.pidfile: + self.log.error("Daemon needs pidfile") + sys.exit(1) + run_single_process(self, self.go_daemon, self.pidfile) + + def stop(self): + """Safely stops processing loop.""" + self.looping = 0 + + def reload(self): + "Reload config." + self.cf.reload() + self.loop_delay = self.cf.getfloat("loop_delay", 1.0) + + def hook_sighup(self, sig, frame): + "Internal SIGHUP handler. Minimal code here." + self.need_reload = 1 + + def hook_sigint(self, sig, frame): + "Internal SIGINT handler. Minimal code here." + self.stop() + + def stat_add(self, key, value): + self.stat_put(key, value) + + def stat_put(self, key, value): + """Sets a stat value.""" + self.stat_dict[key] = value + + def stat_increase(self, key, increase = 1): + """Increases a stat value.""" + if key in self.stat_dict: + self.stat_dict[key] += increase + else: + self.stat_dict[key] = increase + + def send_stats(self): + "Send statistics to log." + + res = [] + for k, v in self.stat_dict.items(): + res.append("%s: %s" % (k, str(v))) + + if len(res) == 0: + return + + logmsg = "{%s}" % ", ".join(res) + self.log.info(logmsg) + self.stat_dict = {} + + def get_database(self, dbname, autocommit = 0, isolation_level = -1, + cache = None, max_age = DEF_CONN_AGE): + """Load cached database connection. + + User must not store it permanently somewhere, + as all connections will be invalidated on reset. + """ + + if not cache: + cache = dbname + if cache in self.db_cache: + dbc = self.db_cache[cache] + else: + loc = self.cf.get(dbname) + dbc = DBCachedConn(cache, loc, max_age) + self.db_cache[cache] = dbc + + return dbc.get_connection(autocommit, isolation_level) + + def close_database(self, dbname): + """Explicitly close a cached connection. + + Next call to get_database() will reconnect. + """ + if dbname in self.db_cache: + dbc = self.db_cache[dbname] + dbc.reset() + + def reset(self): + "Something bad happened, reset all connections." + for dbc in self.db_cache.values(): + dbc.reset() + self.db_cache = {} + + def run(self): + "Thread main loop." + + # run startup, safely + try: + self.startup() + except KeyboardInterrupt, det: + raise + except SystemExit, det: + raise + except Exception, det: + exc, msg, tb = sys.exc_info() + self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % ( + self.job_name, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + del tb + self.reset() + sys.exit(1) + + while self.looping: + # reload config, if needed + if self.need_reload: + self.reload() + self.need_reload = 0 + + # do some work + work = self.run_once() + + # send stats that was added + self.send_stats() + + # reconnect if needed + for dbc in self.db_cache.values(): + dbc.refresh() + + # exit if needed + if self.do_single_loop: + self.log.debug("Only single loop requested, exiting") + break + + # remember work state + self.work_state = work + # should sleep? + if not work: + try: + time.sleep(self.loop_delay) + except Exception, d: + self.log.debug("sleep failed: "+str(d)) + sys.exit(0) + + def run_once(self): + "Run users work function, safely." + try: + return self.work() + except SystemExit, d: + self.send_stats() + self.log.info("got SystemExit(%s), exiting" % str(d)) + self.reset() + raise d + except KeyboardInterrupt, d: + self.send_stats() + self.log.info("got KeyboardInterrupt, exiting") + self.reset() + sys.exit(1) + except Exception, d: + self.send_stats() + exc, msg, tb = sys.exc_info() + self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % ( + self.job_name, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + del tb + self.reset() + if self.looping: + time.sleep(20) + return 1 + + def work(self): + "Here should user's processing happen." + raise Exception("Nothing implemented?") + + def startup(self): + """Will be called just before entering main loop. + + In case of daemon, if will be called in same process as work(), + unlike __init__(). + """ + + # set signals + signal.signal(signal.SIGHUP, self.hook_sighup) + signal.signal(signal.SIGINT, self.hook_sigint) + + diff --git a/python/skytools/skylog.py b/python/skytools/skylog.py new file mode 100644 index 00000000..2f6344ae --- /dev/null +++ b/python/skytools/skylog.py @@ -0,0 +1,173 @@ +"""Our log handlers for Python's logging package. +""" + +import sys, os, time, socket, psycopg +import logging, logging.handlers + +from quoting import quote_json + +# configurable file logger +class EasyRotatingFileHandler(logging.handlers.RotatingFileHandler): + """Easier setup for RotatingFileHandler.""" + def __init__(self, filename, maxBytes = 10*1024*1024, backupCount = 3): + """Args same as for RotatingFileHandler, but in filename '~' is expanded.""" + fn = os.path.expanduser(filename) + logging.handlers.RotatingFileHandler.__init__(self, fn, maxBytes=maxBytes, backupCount=backupCount) + +# send JSON message over UDP +class UdpLogServerHandler(logging.handlers.DatagramHandler): + """Sends log records over UDP to logserver in JSON format.""" + + # map logging levels to logserver levels + _level_map = { + logging.DEBUG : 'DEBUG', + logging.INFO : 'INFO', + logging.WARNING : 'WARN', + logging.ERROR : 'ERROR', + logging.CRITICAL: 'FATAL', + } + + # JSON message template + _log_template = '{\n\t'\ + '"logger": "skytools.UdpLogServer",\n\t'\ + '"timestamp": %.0f,\n\t'\ + '"level": "%s",\n\t'\ + '"thread": null,\n\t'\ + '"message": %s,\n\t'\ + '"properties": {"application":"%s", "hostname":"%s"}\n'\ + '}' + + # cut longer msgs + MAXMSG = 1024 + + def makePickle(self, record): + """Create message in JSON format.""" + # get & cut msg + msg = self.format(record) + if len(msg) > self.MAXMSG: + msg = msg[:self.MAXMSG] + txt_level = self._level_map.get(record.levelno, "ERROR") + pkt = self._log_template % (time.time()*1000, txt_level, + quote_json(msg), record.name, socket.gethostname()) + return pkt + +class LogDBHandler(logging.handlers.SocketHandler): + """Sends log records into PostgreSQL server. + + Additionally, does some statistics aggregating, + to avoid overloading log server. + + It subclasses SocketHandler to get throtthling for + failed connections. + """ + + # map codes to string + _level_map = { + logging.DEBUG : 'DEBUG', + logging.INFO : 'INFO', + logging.WARNING : 'WARNING', + logging.ERROR : 'ERROR', + logging.CRITICAL: 'FATAL', + } + + def __init__(self, connect_string): + """ + Initializes the handler with a specific connection string. + """ + + logging.handlers.SocketHandler.__init__(self, None, None) + self.closeOnError = 1 + + self.connect_string = connect_string + + self.stat_cache = {} + self.stat_flush_period = 60 + # send first stat line immidiately + self.last_stat_flush = 0 + + def createSocket(self): + try: + logging.handlers.SocketHandler.createSocket(self) + except: + self.sock = self.makeSocket() + + def makeSocket(self): + """Create server connection. + In this case its not socket but psycopg conection.""" + + db = psycopg.connect(self.connect_string) + db.autocommit(1) + return db + + def emit(self, record): + """Process log record.""" + + # we do not want log debug messages + if record.levelno < logging.INFO: + return + + try: + self.process_rec(record) + except (SystemExit, KeyboardInterrupt): + raise + except: + self.handleError(record) + + def process_rec(self, record): + """Aggregate stats if needed, and send to logdb.""" + # render msg + msg = self.format(record) + + # dont want to send stats too ofter + if record.levelno == logging.INFO and msg and msg[0] == "{": + self.aggregate_stats(msg) + if time.time() - self.last_stat_flush >= self.stat_flush_period: + self.flush_stats(record.name) + return + + if record.levelno < logging.INFO: + self.flush_stats(record.name) + + # dont send more than one line + ln = msg.find('\n') + if ln > 0: + msg = msg[:ln] + + txt_level = self._level_map.get(record.levelno, "ERROR") + self.send_to_logdb(record.name, txt_level, msg) + + def aggregate_stats(self, msg): + """Sum stats together, to lessen load on logdb.""" + + msg = msg[1:-1] + for rec in msg.split(", "): + k, v = rec.split(": ") + agg = self.stat_cache.get(k, 0) + if v.find('.') >= 0: + agg += float(v) + else: + agg += int(v) + self.stat_cache[k] = agg + + def flush_stats(self, service): + """Send awuired stats to logdb.""" + res = [] + for k, v in self.stat_cache.items(): + res.append("%s: %s" % (k, str(v))) + if len(res) > 0: + logmsg = "{%s}" % ", ".join(res) + self.send_to_logdb(service, "INFO", logmsg) + self.stat_cache = {} + self.last_stat_flush = time.time() + + def send_to_logdb(self, service, type, msg): + """Actual sending is done here.""" + + if self.sock is None: + self.createSocket() + + if self.sock: + logcur = self.sock.cursor() + query = "select * from log.add(%s, %s, %s)" + logcur.execute(query, [type, service, msg]) + diff --git a/python/skytools/sqltools.py b/python/skytools/sqltools.py new file mode 100644 index 00000000..75e209f1 --- /dev/null +++ b/python/skytools/sqltools.py @@ -0,0 +1,398 @@ + +"""Database tools.""" + +import os +from cStringIO import StringIO +from quoting import quote_copy, quote_literal + +# +# Fully qualified table name +# + +def fq_name_parts(tbl): + "Return fully qualified name parts." + + tmp = tbl.split('.') + if len(tmp) == 1: + return ('public', tbl) + elif len(tmp) == 2: + return tmp + else: + raise Exception('Syntax error in table name:'+tbl) + +def fq_name(tbl): + "Return fully qualified name." + return '.'.join(fq_name_parts(tbl)) + +# +# info about table +# +def get_table_oid(curs, table_name): + schema, name = fq_name_parts(table_name) + q = """select c.oid from pg_namespace n, pg_class c + where c.relnamespace = n.oid + and n.nspname = %s and c.relname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchall() + if len(res) == 0: + raise Exception('Table not found: '+table_name) + return res[0][0] + +def get_table_pkeys(curs, tbl): + oid = get_table_oid(curs, tbl) + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = %s AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + curs.execute(q, [oid]) + return map(lambda x: x[0], curs.fetchall()) + +def get_table_columns(curs, tbl): + oid = get_table_oid(curs, tbl) + q = "SELECT k.attname FROM pg_attribute k"\ + " WHERE k.attrelid = %s"\ + " AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + curs.execute(q, [oid]) + return map(lambda x: x[0], curs.fetchall()) + +# +# exist checks +# +def exists_schema(curs, schema): + q = "select count(1) from pg_namespace where nspname = %s" + curs.execute(q, [schema]) + res = curs.fetchone() + return res[0] + +def exists_table(curs, table_name): + schema, name = fq_name_parts(table_name) + q = """select count(1) from pg_namespace n, pg_class c + where c.relnamespace = n.oid and c.relkind = 'r' + and n.nspname = %s and c.relname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchone() + return res[0] + +def exists_type(curs, type_name): + schema, name = fq_name_parts(type_name) + q = """select count(1) from pg_namespace n, pg_type t + where t.typnamespace = n.oid + and n.nspname = %s and t.typname = %s""" + curs.execute(q, [schema, name]) + res = curs.fetchone() + return res[0] + +def exists_function(curs, function_name, nargs): + # this does not check arg types, so may match several functions + schema, name = fq_name_parts(function_name) + q = """select count(1) from pg_namespace n, pg_proc p + where p.pronamespace = n.oid and p.pronargs = %s + and n.nspname = %s and p.proname = %s""" + curs.execute(q, [nargs, schema, name]) + res = curs.fetchone() + return res[0] + +def exists_language(curs, lang_name): + q = """select count(1) from pg_language + where lanname = %s""" + curs.execute(q, [lang_name]) + res = curs.fetchone() + return res[0] + +# +# Support for PostgreSQL snapshot +# + +class Snapshot(object): + "Represents a PostgreSQL snapshot." + + def __init__(self, str): + "Create snapshot from string." + + self.sn_str = str + tmp = str.split(':') + if len(tmp) != 3: + raise Exception('Unknown format for snapshot') + self.xmin = int(tmp[0]) + self.xmax = int(tmp[1]) + self.txid_list = [] + if tmp[2] != "": + for s in tmp[2].split(','): + self.txid_list.append(int(s)) + + def contains(self, txid): + "Is txid visible in snapshot." + + txid = int(txid) + + if txid < self.xmin: + return True + if txid >= self.xmax: + return False + if txid in self.txid_list: + return False + return True + +# +# Copy helpers +# + +def _gen_dict_copy(tbl, row, fields): + tmp = [] + for f in fields: + v = row[f] + tmp.append(quote_copy(v)) + return "\t".join(tmp) + +def _gen_dict_insert(tbl, row, fields): + tmp = [] + for f in fields: + v = row[f] + tmp.append(quote_literal(v)) + fmt = "insert into %s (%s) values (%s);" + return fmt % (tbl, ",".join(fields), ",".join(tmp)) + +def _gen_list_copy(tbl, row, fields): + tmp = [] + for i in range(len(fields)): + v = row[i] + tmp.append(quote_copy(v)) + return "\t".join(tmp) + +def _gen_list_insert(tbl, row, fields): + tmp = [] + for i in range(len(fields)): + v = row[i] + tmp.append(quote_literal(v)) + fmt = "insert into %s (%s) values (%s);" + return fmt % (tbl, ",".join(fields), ",".join(tmp)) + +def magic_insert(curs, tablename, data, fields = None, use_insert = 0): + """Copy/insert a list of dict/list data to database. + + If curs == None, then the copy or insert statements are returned + as string. For list of dict the field list is optional, as its + possible to guess them from dict keys. + """ + if len(data) == 0: + return + + # decide how to process + if type(data[0]) == type({}): + if fields == None: + fields = data[0].keys() + if use_insert: + row_func = _gen_dict_insert + else: + row_func = _gen_dict_copy + else: + if fields == None: + raise Exception("Non-dict data needs field list") + if use_insert: + row_func = _gen_list_insert + else: + row_func = _gen_list_copy + + # init processing + buf = StringIO() + if curs == None and use_insert == 0: + fmt = "COPY %s (%s) FROM STDIN;\n" + buf.write(fmt % (tablename, ",".join(fields))) + + # process data + for row in data: + buf.write(row_func(tablename, row, fields)) + buf.write("\n") + + # if user needs only string, return it + if curs == None: + if use_insert == 0: + buf.write("\\.\n") + return buf.getvalue() + + # do the actual copy/inserts + if use_insert: + curs.execute(buf.getvalue()) + else: + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + curs.copy_from(buf, hdr) + +def db_copy_from_dict(curs, tablename, dict_list, fields = None): + """Do a COPY FROM STDIN using list of dicts as source.""" + + if len(dict_list) == 0: + return + + if fields == None: + fields = dict_list[0].keys() + + buf = StringIO() + for dat in dict_list: + row = [] + for k in fields: + row.append(quote_copy(dat[k])) + buf.write("\t".join(row)) + buf.write("\n") + + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + + curs.copy_from(buf, hdr) + +def db_copy_from_list(curs, tablename, row_list, fields): + """Do a COPY FROM STDIN using list of lists as source.""" + + if len(row_list) == 0: + return + + if fields == None or len(fields) == 0: + raise Exception('Need field list') + + buf = StringIO() + for dat in row_list: + row = [] + for i in range(len(fields)): + row.append(quote_copy(dat[i])) + buf.write("\t".join(row)) + buf.write("\n") + + buf.seek(0) + hdr = "%s (%s)" % (tablename, ",".join(fields)) + + curs.copy_from(buf, hdr) + +# +# Full COPY of table from one db to another +# + +class CopyPipe(object): + "Splits one big COPY to chunks." + + def __init__(self, dstcurs, tablename, limit = 512*1024, cancel_func=None): + self.tablename = tablename + self.dstcurs = dstcurs + self.buf = StringIO() + self.limit = limit + self.cancel_func = None + self.total_rows = 0 + self.total_bytes = 0 + + def write(self, data): + "New data from psycopg" + + self.total_bytes += len(data) + self.total_rows += data.count("\n") + + if self.buf.tell() >= self.limit: + pos = data.find('\n') + if pos >= 0: + # split at newline + p1 = data[:pos + 1] + p2 = data[pos + 1:] + self.buf.write(p1) + self.flush() + + data = p2 + + self.buf.write(data) + + def flush(self): + "Send data out." + + if self.cancel_func: + self.cancel_func() + + if self.buf.tell() > 0: + self.buf.seek(0) + self.dstcurs.copy_from(self.buf, self.tablename) + self.buf.seek(0) + self.buf.truncate() + +def full_copy(tablename, src_curs, dst_curs, column_list = []): + """COPY table from one db to another.""" + + if column_list: + hdr = "%s (%s)" % (tablename, ",".join(column_list)) + else: + hdr = tablename + buf = CopyPipe(dst_curs, hdr) + src_curs.copy_to(buf, hdr) + buf.flush() + + return (buf.total_bytes, buf.total_rows) + + +# +# SQL installer +# + +class DBObject(object): + """Base class for installable DB objects.""" + name = None + sql = None + sql_file = None + def __init__(self, name, sql = None, sql_file = None): + self.name = name + self.sql = sql + self.sql_file = sql_file + def get_sql(self): + if self.sql: + return self.sql + if self.sql_file: + if self.sql_file[0] == "/": + fn = self.sql_file + else: + contrib_list = [ + "/opt/pgsql/share/contrib", + "/usr/share/postgresql/8.0/contrib", + "/usr/share/postgresql/8.0/contrib", + "/usr/share/postgresql/8.1/contrib", + "/usr/share/postgresql/8.2/contrib", + ] + for dir in contrib_list: + fn = os.path.join(dir, self.sql_file) + if os.path.isfile(fn): + return open(fn, "r").read() + raise Exception('File not found: '+self.sql_file) + raise Exception('object not defined') + def create(self, curs): + curs.execute(self.get_sql()) + +class DBSchema(DBObject): + """Handles db schema.""" + def exists(self, curs): + return exists_schema(curs, self.name) + +class DBTable(DBObject): + """Handles db table.""" + def exists(self, curs): + return exists_table(curs, self.name) + +class DBFunction(DBObject): + """Handles db function.""" + def __init__(self, name, nargs, sql = None, sql_file = None): + DBObject.__init__(self, name, sql, sql_file) + self.nargs = nargs + def exists(self, curs): + return exists_function(curs, self.name, self.nargs) + +class DBLanguage(DBObject): + """Handles db language.""" + def __init__(self, name): + DBObject.__init__(self, name, sql = "create language %s" % name) + def exists(self, curs): + return exists_language(curs, self.name) + +def db_install(curs, list, log = None): + """Installs list of objects into db.""" + for obj in list: + if not obj.exists(curs): + if log: + log.info('Installing %s' % obj.name) + obj.create(curs) + else: + if log: + log.info('%s is installed' % obj.name) + diff --git a/python/walmgr.py b/python/walmgr.py new file mode 100755 index 00000000..8f43fd6d --- /dev/null +++ b/python/walmgr.py @@ -0,0 +1,648 @@ +#! /usr/bin/env python + +"""WALShipping manager. + +walmgr [-n] COMMAND + +Master commands: + setup Configure PostgreSQL for WAL archiving + backup Copies all master data to slave + sync Copies in-progress WALs to slave + syncdaemon Daemon mode for regular syncing + stop Stop archiving - de-configure PostgreSQL + +Slave commands: + restore Stop postmaster, move new data dir to right + location and start postmaster in playback mode. + boot Stop playback, accept queries. + pause Just wait, don't play WAL-s + continue Start playing WAL-s again + +Internal commands: + xarchive archive one WAL file (master) + xrestore restore one WAL file (slave) + +Switches: + -n no action, just print commands +""" + +import os, sys, skytools, getopt, re, signal, time, traceback + +MASTER = 1 +SLAVE = 0 + +def usage(err): + if err > 0: + print >>sys.stderr, __doc__ + else: + print __doc__ + sys.exit(err) + +class WalMgr(skytools.DBScript): + def __init__(self, wtype, cf_file, not_really, internal = 0, go_daemon = 0): + self.not_really = not_really + self.pg_backup = 0 + + if wtype == MASTER: + service_name = "wal-master" + else: + service_name = "wal-slave" + + if not os.path.isfile(cf_file): + print "Config not found:", cf_file + sys.exit(1) + + if go_daemon: + s_args = ["-d", cf_file] + else: + s_args = [cf_file] + + skytools.DBScript.__init__(self, service_name, s_args, + force_logfile = internal) + + def pg_start_backup(self, code): + q = "select pg_start_backup('FullBackup')" + self.log.info("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db"))) + if self.not_really: + self.pg_backup = 1 + return + db = self.get_database("master_db") + db.cursor().execute(q) + db.commit() + self.close_database("master_db") + self.pg_backup = 1 + + def pg_stop_backup(self): + if not self.pg_backup: + return + + q = "select pg_stop_backup()" + self.log.debug("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db"))) + if self.not_really: + return + db = self.get_database("master_db") + db.cursor().execute(q) + db.commit() + self.close_database("master_db") + + def signal_postmaster(self, data_dir, sgn): + pidfile = os.path.join(data_dir, "postmaster.pid") + if not os.path.isfile(pidfile): + self.log.info("postmaster is not running") + return + buf = open(pidfile, "r").readline() + pid = int(buf.strip()) + self.log.debug("Signal %d to process %d" % (sgn, pid)) + if not self.not_really: + os.kill(pid, sgn) + + def exec_big_rsync(self, cmdline): + cmd = "' '".join(cmdline) + self.log.debug("Execute big rsync cmd: '%s'" % (cmd)) + if self.not_really: + return + res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) + if res == 24: + self.log.info("Some files vanished, but thats OK") + elif res != 0: + self.log.fatal("exec failed, res=%d" % res) + self.pg_stop_backup() + sys.exit(1) + + def exec_cmd(self, cmdline): + cmd = "' '".join(cmdline) + self.log.debug("Execute cmd: '%s'" % (cmd)) + if self.not_really: + return + res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline) + if res != 0: + self.log.fatal("exec failed, res=%d" % res) + sys.exit(1) + + def chdir(self, loc): + self.log.debug("chdir: '%s'" % (loc)) + if self.not_really: + return + try: + os.chdir(loc) + except os.error: + self.log.fatal("CHDir failed") + self.pg_stop_backup() + sys.exit(1) + + def get_last_complete(self): + """Get the name of last xarchived segment.""" + + data_dir = self.cf.get("master_data") + fn = os.path.join(data_dir, ".walshipping.last") + try: + last = open(fn, "r").read().strip() + return last + except: + self.log.info("Failed to read %s" % fn) + return None + + def set_last_complete(self, last): + """Set the name of last xarchived segment.""" + + data_dir = self.cf.get("master_data") + fn = os.path.join(data_dir, ".walshipping.last") + fn_tmp = fn + ".new" + try: + f = open(fn_tmp, "w") + f.write(last) + f.close() + os.rename(fn_tmp, fn) + except: + self.log.fatal("Cannot write to %s" % fn) + + def master_setup(self): + self.log.info("Configuring WAL archiving") + + script = os.path.abspath(sys.argv[0]) + cf_file = os.path.abspath(self.cf.filename) + cf_val = "%s %s %s" % (script, cf_file, "xarchive %p %f") + + self.master_configure_archiving(cf_val) + + def master_stop(self): + self.log.info("Disabling WAL archiving") + + self.master_configure_archiving('') + + def master_configure_archiving(self, cf_val): + cf_file = self.cf.get("master_config") + data_dir = self.cf.get("master_data") + r_active = re.compile("^[ ]*archive_command[ ]*=[ ]*'(.*)'.*$", re.M) + r_disabled = re.compile("^.*archive_command.*$", re.M) + + cf_full = "archive_command = '%s'" % cf_val + + if not os.path.isfile(cf_file): + self.log.fatal("Config file not found: %s" % cf_file) + self.log.info("Using config file: %s", cf_file) + + buf = open(cf_file, "r").read() + m = r_active.search(buf) + if m: + old_val = m.group(1) + if old_val == cf_val: + self.log.debug("postmaster already configured") + else: + self.log.debug("found active but different conf") + newbuf = "%s%s%s" % (buf[:m.start()], cf_full, buf[m.end():]) + self.change_config(cf_file, newbuf) + else: + m = r_disabled.search(buf) + if m: + self.log.debug("found disabled value") + newbuf = "%s\n%s%s" % (buf[:m.end()], cf_full, buf[m.end():]) + self.change_config(cf_file, newbuf) + else: + self.log.debug("found no value") + newbuf = "%s\n%s\n\n" % (buf, cf_full) + self.change_config(cf_file, newbuf) + + self.log.info("Sending SIGHUP to postmaster") + self.signal_postmaster(data_dir, signal.SIGHUP) + self.log.info("Done") + + def change_config(self, cf_file, buf): + cf_old = cf_file + ".old" + cf_new = cf_file + ".new" + + if self.not_really: + cf_new = "/tmp/postgresql.conf.new" + open(cf_new, "w").write(buf) + self.log.info("Showing diff") + os.system("diff -u %s %s" % (cf_file, cf_new)) + self.log.info("Done diff") + os.remove(cf_new) + return + + # polite method does not work, as usually not enough perms for it + if 0: + open(cf_new, "w").write(buf) + bak = open(cf_file, "r").read() + open(cf_old, "w").write(bak) + os.rename(cf_new, cf_file) + else: + open(cf_file, "w").write(buf) + + def remote_mkdir(self, remdir): + tmp = remdir.split(":", 1) + if len(tmp) != 2: + raise Exception("cannot find hostname") + host, path = tmp + cmdline = ["ssh", host, "mkdir", "-p", path] + self.exec_cmd(cmdline) + + def master_backup(self): + """Copy master data directory to slave.""" + + data_dir = self.cf.get("master_data") + dst_loc = self.cf.get("full_backup") + if dst_loc[-1] != "/": + dst_loc += "/" + + self.pg_start_backup("FullBackup") + + master_spc_dir = os.path.join(data_dir, "pg_tblspc") + slave_spc_dir = dst_loc + "tmpspc" + + # copy data + self.chdir(data_dir) + cmdline = ["rsync", "-a", "--delete", + "--exclude", ".*", + "--exclude", "*.pid", + "--exclude", "*.opts", + "--exclude", "*.conf", + "--exclude", "*.conf.*", + "--exclude", "pg_xlog", + ".", dst_loc] + self.exec_big_rsync(cmdline) + + # copy tblspc first, to test + if os.path.isdir(master_spc_dir): + self.log.info("Checking tablespaces") + list = os.listdir(master_spc_dir) + if len(list) > 0: + self.remote_mkdir(slave_spc_dir) + for tblspc in list: + if tblspc[0] == ".": + continue + tfn = os.path.join(master_spc_dir, tblspc) + if not os.path.islink(tfn): + self.log.info("Suspicious pg_tblspc entry: "+tblspc) + continue + spc_path = os.path.realpath(tfn) + self.log.info("Got tablespace %s: %s" % (tblspc, spc_path)) + dstfn = slave_spc_dir + "/" + tblspc + + try: + os.chdir(spc_path) + except Exception, det: + self.log.warning("Broken link:" + str(det)) + continue + cmdline = ["rsync", "-a", "--delete", + "--exclude", ".*", + ".", dstfn] + self.exec_big_rsync(cmdline) + + # copy pg_xlog + self.chdir(data_dir) + cmdline = ["rsync", "-a", + "--exclude", "*.done", + "--exclude", "*.backup", + "--delete", "pg_xlog", dst_loc] + self.exec_big_rsync(cmdline) + + self.pg_stop_backup() + + self.log.info("Full backup successful") + + def master_xarchive(self, srcpath, srcname): + """Copy a complete WAL segment to slave.""" + + start_time = time.time() + self.log.debug("%s: start copy", srcname) + + self.set_last_complete(srcname) + + dst_loc = self.cf.get("completed_wals") + if dst_loc[-1] != "/": + dst_loc += "/" + + # copy data + cmdline = ["rsync", "-t", srcpath, dst_loc] + self.exec_cmd(cmdline) + + self.log.debug("%s: done", srcname) + end_time = time.time() + self.stat_add('count', 1) + self.stat_add('duration', end_time - start_time) + + def master_sync(self): + """Copy partial WAL segments.""" + + data_dir = self.cf.get("master_data") + xlog_dir = os.path.join(data_dir, "pg_xlog") + dst_loc = self.cf.get("partial_wals") + if dst_loc[-1] != "/": + dst_loc += "/" + + files = os.listdir(xlog_dir) + files.sort() + + last = self.get_last_complete() + if last: + self.log.info("%s: last complete" % last) + else: + self.log.info("last complete not found, copying all") + + for fn in files: + # check if interesting file + if len(fn) < 10: + continue + if fn[0] < "0" or fn[0] > '9': + continue + if fn.find(".") > 0: + continue + # check if to old + if last: + dot = last.find(".") + if dot > 0: + xlast = last[:dot] + if fn < xlast: + continue + else: + if fn <= last: + continue + + # got interesting WAL + xlog = os.path.join(xlog_dir, fn) + # copy data + cmdline = ["rsync", "-t", xlog, dst_loc] + self.exec_cmd(cmdline) + + self.log.info("Partial copy done") + + def slave_xrestore(self, srcname, dstpath): + loop = 1 + while loop: + try: + self.slave_xrestore_unsafe(srcname, dstpath) + loop = 0 + except SystemExit, d: + sys.exit(1) + except Exception, d: + exc, msg, tb = sys.exc_info() + self.log.fatal("xrestore %s crashed: %s: '%s' (%s: %s)" % ( + srcname, str(exc), str(msg).rstrip(), + str(tb), repr(traceback.format_tb(tb)))) + time.sleep(10) + self.log.info("Re-exec: %s", repr(sys.argv)) + os.execv(sys.argv[0], sys.argv) + + def slave_xrestore_unsafe(self, srcname, dstpath): + srcdir = self.cf.get("completed_wals") + partdir = self.cf.get("partial_wals") + keep_old_logs = self.cf.getint("keep_old_logs", 0) + pausefile = os.path.join(srcdir, "PAUSE") + stopfile = os.path.join(srcdir, "STOP") + srcfile = os.path.join(srcdir, srcname) + partfile = os.path.join(partdir, srcname) + + # loop until srcfile or stopfile appears + while 1: + if os.path.isfile(pausefile): + self.log.info("pause requested, sleeping") + time.sleep(20) + continue + + if os.path.isfile(srcfile): + self.log.info("%s: Found" % srcname) + break + + # ignore .history files + unused, ext = os.path.splitext(srcname) + if ext == ".history": + self.log.info("%s: not found, ignoring" % srcname) + sys.exit(1) + + # if stopping, include also partial wals + if os.path.isfile(stopfile): + if os.path.isfile(partfile): + self.log.info("%s: found partial" % srcname) + srcfile = partfile + break + else: + self.log.info("%s: not found, stopping" % srcname) + sys.exit(1) + + # nothing to do, sleep + self.log.debug("%s: not found, sleeping" % srcname) + time.sleep(20) + + # got one, copy it + cmdline = ["cp", srcfile, dstpath] + self.exec_cmd(cmdline) + + self.log.debug("%s: copy done, cleanup" % srcname) + self.slave_cleanup(srcname) + + # it would be nice to have apply time too + self.stat_add('count', 1) + + def slave_startup(self): + data_dir = self.cf.get("slave_data") + full_dir = self.cf.get("full_backup") + stop_cmd = self.cf.get("slave_stop_cmd", "") + start_cmd = self.cf.get("slave_start_cmd") + pidfile = os.path.join(data_dir, "postmaster.pid") + + # stop postmaster if ordered + if stop_cmd and os.path.isfile(pidfile): + self.log.info("Stopping postmaster: " + stop_cmd) + if not self.not_really: + os.system(stop_cmd) + time.sleep(3) + + # is it dead? + if os.path.isfile(pidfile): + self.log.fatal("Postmaster still running. Cannot continue.") + sys.exit(1) + + # find name for data backup + i = 0 + while 1: + bak = "%s.%d" % (data_dir, i) + if not os.path.isdir(bak): + break + i += 1 + + # move old data away + if os.path.isdir(data_dir): + self.log.info("Move %s to %s" % (data_dir, bak)) + if not self.not_really: + os.rename(data_dir, bak) + + # move new data + self.log.info("Move %s to %s" % (full_dir, data_dir)) + if not self.not_really: + os.rename(full_dir, data_dir) + else: + data_dir = full_dir + + # re-link tablespaces + spc_dir = os.path.join(data_dir, "pg_tblspc") + tmp_dir = os.path.join(data_dir, "tmpspc") + if os.path.isdir(spc_dir) and os.path.isdir(tmp_dir): + self.log.info("Linking tablespaces to temporary location") + + # don't look into spc_dir, thus allowing + # user to move them before. re-link only those + # that are still in tmp_dir + list = os.listdir(tmp_dir) + list.sort() + + for d in list: + if d[0] == ".": + continue + link_loc = os.path.join(spc_dir, d) + link_dst = os.path.join(tmp_dir, d) + self.log.info("Linking tablespace %s to %s" % (d, link_dst)) + if not self.not_really: + if os.path.islink(link_loc): + os.remove(link_loc) + os.symlink(link_dst, link_loc) + + # write recovery.conf + rconf = os.path.join(data_dir, "recovery.conf") + script = os.path.abspath(sys.argv[0]) + cf_file = os.path.abspath(self.cf.filename) + conf = "\nrestore_command = '%s %s %s'\n" % ( + script, cf_file, 'xrestore %f "%p"') + self.log.info("Write %s" % rconf) + if self.not_really: + print conf + else: + f = open(rconf, "w") + f.write(conf) + f.close() + + # remove stopfile + srcdir = self.cf.get("completed_wals") + stopfile = os.path.join(srcdir, "STOP") + if os.path.isfile(stopfile): + self.log.info("Removing stopfile: "+stopfile) + if not self.not_really: + os.remove(stopfile) + + # run database in recovery mode + self.log.info("Starting postmaster: " + start_cmd) + if not self.not_really: + os.system(start_cmd) + + def slave_boot(self): + srcdir = self.cf.get("completed_wals") + stopfile = os.path.join(srcdir, "STOP") + open(stopfile, "w").write("1") + self.log.info("Stopping recovery mode") + + def slave_pause(self): + srcdir = self.cf.get("completed_wals") + pausefile = os.path.join(srcdir, "PAUSE") + open(pausefile, "w").write("1") + self.log.info("Pausing recovery mode") + + def slave_continue(self): + srcdir = self.cf.get("completed_wals") + pausefile = os.path.join(srcdir, "PAUSE") + if os.path.isfile(pausefile): + os.remove(pausefile) + self.log.info("Continuing with recovery") + else: + self.log.info("Recovery not paused?") + + def slave_cleanup(self, last_applied): + completed_wals = self.cf.get("completed_wals") + partial_wals = self.cf.get("partial_wals") + + self.log.debug("cleaning completed wals since %s" % last_applied) + last = self.del_wals(completed_wals, last_applied) + if last: + if os.path.isdir(partial_wals): + self.log.debug("cleaning partial wals since %s" % last) + self.del_wals(partial_wals, last) + else: + self.log.warning("partial_wals dir does not exist: %s" + % partial_wals) + self.log.debug("cleaning done") + + def del_wals(self, path, last): + dot = last.find(".") + if dot > 0: + last = last[:dot] + list = os.listdir(path) + list.sort() + cur_last = None + n = len(list) + for i in range(n): + fname = list[i] + full = os.path.join(path, fname) + if fname[0] < "0" or fname[0] > "9": + continue + + ok_del = 0 + if fname < last: + self.log.debug("deleting %s" % full) + os.remove(full) + cur_last = fname + return cur_last + + def work(self): + self.master_sync() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "nh") + except getopt.error, det: + print det + usage(1) + not_really = 0 + for o, v in opts: + if o == "-n": + not_really = 1 + elif o == "-h": + usage(0) + if len(args) < 2: + usage(1) + ini = args[0] + cmd = args[1] + + if cmd == "setup": + script = WalMgr(MASTER, ini, not_really) + script.master_setup() + elif cmd == "stop": + script = WalMgr(MASTER, ini, not_really) + script.master_stop() + elif cmd == "backup": + script = WalMgr(MASTER, ini, not_really) + script.master_backup() + elif cmd == "xarchive": + if len(args) != 4: + print >> sys.stderr, "usage: walmgr INI xarchive %p %f" + sys.exit(1) + script = WalMgr(MASTER, ini, not_really, 1) + script.master_xarchive(args[2], args[3]) + elif cmd == "sync": + script = WalMgr(MASTER, ini, not_really) + script.master_sync() + elif cmd == "syncdaemon": + script = WalMgr(MASTER, ini, not_really, go_daemon=1) + script.start() + elif cmd == "xrestore": + if len(args) != 4: + print >> sys.stderr, "usage: walmgr INI xrestore %p %f" + sys.exit(1) + script = WalMgr(SLAVE, ini, not_really, 1) + script.slave_xrestore(args[2], args[3]) + elif cmd == "restore": + script = WalMgr(SLAVE, ini, not_really) + script.slave_startup() + elif cmd == "boot": + script = WalMgr(SLAVE, ini, not_really) + script.slave_boot() + elif cmd == "pause": + script = WalMgr(SLAVE, ini, not_really) + script.slave_pause() + elif cmd == "continue": + script = WalMgr(SLAVE, ini, not_really) + script.slave_continue() + else: + usage(1) + +if __name__ == '__main__': + main() + diff --git a/scripts/bulk_loader.ini.templ b/scripts/bulk_loader.ini.templ new file mode 100644 index 00000000..187c1be2 --- /dev/null +++ b/scripts/bulk_loader.ini.templ @@ -0,0 +1,13 @@ +[bulk_loader] +job_name = bizgres_loader + +src_db = dbname=bulksrc +dst_db = dbname=bulkdst + +pgq_queue_name = xx + +use_skylog = 1 + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + diff --git a/scripts/bulk_loader.py b/scripts/bulk_loader.py new file mode 100755 index 00000000..a098787e --- /dev/null +++ b/scripts/bulk_loader.py @@ -0,0 +1,181 @@ +#! /usr/bin/env python + +"""Bulkloader for slow databases (Bizgres). + +Idea is following: + - Script reads from queue a batch of urlencoded row changes. + Inserts/updates/deletes, maybe many per one row. + - It changes them to minimal amount of DELETE commands + followed by big COPY of new data. + - One side-effect is that total order of how rows appear + changes, but per-row changes will be kept in order. + +The speedup from the COPY will happen only if the batches are +large enough. So the ticks should happen only after couple +of minutes. + +""" + +import sys, os, pgq, skytools + +def mk_delete_sql(tbl, key_list, data): + """ generate delete command """ + whe_list = [] + for k in key_list: + whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k]))) + whe_str = " and ".join(whe_list) + return "delete from %s where %s;" % (tbl, whe_str) + +class TableCache(object): + """Per-table data hander.""" + + def __init__(self, tbl): + """Init per-batch table data cache.""" + self.name = tbl + self.ev_list = [] + self.pkey_map = {} + self.pkey_list = [] + self.pkey_str = None + self.col_list = None + + def add_event(self, ev): + """Store new event.""" + + # op & data + ev.op = ev.type[0] + ev.row = skytools.db_urldecode(ev.data) + + # get pkey column names + if self.pkey_str is None: + self.pkey_str = ev.type.split(':')[1] + if self.pkey_str: + self.pkey_list = self.pkey_str.split(',') + + # get pkey value + if self.pkey_str: + pk_data = [] + for k in self.pkey_list: + pk_data.append(ev.row[k]) + ev.pk_data = tuple(pk_data) + elif ev.op == 'I': + # fake pkey, just to get them spread out + ev.pk_data = ev.id + else: + raise Exception('non-pk tables not supported: %s' % ev.extra1) + + # get full column list, detect added columns + if not self.col_list: + self.col_list = ev.row.keys() + elif self.col_list != ev.row.keys(): + # ^ supposedly python guarantees same order in keys() + + # find new columns + for c in ev.row.keys(): + if c not in self.col_list: + for oldev in self.ev_list: + oldev.row[c] = None + self.col_list = ev.row.keys() + + # add to list + self.ev_list.append(ev) + + # keep all versions of row data + if ev.pk_data in self.pkey_map: + self.pkey_map[ev.pk_data].append(ev) + else: + self.pkey_map[ev.pk_data] = [ev] + + def finish(self): + """Got all data, prepare for insertion.""" + + del_list = [] + copy_list = [] + for ev_list in self.pkey_map.values(): + # rewrite list of I/U/D events to + # optional DELETE and optional INSERT/COPY command + exists_before = -1 + exists_after = 1 + for ev in ev_list: + if ev.op == "I": + if exists_before < 0: + exists_before = 0 + exists_after = 1 + elif ev.op == "U": + if exists_before < 0: + exists_before = 1 + #exists_after = 1 # this shouldnt be needed + elif ev.op == "D": + if exists_before < 0: + exists_before = 1 + exists_after = 0 + else: + raise Exception('unknown event type: %s' % ev.op) + + # skip short-lived rows + if exists_before == 0 and exists_after == 0: + continue + + # take last event + ev = ev_list[-1] + + # generate needed commands + if exists_before: + del_list.append(mk_delete_sql(self.name, self.pkey_list, ev.row)) + if exists_after: + copy_list.append(ev.row) + + # reorder cols + new_list = self.pkey_list[:] + for k in self.col_list: + if k not in self.pkey_list: + new_list.append(k) + + return del_list, new_list, copy_list + +class BulkLoader(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "bulk_loader", "src_db", "dst_db", args) + + def process_remote_batch(self, batch_id, ev_list, dst_db): + """Content dispatcher.""" + + # add events to per-table caches + tables = {} + for ev in ev_list: + tbl = ev.extra1 + + if not tbl in tables: + tables[tbl] = TableCache(tbl) + cache = tables[tbl] + cache.add_event(ev) + ev.tag_done() + + # then process them + for tbl, cache in tables.items(): + self.process_one_table(dst_db, tbl, cache) + + def process_one_table(self, dst_db, tbl, cache): + self.log.debug("process_one_table: %s" % tbl) + del_list, col_list, copy_list = cache.finish() + curs = dst_db.cursor() + + if not skytools.exists_table(curs, tbl): + self.log.warning("Ignoring events for table: %s" % tbl) + return + + if len(del_list) > 0: + self.log.info("Deleting %d rows from %s" % (len(del_list), tbl)) + + q = " ".join(del_list) + self.log.debug(q) + curs.execute(q) + + if len(copy_list) > 0: + self.log.info("Copying %d rows into %s" % (len(copy_list), tbl)) + self.log.debug("COPY %s (%s)" % (tbl, ','.join(col_list))) + skytools.magic_insert(curs, tbl, copy_list, col_list) + +if __name__ == '__main__': + script = BulkLoader(sys.argv[1:]) + script.start() + diff --git a/scripts/catsql.py b/scripts/catsql.py new file mode 100755 index 00000000..94fbacd8 --- /dev/null +++ b/scripts/catsql.py @@ -0,0 +1,141 @@ +#! /usr/bin/env python + +"""Prints out SQL files with psql command execution. + +Supported psql commands: \i, \cd, \q +Others are skipped. + +Aditionally does some pre-processing for NDoc. +NDoc is looks nice but needs some hand-holding. + +Bug: + +- function def end detection searches for 'as'/'is' but does not check + word boundaries - finds them even in function name. That means in + main conf, as/is must be disabled and $ ' added. This script can + remove the unnecessary AS from output. + +Niceties: + +- Ndoc includes function def in output only if def is after comment. + But for SQL functions its better to have it after def. + This script can swap comment and def. + +- Optionally remove CREATE FUNCTION (OR REPLACE) from def to + keep it shorter in doc. + +Note: + +- NDoc compares real function name and name in comment. if differ, + it decides detection failed. + +""" + +import sys, os, re, getopt + +def usage(x): + print "usage: catsql [--ndoc] FILE [FILE ...]" + sys.exit(x) + +# NDoc specific changes +cf_ndoc = 0 + +# compile regexes +func_re = r"create\s+(or\s+replace\s+)?function\s+" +func_rc = re.compile(func_re, re.I) +comm_rc = re.compile(r"^\s*([#]\s*)?(?P--.*)", re.I) +end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I) +as_rc = re.compile(r"\s+as\s+", re.I) +cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I) + +# conversion func +def fix_func(ln): + # if ndoc, replace AS with ' ' + if cf_ndoc: + return as_rc.sub(' ', ln) + else: + return ln + +# got function def +def proc_func(f, ln): + # remove CREATE OR REPLACE + if cf_ndoc: + ln = func_rc.sub('', ln) + + ln = fix_func(ln) + pre_list = [ln] + comm_list = [] + n_comm = 0 + while 1: + ln = f.readline() + if not ln: + break + + com = None + if cf_ndoc: + com = comm_rc.search(ln) + if cf_ndoc and com: + pos = com.start('com') + comm_list.append(ln[pos:]) + elif end_rc.search(ln): + break + elif len(comm_list) > 0: + break + else: + pre_list.append(fix_func(ln)) + + if len(comm_list) > 2: + map(sys.stdout.write, comm_list) + map(sys.stdout.write, pre_list) + else: + map(sys.stdout.write, pre_list) + map(sys.stdout.write, comm_list) + if ln: + sys.stdout.write(fix_func(ln)) + +def cat_file(fn): + sys.stdout.write("\n") + f = open(fn) + while 1: + ln = f.readline() + if not ln: + break + m = cmd_rc.search(ln) + if m: + cmd = m.group(1) + if cmd == "i": # include a file + fn2 = m.group(2).strip() + cat_file(fn2) + elif cmd == "q": # quit + sys.exit(0) + elif cmd == "cd": # chdir + dir = m.group(2).strip() + os.chdir(dir) + else: # skip all others + pass + else: + if func_rc.search(ln): # function header + proc_func(f, ln) + else: # normal sql + sys.stdout.write(ln) + sys.stdout.write("\n") + +def main(): + global cf_ndoc + + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc']) + except getopt.error, d: + print d + usage(1) + for o, v in opts: + if o == "-h": + usage(0) + elif o == "--ndoc": + cf_ndoc = 1 + for fn in args: + cat_file(fn) + +if __name__ == '__main__': + main() + diff --git a/scripts/cube_dispatcher.ini.templ b/scripts/cube_dispatcher.ini.templ new file mode 100644 index 00000000..dea70697 --- /dev/null +++ b/scripts/cube_dispatcher.ini.templ @@ -0,0 +1,23 @@ +[cube_dispatcher] +job_name = some_queue_to_cube + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = udata.some_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# how many rows are kept: keep_latest, keep_all +mode = keep_latest + +# to_char() fmt for table suffix +#dateformat = YYYY_MM_DD +# following disables table suffixes: +#dateformat = + +part_template = + create table _DEST_TABLE (like _PARENT); + alter table only _DEST_TABLE add primary key (_PKEY); + diff --git a/scripts/cube_dispatcher.py b/scripts/cube_dispatcher.py new file mode 100755 index 00000000..d59ac300 --- /dev/null +++ b/scripts/cube_dispatcher.py @@ -0,0 +1,175 @@ +#! /usr/bin/env python + +# it accepts urlencoded rows for multiple tables from queue +# and insert them into actual tables, with partitioning on tick time + +import sys, os, pgq, skytools + +DEF_CREATE = """ +create table _DEST_TABLE (like _PARENT); +alter table only _DEST_TABLE add primary key (_PKEY); +""" + +class CubeDispatcher(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "cube_dispatcher", "src_db", "dst_db", args) + + self.dateformat = self.cf.get('dateformat', 'YYYY_MM_DD') + + self.part_template = self.cf.get('part_template', DEF_CREATE) + + mode = self.cf.get('mode', 'keep_latest') + if mode == 'keep_latest': + self.keep_latest = 1 + elif mode == 'keep_all': + self.keep_latest = 0 + else: + self.log.fatal('wrong mode setting') + sys.exit(1) + + def get_part_date(self, batch_id): + if not self.dateformat: + return None + + # fetch and format batch date + src_db = self.get_database('src_db') + curs = src_db.cursor() + q = 'select to_char(batch_end, %s) from pgq.get_batch_info(%s)' + curs.execute(q, [self.dateformat, batch_id]) + src_db.commit() + return curs.fetchone()[0] + + def process_remote_batch(self, batch_id, ev_list, dst_db): + + # actual processing + date_str = self.get_part_date(batch_id) + self.dispatch(dst_db, ev_list, self.get_part_date(batch_id)) + + # tag as done + for ev in ev_list: + ev.tag_done() + + def dispatch(self, dst_db, ev_list, date_str): + """Actual event processing.""" + + # get tables and sql + tables = {} + sql_list = [] + for ev in ev_list: + if date_str: + tbl = "%s_%s" % (ev.extra1, date_str) + else: + tbl = ev.extra1 + + if not tbl in tables: + tables[tbl] = self.get_table_info(ev, tbl) + + sql = self.make_sql(tbl, ev) + sql_list.append(sql) + + # create tables if needed + self.check_tables(dst_db, tables) + + # insert into data tables + curs = dst_db.cursor() + block = [] + for sql in sql_list: + self.log.debug(sql) + block.append(sql) + if len(block) > 100: + curs.execute("\n".join(block)) + block = [] + if len(block) > 0: + curs.execute("\n".join(block)) + + def get_table_info(self, ev, tbl): + inf = { + 'parent': ev.extra1, + 'table': tbl, + 'key_list': ev.type.split(':')[1] + } + return inf + + def make_sql(self, tbl, ev): + """Return SQL statement(s) for that event.""" + + # parse data + data = skytools.db_urldecode(ev.data) + + # parse tbl info + op, keys = ev.type.split(':') + key_list = keys.split(',') + if self.keep_latest and len(key_list) == 0: + raise Exception('No pkey on table %s' % tbl) + + # generate sql + if op in ('I', 'U'): + if self.keep_latest: + sql = "%s %s" % (self.mk_delete_sql(tbl, key_list, data), + self.mk_insert_sql(tbl, key_list, data)) + else: + sql = self.mk_insert_sql(tbl, key_list, data) + elif op == "D": + if not self.keep_latest: + raise Exception('Delete op not supported if mode=keep_all') + + sql = self.mk_delete_sql(tbl, key_list, data) + else: + raise Exception('Unknown row op: %s' % op) + return sql + + def mk_delete_sql(self, tbl, key_list, data): + # generate delete command + whe_list = [] + for k in key_list: + whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k]))) + whe_str = " and ".join(whe_list) + return "delete from %s where %s;" % (tbl, whe_str) + + def mk_insert_sql(self, tbl, key_list, data): + # generate insert command + col_list = [] + val_list = [] + for c, v in data.items(): + col_list.append(c) + val_list.append(skytools.quote_literal(v)) + col_str = ",".join(col_list) + val_str = ",".join(val_list) + return "insert into %s (%s) values (%s);" % ( + tbl, col_str, val_str) + + def check_tables(self, dcon, tables): + """Checks that tables needed for copy are there. If not + then creates them. + + Used by other procedures to ensure that table is there + before they start inserting. + + The commits should not be dangerous, as we haven't done anything + with cdr's yet, so they should still be in one TX. + + Although it would be nicer to have a lock for table creation. + """ + + dcur = dcon.cursor() + exist_map = {} + for tbl, inf in tables.items(): + if skytools.exists_table(dcur, tbl): + continue + + sql = self.part_template + sql = sql.replace('_DEST_TABLE', inf['table']) + sql = sql.replace('_PARENT', inf['parent']) + sql = sql.replace('_PKEY', inf['key_list']) + # be similar to table_dispatcher + schema_table = inf['table'].replace(".", "__") + sql = sql.replace('_SCHEMA_TABLE', schema_table) + + dcur.execute(sql) + dcon.commit() + self.log.info('%s: Created table %s' % (self.job_name, tbl)) + +if __name__ == '__main__': + script = CubeDispatcher(sys.argv[1:]) + script.start() + diff --git a/scripts/queue_mover.ini.templ b/scripts/queue_mover.ini.templ new file mode 100644 index 00000000..8d1ff5f1 --- /dev/null +++ b/scripts/queue_mover.ini.templ @@ -0,0 +1,14 @@ +[queue_mover] +job_name = queue_mover_test + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = source_queue +dst_queue_name = dest_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/scripts/queue_mover.py b/scripts/queue_mover.py new file mode 100755 index 00000000..129728a3 --- /dev/null +++ b/scripts/queue_mover.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python + +# this script simply mover events from one queue to another + +import sys, os, pgq, skytools + +class QueueMover(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "queue_mover", "src_db", "dst_db", args) + + self.dst_queue_name = self.cf.get("dst_queue_name") + + def process_remote_batch(self, db, batch_id, ev_list, dst_db): + + # load data + rows = [] + for ev in ev_list: + data = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time] + rows.append(data) + ev.tag_done() + fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time'] + + # insert data + curs = dst_db.cursor() + pgq.bulk_insert_events(curs, rows, fields, self.dst_queue_name) + +if __name__ == '__main__': + script = QueueMover(sys.argv[1:]) + script.start() + diff --git a/scripts/queue_splitter.ini.templ b/scripts/queue_splitter.ini.templ new file mode 100644 index 00000000..68c5ccbb --- /dev/null +++ b/scripts/queue_splitter.ini.templ @@ -0,0 +1,13 @@ +[queue_splitter] +job_name = queue_splitter_test + +src_db = dbname=sourcedb_test +dst_db = dbname=destdb_test + +pgq_queue_name = source_queue + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +use_skylog = 0 + diff --git a/scripts/queue_splitter.py b/scripts/queue_splitter.py new file mode 100755 index 00000000..c6714ca0 --- /dev/null +++ b/scripts/queue_splitter.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python + +# puts events into queue specified by field from 'queue_field' config parameter + +import sys, os, pgq, skytools + +class QueueSplitter(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "queue_splitter", "src_db", "dst_db", args) + + def process_remote_batch(self, db, batch_id, ev_list, dst_db): + cache = {} + queue_field = self.cf.get('queue_field', 'extra1') + for ev in ev_list: + row = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time] + queue = ev.__getattr__(queue_field) + if queue not in cache: + cache[queue] = [] + cache[queue].append(row) + ev.tag_done() + + # should match the composed row + fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time'] + + # now send them to right queues + curs = dst_db.cursor() + for queue, rows in cache.items(): + pgq.bulk_insert_events(curs, rows, fields, queue) + +if __name__ == '__main__': + script = QueueSplitter(sys.argv[1:]) + script.start() + diff --git a/scripts/scriptmgr.ini.templ b/scripts/scriptmgr.ini.templ new file mode 100644 index 00000000..7fa1419d --- /dev/null +++ b/scripts/scriptmgr.ini.templ @@ -0,0 +1,43 @@ + +[scriptmgr] +job_name = scriptmgr_cphdb5 +config_list = ~/dbscripts/conf/*.ini, ~/random/conf/*.ini +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid +#use_skylog = 1 + +# +# defaults for services +# +[DEFAULT] +cwd = ~/dbscripts +args = -v + +# +# service descriptions +# + +[cube_dispatcher] +script = cube_dispatcher.py + +[table_dispatcher] +script = table_dispatcher.py + +[bulk_loader] +script = bulk_loader.py + +[londiste] +script = londiste.py +args = replay + +[pgqadm] +script = pgqadm.py +args = ticker + +# +# services to be ignored +# + +[log_checker] +disabled = 1 + diff --git a/scripts/scriptmgr.py b/scripts/scriptmgr.py new file mode 100755 index 00000000..2ee742b2 --- /dev/null +++ b/scripts/scriptmgr.py @@ -0,0 +1,220 @@ +#! /usr/bin/env python + +"""Bulk start/stop of scripts. + +Reads a bunch of config files and maps them to scripts, then handles those. +""" + +import sys, os, skytools, signal, glob, ConfigParser, time + +command_usage = """ +%prog [options] INI CMD [subcmd args] + +commands: + start [-a | jobname ..] start a job + stop [-a | jobname ..] stop a job + restart [-a | jobname ..] restart job(s) + reload [-a | jobname ..] send reload signal + status +""" + +def job_sort_cmp(j1, j2): + d1 = j1['service'] + j1['job_name'] + d2 = j2['service'] + j2['job_name'] + if d1 < d2: return -1 + elif d1 > d2: return 1 + else: return 0 + +class ScriptMgr(skytools.DBScript): + def init_optparse(self, p = None): + p = skytools.DBScript.init_optparse(self, p) + p.add_option("-a", "--all", action="store_true", help="apply command to all jobs") + p.set_usage(command_usage.strip()) + return p + + def load_jobs(self): + self.svc_list = [] + self.svc_map = {} + self.config_list = [] + + # load services + svc_list = self.cf.sections() + svc_list.remove(self.service_name) + for svc_name in svc_list: + cf = self.cf.clone(svc_name) + disabled = cf.getboolean('disabled', 0) + defscript = None + if disabled: + defscript = '/disabled' + svc = { + 'service': svc_name, + 'script': cf.getfile('script', defscript), + 'cwd': cf.getfile('cwd'), + 'disabled': cf.getboolean('disabled', 0), + 'args': cf.get('args', ''), + } + self.svc_list.append(svc) + self.svc_map[svc_name] = svc + + # generate config list + for tmp in self.cf.getlist('config_list'): + tmp = os.path.expanduser(tmp) + tmp = os.path.expandvars(tmp) + for fn in glob.glob(tmp): + self.config_list.append(fn) + + # read jobs + for fn in self.config_list: + raw = ConfigParser.SafeConfigParser({'job_name':'?', 'service_name':'?'}) + raw.read(fn) + + # skip its own config + if raw.has_section(self.service_name): + continue + + got = 0 + for sect in raw.sections(): + if sect in self.svc_map: + got = 1 + self.add_job(fn, sect) + if not got: + self.log.warning('Cannot find service for %s' % fn) + + def add_job(self, cf_file, service_name): + svc = self.svc_map[service_name] + cf = skytools.Config(service_name, cf_file) + disabled = svc['disabled'] + if not disabled: + disabled = cf.getboolean('disabled', 0) + job = { + 'disabled': disabled, + 'config': cf_file, + 'cwd': svc['cwd'], + 'script': svc['script'], + 'args': svc['args'], + 'service': svc['service'], + 'job_name': cf.get('job_name'), + 'pidfile': cf.getfile('pidfile'), + } + self.job_list.append(job) + self.job_map[job['job_name']] = job + + def cmd_status(self): + for job in self.job_list: + os.chdir(job['cwd']) + cf = skytools.Config(job['service'], job['config']) + pidfile = cf.getfile('pidfile') + name = job['job_name'] + svc = job['service'] + if job['disabled']: + name += " (disabled)" + + if os.path.isfile(pidfile): + print " OK [%s] %s" % (svc, name) + else: + print " STOPPED [%s] %s" % (svc, name) + + def cmd_info(self): + for job in self.job_list: + print job + + def cmd_start(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return 0 + self.log.info('Starting %s' % job_name) + os.chdir(job['cwd']) + pidfile = job['pidfile'] + if os.path.isfile(pidfile): + self.log.warning("Script %s seems running") + return 0 + cmd = "%(script)s %(config)s %(args)s -d" % job + res = os.system(cmd) + self.log.debug(res) + if res != 0: + self.log.error('startup failed: %s' % job_name) + return 1 + else: + return 0 + + def cmd_stop(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return + self.log.info('Stopping %s' % job_name) + self.signal_job(job, signal.SIGINT) + + def cmd_reload(self, job_name): + job = self.job_map[job_name] + if job['disabled']: + self.log.info("Skipping %s" % job_name) + return + self.log.info('Reloading %s' % job_name) + self.signal_job(job, signal.SIGHUP) + + def signal_job(self, job, sig): + os.chdir(job['cwd']) + pidfile = job['pidfile'] + if os.path.isfile(pidfile): + pid = int(open(pidfile).read()) + os.kill(pid, sig) + else: + self.log.warning("Job %s not running" % job['job_name']) + + def work(self): + self.set_single_loop(1) + self.job_list = [] + self.job_map = {} + self.load_jobs() + + if len(self.args) < 2: + print "need command" + sys.exit(1) + + jobs = self.args[2:] + if len(jobs) == 0 and self.options.all: + for job in self.job_list: + jobs.append(job['job_name']) + + self.job_list.sort(job_sort_cmp) + + cmd = self.args[1] + if cmd == "status": + self.cmd_status() + return + elif cmd == "info": + self.cmd_info() + return + + if len(jobs) == 0: + print "no jobs given?" + sys.exit(1) + + if cmd == "start": + err = 0 + for n in jobs: + err += self.cmd_start(n) + if err > 0: + self.log.error('some scripts failed') + sys.exit(1) + elif cmd == "stop": + for n in jobs: + self.cmd_stop(n) + elif cmd == "restart": + for n in jobs: + self.cmd_stop(n) + time.sleep(2) + self.cmd_start(n) + elif cmd == "reload": + for n in self.jobs: + self.cmd_reload(n) + else: + print "unknown command:", cmd + sys.exit(1) + +if __name__ == '__main__': + script = ScriptMgr('scriptmgr', sys.argv[1:]) + script.start() + diff --git a/scripts/table_dispatcher.ini.templ b/scripts/table_dispatcher.ini.templ new file mode 100644 index 00000000..131dd7fe --- /dev/null +++ b/scripts/table_dispatcher.ini.templ @@ -0,0 +1,31 @@ +[udata_dispatcher] +job_name = test_move + +src_db = dbname=sourcedb_test +dst_db = dbname=dataminedb_test + +pgq_queue_name = OrderLog + +logfile = ~/log/%(job_name)s.log +pidfile = ~/pid/%(job_name)s.pid + +# where to put data. when partitioning, will be used as base name +dest_table = orders + +# date field with will be used for partitioning +# special value: _EVTIME - event creation time +part_column = start_date + +#fields = * +#fields = id, name +#fields = id:newid, name, bar:baz + + +# template used for creating partition tables +# _DEST_TABLE +part_template = + create table _DEST_TABLE () inherits (orders); + alter table only _DEST_TABLE add constraint _DEST_TABLE_pkey primary key (id); + grant select on _DEST_TABLE to group reporting; + + diff --git a/scripts/table_dispatcher.py b/scripts/table_dispatcher.py new file mode 100755 index 00000000..054ced9a --- /dev/null +++ b/scripts/table_dispatcher.py @@ -0,0 +1,124 @@ +#! /usr/bin/env python + +# it loads urlencoded rows for one trable from queue and inserts +# them into actual tables, with optional partitioning + +import sys, os, pgq, skytools + +DEST_TABLE = "_DEST_TABLE" +SCHEMA_TABLE = "_SCHEMA_TABLE" + +class TableDispatcher(pgq.SerialConsumer): + def __init__(self, args): + pgq.SerialConsumer.__init__(self, "table_dispatcher", "src_db", "dst_db", args) + + self.part_template = self.cf.get("part_template", '') + self.dest_table = self.cf.get("dest_table") + self.part_field = self.cf.get("part_field", '') + self.part_method = self.cf.get("part_method", 'daily') + if self.part_method not in ('daily', 'monthly'): + raise Exception('bad part_method') + + if self.cf.get("fields", "*") == "*": + self.field_map = None + else: + self.field_map = {} + for fval in self.cf.getlist('fields'): + tmp = fval.split(':') + if len(tmp) == 1: + self.field_map[tmp[0]] = tmp[0] + else: + self.field_map[tmp[0]] = tmp[1] + + def process_remote_batch(self, batch_id, ev_list, dst_db): + if len(ev_list) == 0: + return + + # actual processing + self.dispatch(dst_db, ev_list) + + # tag as done + for ev in ev_list: + ev.tag_done() + + def dispatch(self, dst_db, ev_list): + """Generic dispatcher.""" + + # load data + tables = {} + for ev in ev_list: + row = skytools.db_urldecode(ev.data) + + # guess dest table + if self.part_field: + if self.part_field == "_EVTIME": + partval = str(ev.creation_date) + else: + partval = str(row[self.part_field]) + partval = partval.split(' ')[0] + date = partval.split('-') + if self.part_method == 'monthly': + date = date[:2] + suffix = '_'.join(date) + tbl = "%s_%s" % (self.dest_table, suffix) + else: + tbl = self.dest_table + + # map fields + if self.field_map is None: + dstrow = row + else: + dstrow = {} + for k, v in self.field_map.items(): + dstrow[v] = row[k] + + # add row into table + if not tbl in tables: + tables[tbl] = [dstrow] + else: + tables[tbl].append(dstrow) + + # create tables if needed + self.check_tables(dst_db, tables) + + # insert into data tables + curs = dst_db.cursor() + for tbl, tbl_rows in tables.items(): + skytools.magic_insert(curs, tbl, tbl_rows) + + def check_tables(self, dcon, tables): + """Checks that tables needed for copy are there. If not + then creates them. + + Used by other procedures to ensure that table is there + before they start inserting. + + The commits should not be dangerous, as we haven't done anything + with cdr's yet, so they should still be in one TX. + + Although it would be nicer to have a lock for table creation. + """ + + dcur = dcon.cursor() + exist_map = {} + for tbl in tables.keys(): + if not skytools.exists_table(dcur, tbl): + if not self.part_template: + raise Exception('Dest table does not exists and no way to create it.') + + sql = self.part_template + sql = sql.replace(DEST_TABLE, tbl) + + # we do this to make sure that constraints for + # tables who contain a schema will still work + schema_table = tbl.replace(".", "__") + sql = sql.replace(SCHEMA_TABLE, schema_table) + + dcur.execute(sql) + dcon.commit() + self.log.info('%s: Created table %s' % (self.job_name, tbl)) + +if __name__ == '__main__': + script = TableDispatcher(sys.argv[1:]) + script.start() + diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..26e59ff4 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#! /usr/bin/env python + +from distutils.core import setup + +import re +buf = open("configure.ac","r").read(256) +m = re.search("AC_INIT[(][^,]*,\s+([^)]*)[)]", buf) +ac_ver = m.group(1) + +setup( + name = "skytools", + license = "BSD", + version = ac_ver, + maintainer = "Marko Kreen", + maintainer_email = "marko.kreen@skype.net", + url = "http://pgfoundry.org/projects/skytools/", + package_dir = {'': 'python'}, + packages = ['skytools', 'londiste', 'pgq'], + scripts = ['python/londiste.py', 'python/pgqadm.py', 'python/walmgr.py', + 'scripts/cube_dispatcher.py', 'scripts/queue_mover.py', + 'scripts/table_dispatcher.py', 'scripts/bulk_loader.py', + 'scripts/scriptmgr.py', 'scripts/queue_splitter.py'], + data_files = [ ('share/doc/skytools/conf', [ + 'python/conf/londiste.ini', + 'python/conf/pgqadm.ini', + 'python/conf/wal-master.ini', + 'python/conf/wal-slave.ini', + 'scripts/queue_mover.ini.templ', + 'scripts/queue_splitter.ini.templ', + 'scripts/cube_dispatcher.ini.templ', + 'scripts/table_dispatcher.ini.templ', + 'scripts/bulk_loader.ini.templ', + 'scripts/scriptmgr.ini.templ', + ])] +) + diff --git a/source.cfg b/source.cfg new file mode 100644 index 00000000..b3dc1683 --- /dev/null +++ b/source.cfg @@ -0,0 +1,13 @@ +# what to include in source distribution + +# MANIFEST.in for Python Distutils + +include Makefile COPYRIGHT README NEWS config.mak.in configure configure.ac source.cfg + +recursive-include sql *.sql Makefile *.out *.in *.[ch] README* *.in +recursive-include python/conf *.ini +recursive-include scripts *.py *.templ +recursive-include debian changelog packages.in +recursive-include doc * +recursive-include tests * + diff --git a/sql/Makefile b/sql/Makefile new file mode 100644 index 00000000..3ea6c12d --- /dev/null +++ b/sql/Makefile @@ -0,0 +1,10 @@ + +include ../config.mak + +SUBDIRS = logtriga londiste pgq pgq_ext txid + +all install clean distclean installcheck: + for dir in $(SUBDIRS); do \ + make -C $$dir $@ DESTDIR=$(DESTDIR) || exit $?; \ + done + diff --git a/sql/logtriga/Makefile b/sql/logtriga/Makefile new file mode 100644 index 00000000..7f055489 --- /dev/null +++ b/sql/logtriga/Makefile @@ -0,0 +1,12 @@ + +include ../../config.mak + +MODULE_big = logtriga +SRCS = logtriga.c textbuf.c +OBJS = $(SRCS:.c=.o) +DATA_built = logtriga.sql + +REGRESS = logtriga + +include $(PGXS) + diff --git a/sql/logtriga/README.logtriga b/sql/logtriga/README.logtriga new file mode 100644 index 00000000..747aaede --- /dev/null +++ b/sql/logtriga/README.logtriga @@ -0,0 +1,47 @@ + +logtriga - generic table changes logger +======================================= + +logtriga provides generic table changes logging trigger. +It prepares partial SQL statement about a change and +gives it to user query. + +Usage +----- + + CREATE TRIGGER foo_log AFTER INSERT OR UPDATE OR DELETE ON foo_tbl + FOR EACH ROW EXECUTE PROCEDURE logtriga(column_types, query); + +Where column_types is a string where each charater defines type of +that column. Known types: + + * k - one of primary key columns for table. + * v - data column + * i - uninteresting column, to be ignored. + +Trigger function prepares 2 string arguments for query and executes it. + + * $1 - Operation type: I/U/D. + * $2 - Partial SQL for event playback. + + * INSERT INTO FOO_TBL (field, list) values (val1, val2) + * UPDATE FOO_TBL SET field1 = val1, field2 = val2 where key1 = kval1 + * DELETE FROM FOO_TBL WHERE key1 = keyval1 + +The upper-case part is left out. + +Example +------- + +Following query emulates Slony-I behaviour: + + insert into SL_SCHEMA.sl_log_1 + (log_origin, log_xid, log_tableid, + log_actionseq, log_cmdtype, log_cmddata) + values (CLUSTER_IDENT, SL_SCHEMA.getCurrentXid(), TABLE_OID, + nextval('SL_SCHEMA.sl_action_seq'), $1, $2) + +The upper-case strings should be replaced with actual values +on trigger creation. + + diff --git a/sql/logtriga/expected/logtriga.out b/sql/logtriga/expected/logtriga.out new file mode 100644 index 00000000..64daf912 --- /dev/null +++ b/sql/logtriga/expected/logtriga.out @@ -0,0 +1,95 @@ +-- init +\set ECHO none +create table rtest ( + id integer primary key, + dat text +); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest" +create table clog ( + id serial, + op text, + data text +); +NOTICE: CREATE TABLE will create implicit sequence "clog_id_seq" for serial column "clog.id" +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kv', +'insert into clog (op, data) values ($1, $2)'); +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+-------------------------------- + 1 | I | (id,dat) values ('1','value1') + 2 | U | dat='value2' where id='1' + 3 | D | id='1' +(3 rows) + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+-------------------------------- + 4 | I | (id,dat) values ('1','value1') + 5 | U | dat='value2' where id='1' + 6 | D | id='1' +(3 rows) + +-- test field rename +alter table rtest alter column dat type integer using 0; +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+----------------------------- + 7 | I | (id,dat) values ('1','666') + 8 | U | dat='5' where id='1' + 9 | D | id='1' +(3 rows) + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kiv', +'insert into clog (op, data) values ($1, $2)'); +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; +select * from clog; delete from clog; + id | op | data +----+----+--------------------------------- + 10 | I | (id,dat2) values ('1','newdat') + 11 | U | dat2='newdat2' where id='1' + 12 | D | id='1' +(3 rows) + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('vik', +'insert into clog (op, data) values ($1, $2)'); +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; +ERROR: logtriga: Unexpected NULL key value +delete from rtest where id=1; +ERROR: logtriga: Unexpected NULL key value +delete from rtest where id=2; +ERROR: logtriga: Unexpected NULL key value +select * from clog; delete from clog; + id | op | data +----+----+---------------------------------------- + 13 | I | (id,dat2) values ('1','non-null') + 14 | I | (id,dat2) values ('2',null) + 15 | U | dat2='non-null2' where dat2='non-null' + 16 | U | dat2=NULL where dat2='non-null2' +(4 rows) + diff --git a/sql/logtriga/logtriga.c b/sql/logtriga/logtriga.c new file mode 100644 index 00000000..af284b8d --- /dev/null +++ b/sql/logtriga/logtriga.c @@ -0,0 +1,500 @@ +/* ---------------------------------------------------------------------- + * logtriga.c + * + * Generic trigger for logging table changes. + * Based on Slony-I log trigger. + * Does not depend on event storage. + * + * Copyright (c) 2003-2006, PostgreSQL Global Development Group + * Author: Jan Wieck, Afilias USA INC. + * + * Generalized by Marko Kreen. + * ---------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "executor/spi.h" +#include "commands/trigger.h" +#include "catalog/pg_operator.h" +#include "utils/typcache.h" + +#include "textbuf.h" + +PG_FUNCTION_INFO_V1(logtriga); +Datum logtriga(PG_FUNCTION_ARGS); + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* + * There may be several plans to be cached. + * + * FIXME: plans are kept in singe-linked list + * so not very fast access. Probably they should be + * handled more intelligently. + */ +typedef struct PlanCache PlanCache; + +struct PlanCache { + PlanCache *next; + char *query; + void *plan; +}; + +/* + * Cache result allocations. + */ +typedef struct ArgCache +{ + TBuf *op_type; + TBuf *op_data; +} ArgCache; + + +static PlanCache *plan_cache = NULL; +static ArgCache *arg_cache = NULL; + +/* + * Cache helpers + */ + +static void *get_plan(const char *query) +{ + PlanCache *c; + void *plan; + Oid plan_types[2]; + + for (c = plan_cache; c; c = c->next) + if (strcmp(query, c->query) == 0) + return c->plan; + + /* + * Plan not cached, prepare new plan then. + */ + plan_types[0] = TEXTOID; + plan_types[1] = TEXTOID; + plan = SPI_saveplan(SPI_prepare(query, 2, plan_types)); + if (plan == NULL) + elog(ERROR, "logtriga: SPI_prepare() failed"); + + /* create cache object */ + c = malloc(sizeof(*c)); + if (!c) + elog(ERROR, "logtriga: no memory for plan cache"); + + c->plan = plan; + c->query = strdup(query); + + /* insert at start */ + c->next = plan_cache; + plan_cache = c; + + return plan; +} + +static ArgCache * +get_arg_cache(void) +{ + if (arg_cache == NULL) { + ArgCache *a = malloc(sizeof(*a)); + if (!a) + elog(ERROR, "logtriga: no memory"); + memset(a, 0, sizeof(*a)); + a->op_type = tbuf_alloc(8); + a->op_data = tbuf_alloc(8192); + arg_cache = a; + } + return arg_cache; +} + +static void +append_key_eq(TBuf *tbuf, const char *col_ident, const char *col_value) +{ + if (col_value == NULL) + elog(ERROR, "logtriga: Unexpected NULL key value"); + + tbuf_encode_cstring(tbuf, col_ident, "quote_ident"); + tbuf_append_char(tbuf, '='); + tbuf_encode_cstring(tbuf, col_value, "quote_literal"); +} + +static void +append_normal_eq(TBuf *tbuf, const char *col_ident, const char *col_value) +{ + tbuf_encode_cstring(tbuf, col_ident, "quote_ident"); + tbuf_append_char(tbuf, '='); + if (col_value != NULL) + tbuf_encode_cstring(tbuf, col_value, "quote_literal"); + else + tbuf_append_cstring(tbuf, "NULL"); +} + +static void process_insert(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple new_row = tg->tg_trigtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + int i; + int need_comma = false; + int attkind_idx; + + /* + * INSERT + * + * op_type = 'I' op_data = ("non-NULL-col" [, ...]) values ('value' [, + * ...]) + */ + tbuf_append_cstring(cs->op_type, "I"); + + /* + * Specify all the columns + */ + tbuf_append_char(cs->op_data, '('); + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + char *col_ident; + + /* Skip dropped columns */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + /* Check if allowed by colstring */ + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + if (attkind[attkind_idx] == 'i') + continue; + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + /* quote column name */ + col_ident = SPI_fname(tupdesc, i + 1); + tbuf_encode_cstring(cs->op_data, col_ident, "quote_ident"); + } + + /* + * Append the string ") values (" + */ + tbuf_append_cstring(cs->op_data, ") values ("); + + /* + * Append the values + */ + need_comma = false; + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + char *col_value; + + /* Skip dropped columns */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + /* Check if allowed by colstring */ + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + if (attkind[attkind_idx] == 'i') + continue; + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + /* quote column value */ + col_value = SPI_getvalue(new_row, tupdesc, i + 1); + if (col_value == NULL) + tbuf_append_cstring(cs->op_data, "null"); + else + tbuf_encode_cstring(cs->op_data, col_value, "quote_literal"); + } + + /* + * Terminate and done + */ + tbuf_append_char(cs->op_data, ')'); +} + +static int process_update(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple old_row = tg->tg_trigtuple; + HeapTuple new_row = tg->tg_newtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + Datum old_value; + Datum new_value; + bool old_isnull; + bool new_isnull; + + char *col_ident; + char *col_value; + int i; + int need_comma = false; + int need_and = false; + int attkind_idx; + int ignore_count = 0; + + /* + * UPDATE + * + * op_type = 'U' op_data = "col_ident"='value' [, ...] where "pk_ident" = + * 'value' [ and ...] + */ + tbuf_append_cstring(cs->op_type, "U"); + + attkind_idx = -1; + for (i = 0; i < tg->tg_relation->rd_att->natts; i++) + { + /* + * Ignore dropped columns + */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] == '\0') + break; + + old_value = SPI_getbinval(old_row, tupdesc, i + 1, &old_isnull); + new_value = SPI_getbinval(new_row, tupdesc, i + 1, &new_isnull); + + /* + * If old and new value are NULL, the column is unchanged + */ + if (old_isnull && new_isnull) + continue; + + /* + * If both are NOT NULL, we need to compare the values and skip + * setting the column if equal + */ + if (!old_isnull && !new_isnull) + { + Oid opr_oid; + FmgrInfo *opr_finfo_p; + + /* + * Lookup the equal operators function call info using the + * typecache if available + */ + TypeCacheEntry *type_cache; + + type_cache = lookup_type_cache(SPI_gettypeid(tupdesc, i + 1), + TYPECACHE_EQ_OPR | TYPECACHE_EQ_OPR_FINFO); + opr_oid = type_cache->eq_opr; + if (opr_oid == ARRAY_EQ_OP) + opr_oid = InvalidOid; + else + opr_finfo_p = &(type_cache->eq_opr_finfo); + + /* + * If we have an equal operator, use that to do binary + * comparision. Else get the string representation of both + * attributes and do string comparision. + */ + if (OidIsValid(opr_oid)) + { + if (DatumGetBool(FunctionCall2(opr_finfo_p, + old_value, new_value))) + continue; + } + else + { + char *old_strval = SPI_getvalue(old_row, tupdesc, i + 1); + char *new_strval = SPI_getvalue(new_row, tupdesc, i + 1); + + if (strcmp(old_strval, new_strval) == 0) + continue; + } + } + + if (attkind[attkind_idx] == 'i') + { + /* this change should be ignored */ + ignore_count++; + continue; + } + + if (need_comma) + tbuf_append_char(cs->op_data, ','); + else + need_comma = true; + + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(new_row, tupdesc, i + 1); + + append_normal_eq(cs->op_data, col_ident, col_value); + } + + /* + * It can happen that the only UPDATE an application does is to set a + * column to the same value again. In that case, we'd end up here with + * no columns in the SET clause yet. We add the first key column here + * with it's old value to simulate the same for the replication + * engine. + */ + if (!need_comma) + { + /* there was change in ignored columns, skip whole event */ + if (ignore_count > 0) + return 0; + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] == 'k') + break; + } + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + append_key_eq(cs->op_data, col_ident, col_value); + } + + tbuf_append_cstring(cs->op_data, " where "); + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + /* + * Ignore dropped columns + */ + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] != 'k') + continue; + if (attkind[attkind_idx] == '\0') + break; + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + if (need_and) + tbuf_append_cstring(cs->op_data, " and "); + else + need_and = true; + + append_key_eq(cs->op_data, col_ident, col_value); + } + return 1; +} + +static void process_delete(ArgCache *cs, TriggerData *tg, char *attkind) +{ + HeapTuple old_row = tg->tg_trigtuple; + TupleDesc tupdesc = tg->tg_relation->rd_att; + char *col_ident; + char *col_value; + int i; + int need_and = false; + int attkind_idx; + + /* + * DELETE + * + * op_type = 'D' op_data = "pk_ident"='value' [and ...] + */ + tbuf_append_cstring(cs->op_type, "D"); + + for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++) + { + if (tupdesc->attrs[i]->attisdropped) + continue; + + attkind_idx++; + if (attkind[attkind_idx] != 'k') + continue; + if (attkind[attkind_idx] == '\0') + break; + col_ident = SPI_fname(tupdesc, i + 1); + col_value = SPI_getvalue(old_row, tupdesc, i + 1); + + if (need_and) + tbuf_append_cstring(cs->op_data, " and "); + else + need_and = true; + + append_key_eq(cs->op_data, col_ident, col_value); + } +} + +Datum logtriga(PG_FUNCTION_ARGS) +{ + TriggerData *tg; + Datum argv[2]; + int rc; + ArgCache *cs; + char *attkind; + char *query; + int need_event = 1; + + /* + * Get the trigger call context + */ + if (!CALLED_AS_TRIGGER(fcinfo)) + elog(ERROR, "logtriga not called as trigger"); + tg = (TriggerData *) (fcinfo->context); + + /* + * Check all logTrigger() calling conventions + */ + if (!TRIGGER_FIRED_AFTER(tg->tg_event)) + elog(ERROR, "logtriga must be fired AFTER"); + if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event)) + elog(ERROR, "logtriga must be fired FOR EACH ROW"); + if (tg->tg_trigger->tgnargs != 2) + elog(ERROR, "logtriga must be defined with 2 args"); + + /* + * Connect to the SPI manager + */ + if ((rc = SPI_connect()) < 0) + elog(ERROR, "logtriga: SPI_connect() failed"); + + cs = get_arg_cache(); + + tbuf_reset(cs->op_type); + tbuf_reset(cs->op_data); + + /* + * Get all the trigger arguments + */ + attkind = tg->tg_trigger->tgargs[0]; + query = tg->tg_trigger->tgargs[1]; + + if (strchr(attkind, 'k') == NULL) + elog(ERROR, "logtriga: need at least one key column"); + + /* + * Determine cmdtype and op_data depending on the command type + */ + if (TRIGGER_FIRED_BY_INSERT(tg->tg_event)) + process_insert(cs, tg, attkind); + else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) + need_event = process_update(cs, tg, attkind); + else if (TRIGGER_FIRED_BY_DELETE(tg->tg_event)) + process_delete(cs, tg, attkind); + else + elog(ERROR, "logtriga fired for unhandled event"); + + /* + * Construct the parameter array and insert the log row. + */ + if (need_event) + { + argv[0] = PointerGetDatum(tbuf_look_text(cs->op_type)); + argv[1] = PointerGetDatum(tbuf_look_text(cs->op_data)); + SPI_execp(get_plan(query), argv, NULL, 0); + } + SPI_finish(); + return PointerGetDatum(NULL); +} + diff --git a/sql/logtriga/logtriga.sql.in b/sql/logtriga/logtriga.sql.in new file mode 100644 index 00000000..7bd36e7f --- /dev/null +++ b/sql/logtriga/logtriga.sql.in @@ -0,0 +1,10 @@ + +-- usage: logtriga(flds, query) +-- +-- query should include 2 args: +-- $1 - for op type I/U/D, +-- $2 - for op data + +CREATE OR REPLACE FUNCTION logtriga() RETURNS trigger +AS 'MODULE_PATHNAME', 'logtriga' LANGUAGE C; + diff --git a/sql/logtriga/sql/logtriga.sql b/sql/logtriga/sql/logtriga.sql new file mode 100644 index 00000000..f0acbf53 --- /dev/null +++ b/sql/logtriga/sql/logtriga.sql @@ -0,0 +1,74 @@ +-- init +\set ECHO none +\i logtriga.sql +\set ECHO all + +create table rtest ( + id integer primary key, + dat text +); + +create table clog ( + id serial, + op text, + data text +); + +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kv', +'insert into clog (op, data) values ($1, $2)'); + +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; +select * from clog; delete from clog; + +-- test field rename +alter table rtest alter column dat type integer using 0; +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5; +delete from rtest; +select * from clog; delete from clog; + + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('kiv', +'insert into clog (op, data) values ($1, $2)'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; +select * from clog; delete from clog; + + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure logtriga('vik', +'insert into clog (op, data) values ($1, $2)'); + +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; + +delete from rtest where id=1; +delete from rtest where id=2; + +select * from clog; delete from clog; + + + + diff --git a/sql/logtriga/textbuf.c b/sql/logtriga/textbuf.c new file mode 100644 index 00000000..1c7b5d33 --- /dev/null +++ b/sql/logtriga/textbuf.c @@ -0,0 +1,334 @@ + +#include +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "parser/keywords.h" + +#if 1 +#define talloc(len) malloc(len) +#define trealloc(p, len) realloc(p, len) +#define tfree(p) free(p) +#else +#define talloc(len) palloc(len) +#define trealloc(p, len) repalloc(p, len) +#define tfree(p) pfree(p) +#endif + +#include "textbuf.h" + +struct TBuf { + text *data; + int size; +}; + +static void request_avail(TBuf *tbuf, int len) +{ + int newlen = tbuf->size; + int need = VARSIZE(tbuf->data) + len; + if (need < newlen) + return; + while (need > newlen) + newlen *= 2; + tbuf->data = trealloc(tbuf->data, newlen); + tbuf->size = newlen; +} + +static inline char *get_endp(TBuf *tbuf) +{ + char *p = VARDATA(tbuf->data); + int len = VARSIZE(tbuf->data) - VARHDRSZ; + return p + len; +} + +static inline void inc_used(TBuf *tbuf, int len) +{ + VARATT_SIZEP(tbuf->data) += len; +} + +static void tbuf_init(TBuf *tbuf, int start_size) +{ + if (start_size < VARHDRSZ) + start_size = VARHDRSZ; + tbuf->data = talloc(start_size); + tbuf->size = start_size; + VARATT_SIZEP(tbuf->data) = VARHDRSZ; +} + +TBuf *tbuf_alloc(int start_size) +{ + TBuf *res; + res = talloc(sizeof(TBuf)); + tbuf_init(res, start_size); + return res; +} + +void tbuf_free(TBuf *tbuf) +{ + if (tbuf->data) + tfree(tbuf->data); + tfree(tbuf); +} + +int tbuf_get_size(TBuf *tbuf) +{ + return VARSIZE(tbuf->data) - VARHDRSZ; +} + +void tbuf_reset(TBuf *tbuf) +{ + VARATT_SIZEP(tbuf->data) = VARHDRSZ; +} + +const text *tbuf_look_text(TBuf *tbuf) +{ + return tbuf->data; +} + +const char *tbuf_look_cstring(TBuf *tbuf) +{ + char *p; + request_avail(tbuf, 1); + p = get_endp(tbuf); + *p = 0; + return VARDATA(tbuf->data); +} + +void tbuf_append_cstring(TBuf *tbuf, const char *str) +{ + int len = strlen(str); + request_avail(tbuf, len); + memcpy(get_endp(tbuf), str, len); + inc_used(tbuf, len); +} + +void tbuf_append_text(TBuf *tbuf, const text *str) +{ + int len = VARSIZE(str) - VARHDRSZ; + request_avail(tbuf, len); + memcpy(get_endp(tbuf), VARDATA(str), len); + inc_used(tbuf, len); +} + +void tbuf_append_char(TBuf *tbuf, char chr) +{ + char *p; + request_avail(tbuf, 1); + p = get_endp(tbuf); + *p = chr; + inc_used(tbuf, 1); +} + +text *tbuf_steal_text(TBuf *tbuf) +{ + text *data = tbuf->data; + tbuf->data = NULL; + return data; +} + +static const char b64tbl[] = +"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static int b64encode(char *dst, const uint8 *src, int srclen) +{ + char *p = dst; + const uint8 *s = src, *end = src + srclen; + int pos = 2; + uint32 buf = 0; + + while (s < end) { + buf |= (unsigned char) *s << (pos << 3); + pos--; + s++; + /* write it out */ + if (pos < 0) { + *p++ = b64tbl[ (buf >> 18) & 0x3f ]; + *p++ = b64tbl[ (buf >> 12) & 0x3f ]; + *p++ = b64tbl[ (buf >> 6) & 0x3f ]; + *p++ = b64tbl[ buf & 0x3f ]; + pos = 2; + buf = 0; + } + } + if (pos != 2) { + *p++ = b64tbl[ (buf >> 18) & 0x3f ]; + *p++ = b64tbl[ (buf >> 12) & 0x3f ]; + *p++ = (pos == 0) ? b64tbl[ (buf >> 6) & 0x3f ] : '='; + *p++ = '='; + } + return p - dst; +} + +static const char hextbl[] = "0123456789abcdef"; +static int urlencode(char *dst, const uint8 *src, int srclen) +{ + const uint8 *end = src + srclen; + char *p = dst; + while (src < end) { + if (*src == '=') + *p++ = '+'; + else if ((*src >= '0' && *src <= '9') + || (*src >= 'A' && *src <= 'Z') + || (*src >= 'a' && *src <= 'z')) + *p++ = *src; + else { + *p++ = '%'; + *p++ = hextbl[*src >> 4]; + *p++ = hextbl[*src & 15]; + } + } + return p - dst; +} + +static int quote_literal(char *dst, const uint8 *src, int srclen) +{ + const uint8 *cp1; + char *cp2; + int wl; + + cp1 = src; + cp2 = dst; + + *cp2++ = '\''; + while (srclen > 0) + { + if ((wl = pg_mblen(cp1)) != 1) + { + srclen -= wl; + + while (wl-- > 0) + *cp2++ = *cp1++; + continue; + } + + if (*cp1 == '\'') + *cp2++ = '\''; + if (*cp1 == '\\') + *cp2++ = '\\'; + *cp2++ = *cp1++; + srclen--; + } + + *cp2++ = '\''; + + return cp2 - dst; +} + + +/* + * slon_quote_identifier - Quote an identifier only if needed + * + * When quotes are needed, we palloc the required space; slightly + * space-wasteful but well worth it for notational simplicity. + * + * Version: pgsql/src/backend/utils/adt/ruleutils.c,v 1.188 2005/01/13 17:19:10 + */ +static int +quote_ident(char *dst, const uint8 *src, int srclen) +{ + /* + * Can avoid quoting if ident starts with a lowercase letter or + * underscore and contains only lowercase letters, digits, and + * underscores, *and* is not any SQL keyword. Otherwise, supply + * quotes. + */ + int nquotes = 0; + bool safe; + const char *ptr; + char *optr; + char ident[NAMEDATALEN + 1]; + + /* expect idents be not bigger than NAMEDATALEN */ + if (srclen > NAMEDATALEN) + srclen = NAMEDATALEN; + memcpy(ident, src, srclen); + ident[srclen] = 0; + + /* + * would like to use macros here, but they might yield + * unwanted locale-specific results... + */ + safe = ((ident[0] >= 'a' && ident[0] <= 'z') || ident[0] == '_'); + + for (ptr = ident; *ptr; ptr++) + { + char ch = *ptr; + + if ((ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9') || + (ch == '_')) + continue; /* okay */ + + safe = false; + if (ch == '"') + nquotes++; + } + + if (safe) + { + /* + * Check for keyword. This test is overly strong, since many of + * the "keywords" known to the parser are usable as column names, + * but the parser doesn't provide any easy way to test for whether + * an identifier is safe or not... so be safe not sorry. + * + * Note: ScanKeywordLookup() does case-insensitive comparison, but + * that's fine, since we already know we have all-lower-case. + */ + if (ScanKeywordLookup(ident) != NULL) + safe = false; + } + + optr = dst; + if (!safe) + *optr++ = '"'; + + for (ptr = ident; *ptr; ptr++) + { + char ch = *ptr; + + if (ch == '"') + *optr++ = '"'; + *optr++ = ch; + } + if (!safe) + *optr++ = '"'; + + return optr - dst; +} + + +void tbuf_encode_cstring(TBuf *tbuf, + const char *str, + const char *encoding) +{ + if (str == NULL) + elog(ERROR, "tbuf_encode_cstring: NULL"); + tbuf_encode_data(tbuf, (const uint8 *)str, strlen(str), encoding); +} + +void tbuf_encode_data(TBuf *tbuf, + const uint8 *data, int len, + const char *encoding) +{ + int dlen = 0; + char *dst; + if (strcmp(encoding, "url") == 0) { + request_avail(tbuf, len*3); + dst = get_endp(tbuf); + dlen = urlencode(dst, data, len); + } else if (strcmp(encoding, "base64") == 0) { + request_avail(tbuf, (len + 2) * 4 / 3); + dst = get_endp(tbuf); + dlen = b64encode(dst, data, len); + } else if (strcmp(encoding, "quote_literal") == 0) { + request_avail(tbuf, len * 2 + 2); + dst = get_endp(tbuf); + dlen = quote_literal(dst, data, len); + } else if (strcmp(encoding, "quote_ident") == 0) { + request_avail(tbuf, len * 2 + 2); + dst = get_endp(tbuf); + dlen = quote_ident(dst, data, len); + } else + elog(ERROR, "bad encoding"); + inc_used(tbuf, dlen); +} + diff --git a/sql/logtriga/textbuf.h b/sql/logtriga/textbuf.h new file mode 100644 index 00000000..acdff685 --- /dev/null +++ b/sql/logtriga/textbuf.h @@ -0,0 +1,26 @@ +struct TBuf; + +typedef struct TBuf TBuf; + +TBuf *tbuf_alloc(int start_size); +void tbuf_free(TBuf *tbuf); +int tbuf_get_size(TBuf *tbuf); +void tbuf_reset(TBuf *tbuf); + +const text *tbuf_look_text(TBuf *tbuf); +const char *tbuf_look_cstring(TBuf *tbuf); + +void tbuf_append_cstring(TBuf *tbuf, const char *str); +void tbuf_append_text(TBuf *tbuf, const text *str); +void tbuf_append_char(TBuf *tbuf, char chr); + +text *tbuf_steal_text(TBuf *tbuf); + +void tbuf_encode_cstring(TBuf *tbuf, + const char *str, + const char *encoding); + +void tbuf_encode_data(TBuf *tbuf, + const uint8 *data, int len, + const char *encoding); + diff --git a/sql/londiste/Makefile b/sql/londiste/Makefile new file mode 100644 index 00000000..154da071 --- /dev/null +++ b/sql/londiste/Makefile @@ -0,0 +1,20 @@ + +DATA_built = londiste.sql londiste.upgrade.sql +DOCS = README.londiste + +FUNCS = $(wildcard functions/*.sql) +SRCS = structure/tables.sql structure/types.sql $(FUNCS) + +REGRESS = londiste_install londiste_denytrigger londiste_provider londiste_subscriber +REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql + +include ../../config.mak + +include $(PGXS) + +londiste.sql: $(SRCS) + cat $(SRCS) > $@ + +londiste.upgrade.sql: $(FUNCS) + cat $(FUNCS) > $@ + diff --git a/sql/londiste/README.londiste b/sql/londiste/README.londiste new file mode 100644 index 00000000..5104f4ff --- /dev/null +++ b/sql/londiste/README.londiste @@ -0,0 +1,29 @@ + +londiste database backend +-------------------------- + +Provider side: +-------------- + +londiste.provider_table +londiste.provider_seq + + +Subscriber side +--------------- + +table londiste.completed +table londiste.subscriber_table +table londiste.subscriber_seq + + +Open issues +------------ + +- notify behaviour +- should notify-s given to db for processing? +- link init functions +- switchover +- are set_last_tick()/get_last_tick() functions needed anymore? +- typecheck for add_table()? + diff --git a/sql/londiste/expected/londiste_denytrigger.out b/sql/londiste/expected/londiste_denytrigger.out new file mode 100644 index 00000000..4fe2f408 --- /dev/null +++ b/sql/londiste/expected/londiste_denytrigger.out @@ -0,0 +1,40 @@ +create table denytest ( val integer); +insert into denytest values (1); +create trigger xdeny after insert or update or delete +on denytest for each row execute procedure londiste.deny_trigger(); +insert into denytest values (2); +ERROR: ('Changes no allowed on this table',) +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) +delete from denytest; +ERROR: ('Changes no allowed on this table',) +select londiste.disable_deny_trigger(true); + disable_deny_trigger +---------------------- + t +(1 row) + +update denytest set val = 2; +select londiste.disable_deny_trigger(true); + disable_deny_trigger +---------------------- + t +(1 row) + +update denytest set val = 2; +select londiste.disable_deny_trigger(false); + disable_deny_trigger +---------------------- + f +(1 row) + +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) +select londiste.disable_deny_trigger(false); + disable_deny_trigger +---------------------- + f +(1 row) + +update denytest set val = 2; +ERROR: ('Changes no allowed on this table',) diff --git a/sql/londiste/expected/londiste_install.out b/sql/londiste/expected/londiste_install.out new file mode 100644 index 00000000..e4527e08 --- /dev/null +++ b/sql/londiste/expected/londiste_install.out @@ -0,0 +1 @@ +\set ECHO off diff --git a/sql/londiste/expected/londiste_provider.out b/sql/londiste/expected/londiste_provider.out new file mode 100644 index 00000000..1c081936 --- /dev/null +++ b/sql/londiste/expected/londiste_provider.out @@ -0,0 +1,135 @@ +set client_min_messages = 'warning'; +-- +-- tables +-- +create table testdata ( + id serial primary key, + data text +); +create table testdata_nopk ( + id serial, + data text +); +select londiste.provider_add_table('pqueue', 'public.testdata_nopk'); +ERROR: need key column +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: no such event queue +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select pgq.create_queue('pqueue'); + create_queue +-------------- + 1 +(1 row) + +select londiste.provider_add_table('pqueue', 'public.testdata'); + provider_add_table +-------------------- + 1 +(1 row) + +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "provider_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.provider_table (queue_name, table_name, trigger_name) values ( $1 , $2 , $3 )" +PL/pgSQL function "provider_add_table" line 23 at SQL statement +PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_refresh_trigger('pqueue', 'public.testdata'); + provider_refresh_trigger +-------------------------- + 1 +(1 row) + +select * from londiste.provider_get_table_list('pqueue'); + table_name | trigger_name +-----------------+--------------- + public.testdata | pqueue_logger +(1 row) + +select londiste.provider_remove_table('pqueue', 'public.nonexist'); +ERROR: no such table registered +select londiste.provider_remove_table('pqueue', 'public.testdata'); + provider_remove_table +----------------------- + 1 +(1 row) + +select * from londiste.provider_get_table_list('pqueue'); + table_name | trigger_name +------------+-------------- +(0 rows) + +-- +-- seqs +-- +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +----------------------- +(0 rows) + +select londiste.provider_add_seq('pqueue', 'public.no_seq'); +ERROR: seq not found +CONTEXT: PL/pgSQL function "find_seq_oid" line 2 at return +SQL statement "SELECT 1 from pg_class where oid = londiste.find_seq_oid( $1 )" +PL/pgSQL function "provider_add_seq" line 10 at perform +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); + provider_add_seq +------------------ + 0 +(1 row) + +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: duplicate key violates unique constraint "provider_seq_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.provider_seq (queue_name, seq_name) values ( $1 , $2 )" +PL/pgSQL function "provider_add_seq" line 16 at SQL statement +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +------------------------ + public.testdata_id_seq +(1 row) + +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); + provider_remove_seq +--------------------- + 0 +(1 row) + +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +ERROR: seq not attached +select * from londiste.provider_get_seq_list('pqueue'); + provider_get_seq_list +----------------------- +(0 rows) + +-- +-- linked queue +-- +select londiste.provider_add_table('pqueue', 'public.testdata'); + provider_add_table +-------------------- + 1 +(1 row) + +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); +select londiste.provider_add_table('pqueue', 'public.testdata'); +ERROR: Linked queue, manipulation not allowed +CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return +select londiste.provider_remove_table('pqueue', 'public.testdata'); +ERROR: Linked queue, manipulation not allowed +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: Linked queue, cannot modify +select londiste.provider_remove_seq('pqueue', 'public.testdata_seq'); +ERROR: Linked queue, cannot modify +-- +-- cleanup +-- +delete from londiste.link; +drop table testdata; +drop table testdata_nopk; +delete from londiste.provider_seq; +delete from londiste.provider_table; +select pgq.drop_queue('pqueue'); + drop_queue +------------ + 1 +(1 row) + diff --git a/sql/londiste/expected/londiste_subscriber.out b/sql/londiste/expected/londiste_subscriber.out new file mode 100644 index 00000000..7ec6944e --- /dev/null +++ b/sql/londiste/expected/londiste_subscriber.out @@ -0,0 +1,128 @@ +set client_min_messages = 'warning'; +create table testdata ( + id serial primary key, + data text +); +-- +-- tables +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk'); + subscriber_add_table +---------------------- + 0 +(1 row) + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + subscriber_add_table +---------------------- + 0 +(1 row) + +select pgq.create_queue('pqueue'); + create_queue +-------------- + 1 +(1 row) + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select * from londiste.subscriber_get_table_list('pqueue'); + table_name | merge_state | snapshot | trigger_name +----------------------+-------------+----------+-------------- + public.testdata_nopk | | | + public.testdata | | | +(2 rows) + +select londiste.subscriber_remove_table('pqueue', 'public.nonexist'); +ERROR: no such table +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + subscriber_remove_table +------------------------- + 0 +(1 row) + +select * from londiste.subscriber_get_table_list('pqueue'); + table_name | merge_state | snapshot | trigger_name +----------------------+-------------+----------+-------------- + public.testdata_nopk | | | +(1 row) + +-- +-- seqs +-- +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- +(0 rows) + +select londiste.subscriber_add_seq('pqueue', 'public.no_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +ERROR: duplicate key violates unique constraint "subscriber_seq_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_seq (queue_name, seq_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_seq" line 4 at SQL statement +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- + public.no_seq + public.testdata_id_seq +(2 rows) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); + subscriber_remove_seq +----------------------- + 0 +(1 row) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +ERROR: no such seq? +select * from londiste.subscriber_get_seq_list('pqueue'); + subscriber_get_seq_list +------------------------- + public.no_seq +(1 row) + +-- +-- linked queue +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + subscriber_add_table +---------------------- + 0 +(1 row) + +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +ERROR: duplicate key violates unique constraint "subscriber_table_pkey" +CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )" +PL/pgSQL function "subscriber_add_table" line 2 at SQL statement +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + subscriber_remove_table +------------------------- + 0 +(1 row) + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); + subscriber_add_seq +-------------------- + 0 +(1 row) + +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq'); +ERROR: no such seq? diff --git a/sql/londiste/functions/londiste.denytrigger.sql b/sql/londiste/functions/londiste.denytrigger.sql new file mode 100644 index 00000000..5f69ec05 --- /dev/null +++ b/sql/londiste/functions/londiste.denytrigger.sql @@ -0,0 +1,19 @@ + +create or replace function londiste.deny_trigger() +returns trigger as $$ + if 'undeny' in GD: + return 'OK' + plpy.error('Changes no allowed on this table') +$$ language plpythonu; + +create or replace function londiste.disable_deny_trigger(i_allow boolean) +returns boolean as $$ + if args[0]: + GD['undeny'] = 1 + return True + else: + if 'undeny' in GD: + del GD['undeny'] + return False +$$ language plpythonu; + diff --git a/sql/londiste/functions/londiste.find_column_types.sql b/sql/londiste/functions/londiste.find_column_types.sql new file mode 100644 index 00000000..52f8864a --- /dev/null +++ b/sql/londiste/functions/londiste.find_column_types.sql @@ -0,0 +1,26 @@ +create or replace function londiste.find_column_types(tbl text) +returns text as $$ +declare + res text; + col record; + tbl_oid oid; +begin + tbl_oid := londiste.find_table_oid(tbl); + res := ''; + for col in + SELECT CASE WHEN k.attname IS NOT NULL THEN 'k' ELSE 'v' END AS type + FROM pg_attribute a LEFT JOIN ( + SELECT k.attname FROM pg_index i, pg_attribute k + WHERE i.indrelid = tbl_oid AND k.attrelid = i.indexrelid + AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped + ) k ON (k.attname = a.attname) + WHERE a.attrelid = tbl_oid AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + loop + res := res || col.type; + end loop; + + return res; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.find_table_oid.sql b/sql/londiste/functions/londiste.find_table_oid.sql new file mode 100644 index 00000000..907b71ba --- /dev/null +++ b/sql/londiste/functions/londiste.find_table_oid.sql @@ -0,0 +1,49 @@ +create or replace function londiste.find_rel_oid(tbl text, kind text) +returns oid as $$ +declare + res oid; + pos integer; + schema text; + name text; +begin + pos := position('.' in tbl); + if pos > 0 then + schema := substring(tbl for pos - 1); + name := substring(tbl from pos + 1); + else + schema := 'public'; + name := tbl; + end if; + select c.oid into res + from pg_namespace n, pg_class c + where c.relnamespace = n.oid + and c.relkind = kind + and n.nspname = schema and c.relname = name; + if not found then + if kind = 'r' then + raise exception 'table not found'; + elsif kind = 'S' then + raise exception 'seq not found'; + else + raise exception 'weird relkind'; + end if; + end if; + + return res; +end; +$$ language plpgsql; + +create or replace function londiste.find_table_oid(tbl text) +returns oid as $$ +begin + return londiste.find_rel_oid(tbl, 'r'); +end; +$$ language plpgsql; + +create or replace function londiste.find_seq_oid(tbl text) +returns oid as $$ +begin + return londiste.find_rel_oid(tbl, 'S'); +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.get_last_tick.sql b/sql/londiste/functions/londiste.get_last_tick.sql new file mode 100644 index 00000000..e50d6e99 --- /dev/null +++ b/sql/londiste/functions/londiste.get_last_tick.sql @@ -0,0 +1,13 @@ + +create or replace function londiste.get_last_tick(i_consumer text) +returns bigint as $$ +declare + res bigint; +begin + select last_tick_id into res + from londiste.completed + where consumer_id = i_consumer; + return res; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.link.sql b/sql/londiste/functions/londiste.link.sql new file mode 100644 index 00000000..befedf04 --- /dev/null +++ b/sql/londiste/functions/londiste.link.sql @@ -0,0 +1,112 @@ + +create or replace function londiste.link_source(i_dst_name text) +returns text as $$ +declare + res text; +begin + select source into res from londiste.link + where dest = i_dst_name; + return res; +end; +$$ language plpgsql security definer; + +create or replace function londiste.link_dest(i_source_name text) +returns text as $$ +declare + res text; +begin + select dest into res from londiste.link + where source = i_source_name; + return res; +end; +$$ language plpgsql security definer; + +create or replace function londiste.cmp_list(list1 text, a_queue text, a_table text, a_field text) +returns boolean as $$ +declare + sql text; + tmp record; + list2 text; +begin + sql := 'select ' || a_field || ' as name from ' || a_table + || ' where queue_name = ' || quote_literal(a_queue) + || ' order by 1'; + list2 := ''; + for tmp in execute sql loop + if list2 = '' then + list2 := tmp.name; + else + list2 := list2 || ',' || tmp.name; + end if; + end loop; + return list1 = list2; +end; +$$ language plpgsql; + +create or replace function londiste.link(i_source_name text, i_dest_name text, prov_tick_id bigint, prov_tbl_list text, prov_seq_list text) +returns text as $$ +declare + tmp text; + list text; + tick_seq text; + external boolean; + last_tick bigint; +begin + -- check if all matches + if not londiste.cmp_list(prov_tbl_list, i_source_name, + 'londiste.subscriber_table', 'table_name') + then + raise exception 'not all tables copied into subscriber'; + end if; + if not londiste.cmp_list(prov_seq_list, i_source_name, + 'londiste.subscriber_seq', 'seq_name') + then + raise exception 'not all seqs copied into subscriber'; + end if; + if not londiste.cmp_list(prov_seq_list, i_dest_name, + 'londiste.provider_table', 'table_name') + then + raise exception 'linked provider queue does not have all tables'; + end if; + if not londiste.cmp_list(prov_seq_list, i_dest_name, + 'londiste.provider_seq', 'seq_name') + then + raise exception 'linked provider queue does not have all seqs'; + end if; + + -- check pgq + select queue_external_ticker, queue_tick_seq into external, tick_seq + from pgq.queue where queue_name = i_dest_name; + if not found then + raise exception 'dest queue does not exist'; + end if; + if external then + raise exception 'dest queue has already external_ticker turned on?'; + end if; + + if nextval(tick_seq) >= prov_tick_id then + raise exception 'dest queue ticks larger'; + end if; + + update pgq.queue set queue_external_ticker = true + where queue_name = i_dest_name; + + insert into londiste.link (source, dest) values (i_source_name, i_dest_name); + + return null; +end; +$$ language plpgsql security definer; + +create or replace function londiste.link_del(i_source_name text, i_dest_name text) +returns text as $$ +begin + delete from londiste.link + where source = i_source_name + and dest = i_dest_name; + if not found then + raise exception 'no suck link'; + end if; + return null; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_add_seq.sql b/sql/londiste/functions/londiste.provider_add_seq.sql new file mode 100644 index 00000000..6658ef63 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_add_seq.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.provider_add_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + -- check if linked queue + link := londiste.link_source(i_queue_name); + if link is not null then + raise exception 'Linked queue, cannot modify'; + end if; + + perform 1 from pg_class + where oid = londiste.find_seq_oid(i_seq_name); + if not found then + raise exception 'seq not found'; + end if; + + insert into londiste.provider_seq (queue_name, seq_name) + values (i_queue_name, i_seq_name); + perform londiste.provider_notify_change(i_queue_name); + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_add_table.sql b/sql/londiste/functions/londiste.provider_add_table.sql new file mode 100644 index 00000000..354b5572 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_add_table.sql @@ -0,0 +1,48 @@ +create or replace function londiste.provider_add_table( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + tgname text; + sql text; +begin + if londiste.link_source(i_queue_name) is not null then + raise exception 'Linked queue, manipulation not allowed'; + end if; + + if position('k' in i_col_types) < 1 then + raise exception 'need key column'; + end if; + if position('.' in i_table_name) < 1 then + raise exception 'need fully-qualified table name'; + end if; + select queue_name into tgname + from pgq.queue where queue_name = i_queue_name; + if not found then + raise exception 'no such event queue'; + end if; + + tgname := i_queue_name || '_logger'; + tgname := replace(lower(tgname), '.', '_'); + insert into londiste.provider_table + (queue_name, table_name, trigger_name) + values (i_queue_name, i_table_name, tgname); + + perform londiste.provider_create_trigger( + i_queue_name, i_table_name, i_col_types); + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.provider_add_table( + i_queue_name text, + i_table_name text +) returns integer as $$ +begin + return londiste.provider_add_table(i_queue_name, i_table_name, + londiste.find_column_types(i_table_name)); +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.provider_create_trigger.sql b/sql/londiste/functions/londiste.provider_create_trigger.sql new file mode 100644 index 00000000..8b16f3be --- /dev/null +++ b/sql/londiste/functions/londiste.provider_create_trigger.sql @@ -0,0 +1,33 @@ + +create or replace function londiste.provider_create_trigger( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + tgname text; + sql text; +begin + select trigger_name into tgname + from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'table not found'; + end if; + + sql := 'select pgq.insert_event(' + || quote_literal(i_queue_name) + || ', $1, $2, ' + || quote_literal(i_table_name) + || ', NULL, NULL, NULL)'; + execute 'create trigger ' || tgname + || ' after insert or update or delete on ' + || i_table_name + || ' for each row execute procedure logtriga($arg1$' + || i_col_types || '$arg1$, $arg2$' || sql || '$arg2$)'; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_get_seq_list.sql b/sql/londiste/functions/londiste.provider_get_seq_list.sql new file mode 100644 index 00000000..3c053fca --- /dev/null +++ b/sql/londiste/functions/londiste.provider_get_seq_list.sql @@ -0,0 +1,17 @@ + +create or replace function londiste.provider_get_seq_list(i_queue_name text) +returns setof text as $$ +declare + rec record; +begin + for rec in + select seq_name from londiste.provider_seq + where queue_name = i_queue_name + order by nr + loop + return next rec.seq_name; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_get_table_list.sql b/sql/londiste/functions/londiste.provider_get_table_list.sql new file mode 100644 index 00000000..9627802c --- /dev/null +++ b/sql/londiste/functions/londiste.provider_get_table_list.sql @@ -0,0 +1,18 @@ + +create or replace function londiste.provider_get_table_list(i_queue text) +returns setof londiste.ret_provider_table_list as $$ +declare + rec londiste.ret_provider_table_list%rowtype; +begin + for rec in + select table_name, trigger_name + from londiste.provider_table + where queue_name = i_queue + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_notify_change.sql b/sql/londiste/functions/londiste.provider_notify_change.sql new file mode 100644 index 00000000..65505fb0 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_notify_change.sql @@ -0,0 +1,26 @@ + +create or replace function londiste.provider_notify_change(i_queue_name text) +returns integer as $$ +declare + res text; + tbl record; +begin + res := ''; + for tbl in + select table_name from londiste.provider_table + where queue_name = i_queue_name + order by nr + loop + if res = '' then + res := tbl.table_name; + else + res := res || ',' || tbl.table_name; + end if; + end loop; + + perform pgq.insert_event(i_queue_name, 'T', res); + + return 1; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.provider_refresh_trigger.sql b/sql/londiste/functions/londiste.provider_refresh_trigger.sql new file mode 100644 index 00000000..fe361c19 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_refresh_trigger.sql @@ -0,0 +1,44 @@ + +create or replace function londiste.provider_refresh_trigger( + i_queue_name text, + i_table_name text, + i_col_types text +) returns integer strict as $$ +declare + t_name text; + tbl_oid oid; +begin + select trigger_name into t_name + from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'table not found'; + end if; + + tbl_oid := londiste.find_table_oid(i_table_name); + perform 1 from pg_trigger + where tgrelid = tbl_oid + and tgname = t_name; + if found then + execute 'drop trigger ' || t_name || ' on ' || i_table_name; + end if; + + perform londiste.provider_create_trigger(i_queue_name, i_table_name, i_col_types); + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.provider_refresh_trigger( + i_queue_name text, + i_table_name text +) returns integer strict as $$ +begin + return londiste.provider_refresh_trigger(i_queue_name, i_table_name, + londiste.find_column_types(i_table_name)); +end; +$$ language plpgsql security definer; + + + diff --git a/sql/londiste/functions/londiste.provider_remove_seq.sql b/sql/londiste/functions/londiste.provider_remove_seq.sql new file mode 100644 index 00000000..47754b84 --- /dev/null +++ b/sql/londiste/functions/londiste.provider_remove_seq.sql @@ -0,0 +1,26 @@ + +create or replace function londiste.provider_remove_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + -- check if linked queue + link := londiste.link_source(i_queue_name); + if link is not null then + raise exception 'Linked queue, cannot modify'; + end if; + + delete from londiste.provider_seq + where queue_name = i_queue_name + and seq_name = i_seq_name; + if not found then + raise exception 'seq not attached'; + end if; + + perform londiste.provider_notify_change(i_queue_name); + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.provider_remove_table.sql b/sql/londiste/functions/londiste.provider_remove_table.sql new file mode 100644 index 00000000..6143bb2c --- /dev/null +++ b/sql/londiste/functions/londiste.provider_remove_table.sql @@ -0,0 +1,30 @@ + +create or replace function londiste.provider_remove_table( + i_queue_name text, + i_table_name text +) returns integer as $$ +declare + tgname text; +begin + if londiste.link_source(i_queue_name) is not null then + raise exception 'Linked queue, manipulation not allowed'; + end if; + + select trigger_name into tgname from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'no such table registered'; + end if; + + execute 'drop trigger ' || tgname || ' on ' || i_table_name; + + delete from londiste.provider_table + where queue_name = i_queue_name + and table_name = i_table_name; + + return 1; +end; +$$ language plpgsql security definer; + + diff --git a/sql/londiste/functions/londiste.set_last_tick.sql b/sql/londiste/functions/londiste.set_last_tick.sql new file mode 100644 index 00000000..61378a84 --- /dev/null +++ b/sql/londiste/functions/londiste.set_last_tick.sql @@ -0,0 +1,18 @@ + +create or replace function londiste.set_last_tick( + i_consumer text, + i_tick_id bigint) +returns integer as $$ +begin + update londiste.completed + set last_tick_id = i_tick_id + where consumer_id = i_consumer; + if not found then + insert into londiste.completed (consumer_id, last_tick_id) + values (i_consumer, i_tick_id); + end if; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_add_seq.sql b/sql/londiste/functions/londiste.subscriber_add_seq.sql new file mode 100644 index 00000000..c144e47c --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_add_seq.sql @@ -0,0 +1,23 @@ + +create or replace function londiste.subscriber_add_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + insert into londiste.subscriber_seq (queue_name, seq_name) + values (i_queue_name, i_seq_name); + + -- update linked queue if needed + link := londiste.link_dest(i_queue_name); + if link is not null then + insert into londiste.provider_seq + (queue_name, seq_name) + values (link, i_seq_name); + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_add_table.sql b/sql/londiste/functions/londiste.subscriber_add_table.sql new file mode 100644 index 00000000..d5a73314 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_add_table.sql @@ -0,0 +1,14 @@ + +create or replace function londiste.subscriber_add_table( + i_queue_name text, i_table text) +returns integer as $$ +begin + insert into londiste.subscriber_table (queue_name, table_name) + values (i_queue_name, i_table); + + -- linked queue is updated, when the table is copied + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_get_seq_list.sql b/sql/londiste/functions/londiste.subscriber_get_seq_list.sql new file mode 100644 index 00000000..1d218f48 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_get_seq_list.sql @@ -0,0 +1,17 @@ + +create or replace function londiste.subscriber_get_seq_list(i_queue_name text) +returns setof text as $$ +declare + rec record; +begin + for rec in + select seq_name from londiste.subscriber_seq + where queue_name = i_queue_name + order by nr + loop + return next rec.seq_name; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_get_table_list.sql b/sql/londiste/functions/londiste.subscriber_get_table_list.sql new file mode 100644 index 00000000..4a17d5da --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_get_table_list.sql @@ -0,0 +1,35 @@ + +create or replace function londiste.subscriber_get_table_list(i_queue_name text) +returns setof londiste.ret_subscriber_table as $$ +declare + rec londiste.ret_subscriber_table%rowtype; +begin + for rec in + select table_name, merge_state, snapshot, trigger_name + from londiste.subscriber_table + where queue_name = i_queue_name + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +-- compat +create or replace function londiste.get_table_state(i_queue text) +returns setof londiste.subscriber_table as $$ +declare + rec londiste.subscriber_table%rowtype; +begin + for rec in + select * from londiste.subscriber_table + where queue_name = i_queue + order by nr + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_remove_seq.sql b/sql/londiste/functions/londiste.subscriber_remove_seq.sql new file mode 100644 index 00000000..f8715a49 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_remove_seq.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.subscriber_remove_seq( + i_queue_name text, i_seq_name text) +returns integer as $$ +declare + link text; +begin + delete from londiste.subscriber_seq + where queue_name = i_queue_name + and seq_name = i_seq_name; + if not found then + raise exception 'no such seq?'; + end if; + + -- update linked queue if needed + link := londiste.link_dest(i_queue_name); + if link is not null then + delete from londiste.provider_seq + where queue_name = link + and seq_name = i_seq_name; + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/londiste/functions/londiste.subscriber_remove_table.sql b/sql/londiste/functions/londiste.subscriber_remove_table.sql new file mode 100644 index 00000000..49af4053 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_remove_table.sql @@ -0,0 +1,27 @@ + +create or replace function londiste.subscriber_remove_table( + i_queue_name text, i_table text) +returns integer as $$ +declare + link text; +begin + delete from londiste.subscriber_table + where queue_name = i_queue_name + and table_name = i_table; + if not found then + raise exception 'no such table'; + end if; + + -- sync link + link := londiste.link_dest(i_queue_name); + if link is not null then + delete from londiste.provider_table + where queue_name = link + and table_name = i_table; + perform londiste.provider_notify_change(link); + end if; + + return 0; +end; +$$ language plpgsql; + diff --git a/sql/londiste/functions/londiste.subscriber_set_table_state.sql b/sql/londiste/functions/londiste.subscriber_set_table_state.sql new file mode 100644 index 00000000..cab12444 --- /dev/null +++ b/sql/londiste/functions/londiste.subscriber_set_table_state.sql @@ -0,0 +1,58 @@ + +create or replace function londiste.subscriber_set_table_state( + i_queue_name text, + i_table_name text, + i_snapshot text, + i_merge_state text) +returns integer as $$ +declare + link text; + ok integer; +begin + update londiste.subscriber_table + set snapshot = i_snapshot, + merge_state = i_merge_state + where queue_name = i_queue_name + and table_name = i_table_name; + if not found then + raise exception 'no such table'; + end if; + + -- sync link state also + link := londiste.link_dest(i_queue_name); + if link then + select * from londiste.provider_table + where queue_name = linkdst + and table_name = i_table_name; + if found then + if i_merge_state is null or i_merge_state <> 'ok' then + delete from londiste.provider_table + where queue_name = link + and table_name = i_table_name; + perform londiste.notify_change(link); + end if; + else + if i_merge_state = 'ok' then + insert into londiste.provider_table (queue_name, table_name) + values (link, i_table_name); + perform londiste.notify_change(link); + end if; + end if; + end if; + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function londiste.set_table_state( + i_queue_name text, + i_table_name text, + i_snapshot text, + i_merge_state text) +returns integer as $$ +begin + return londiste.subscriber_set_table_state(i_queue_name, i_table_name, i_snapshot, i_merge_state); +end; +$$ language plpgsql security definer; + + diff --git a/sql/londiste/sql/londiste_denytrigger.sql b/sql/londiste/sql/londiste_denytrigger.sql new file mode 100644 index 00000000..dad81ffc --- /dev/null +++ b/sql/londiste/sql/londiste_denytrigger.sql @@ -0,0 +1,19 @@ + +create table denytest ( val integer); +insert into denytest values (1); +create trigger xdeny after insert or update or delete +on denytest for each row execute procedure londiste.deny_trigger(); + +insert into denytest values (2); +update denytest set val = 2; +delete from denytest; + +select londiste.disable_deny_trigger(true); +update denytest set val = 2; +select londiste.disable_deny_trigger(true); +update denytest set val = 2; +select londiste.disable_deny_trigger(false); +update denytest set val = 2; +select londiste.disable_deny_trigger(false); +update denytest set val = 2; + diff --git a/sql/londiste/sql/londiste_install.sql b/sql/londiste/sql/londiste_install.sql new file mode 100644 index 00000000..4637659f --- /dev/null +++ b/sql/londiste/sql/londiste_install.sql @@ -0,0 +1,8 @@ +\set ECHO off +set log_error_verbosity = 'terse'; +\i ../txid/txid.sql +\i ../pgq/pgq.sql +\i ../logtriga/logtriga.sql +\i londiste.sql +\set ECHO all + diff --git a/sql/londiste/sql/londiste_provider.sql b/sql/londiste/sql/londiste_provider.sql new file mode 100644 index 00000000..74075383 --- /dev/null +++ b/sql/londiste/sql/londiste_provider.sql @@ -0,0 +1,68 @@ + +set client_min_messages = 'warning'; + +-- +-- tables +-- +create table testdata ( + id serial primary key, + data text +); +create table testdata_nopk ( + id serial, + data text +); + +select londiste.provider_add_table('pqueue', 'public.testdata_nopk'); +select londiste.provider_add_table('pqueue', 'public.testdata'); + +select pgq.create_queue('pqueue'); +select londiste.provider_add_table('pqueue', 'public.testdata'); +select londiste.provider_add_table('pqueue', 'public.testdata'); + +select londiste.provider_refresh_trigger('pqueue', 'public.testdata'); + +select * from londiste.provider_get_table_list('pqueue'); + +select londiste.provider_remove_table('pqueue', 'public.nonexist'); +select londiste.provider_remove_table('pqueue', 'public.testdata'); + +select * from londiste.provider_get_table_list('pqueue'); + +-- +-- seqs +-- + +select * from londiste.provider_get_seq_list('pqueue'); +select londiste.provider_add_seq('pqueue', 'public.no_seq'); +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.provider_get_seq_list('pqueue'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.provider_get_seq_list('pqueue'); + +-- +-- linked queue +-- +select londiste.provider_add_table('pqueue', 'public.testdata'); +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); + + +select londiste.provider_add_table('pqueue', 'public.testdata'); +select londiste.provider_remove_table('pqueue', 'public.testdata'); + +select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.provider_remove_seq('pqueue', 'public.testdata_seq'); + +-- +-- cleanup +-- + +delete from londiste.link; +drop table testdata; +drop table testdata_nopk; +delete from londiste.provider_seq; +delete from londiste.provider_table; +select pgq.drop_queue('pqueue'); + diff --git a/sql/londiste/sql/londiste_subscriber.sql b/sql/londiste/sql/londiste_subscriber.sql new file mode 100644 index 00000000..0583a395 --- /dev/null +++ b/sql/londiste/sql/londiste_subscriber.sql @@ -0,0 +1,53 @@ + +set client_min_messages = 'warning'; + +create table testdata ( + id serial primary key, + data text +); + +-- +-- tables +-- + +select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + +select pgq.create_queue('pqueue'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +select londiste.subscriber_add_table('pqueue', 'public.testdata'); + +select * from londiste.subscriber_get_table_list('pqueue'); + +select londiste.subscriber_remove_table('pqueue', 'public.nonexist'); +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + +select * from londiste.subscriber_get_table_list('pqueue'); + +-- +-- seqs +-- + +select * from londiste.subscriber_get_seq_list('pqueue'); +select londiste.subscriber_add_seq('pqueue', 'public.no_seq'); +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.subscriber_get_seq_list('pqueue'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq'); +select * from londiste.subscriber_get_seq_list('pqueue'); + +-- +-- linked queue +-- +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +insert into londiste.link (source, dest) values ('mqueue', 'pqueue'); + + +select londiste.subscriber_add_table('pqueue', 'public.testdata'); +select londiste.subscriber_remove_table('pqueue', 'public.testdata'); + +select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq'); +select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq'); + + diff --git a/sql/londiste/structure/tables.sql b/sql/londiste/structure/tables.sql new file mode 100644 index 00000000..83ac24f4 --- /dev/null +++ b/sql/londiste/structure/tables.sql @@ -0,0 +1,53 @@ +set default_with_oids = 'off'; + +create schema londiste; +grant usage on schema londiste to public; + +create table londiste.provider_table ( + nr serial not null, + queue_name text not null, + table_name text not null, + trigger_name text, + primary key (queue_name, table_name) +); + +create table londiste.provider_seq ( + nr serial not null, + queue_name text not null, + seq_name text not null, + primary key (queue_name, seq_name) +); + +create table londiste.completed ( + consumer_id text not null, + last_tick_id bigint not null, + + primary key (consumer_id) +); + +create table londiste.link ( + source text not null, + dest text not null, + primary key (source), + unique (dest) +); + +create table londiste.subscriber_table ( + nr serial not null, + queue_name text not null, + table_name text not null, + snapshot text, + merge_state text, + trigger_name text, + + primary key (queue_name, table_name) +); + +create table londiste.subscriber_seq ( + nr serial not null, + queue_name text not null, + seq_name text not null, + + primary key (queue_name, seq_name) +); + diff --git a/sql/londiste/structure/types.sql b/sql/londiste/structure/types.sql new file mode 100644 index 00000000..e5d64655 --- /dev/null +++ b/sql/londiste/structure/types.sql @@ -0,0 +1,13 @@ + +create type londiste.ret_provider_table_list as ( + table_name text, + trigger_name text +); + +create type londiste.ret_subscriber_table as ( + table_name text, + merge_state text, + snapshot text, + trigger_name text +); + diff --git a/sql/pgq/Makefile b/sql/pgq/Makefile new file mode 100644 index 00000000..6ea78852 --- /dev/null +++ b/sql/pgq/Makefile @@ -0,0 +1,48 @@ + +DOCS = README.pgq +DATA_built = pgq.sql +DATA = structure/uninstall_pgq.sql + +SRCS = $(wildcard structure/*.sql) \ + $(wildcard functions/*.sql) \ + $(wildcard triggers/*.sql) + +REGRESS = pgq_init logutriga sqltriga +REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql + +PGXS = $(shell pg_config --pgxs) +include $(PGXS) + +NDOC = NaturalDocs +NDOCARGS = -r -o html docs/html -p docs -i docs/sql +CATSQL = ../../scripts/catsql.py + +pgq.sql: $(SRCS) + $(CATSQL) structure/install.sql > $@ + +dox: cleandox + mkdir -p docs/html + mkdir -p docs/sql + $(CATSQL) --ndoc structure/tables.sql structure/types.sql > docs/sql/schema.sql + $(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql + $(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql + $(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql + $(NDOC) $(NDOCARGS) + +cleandox: + rm -rf docs/html docs/Data docs/sql + +clean: cleandox + +test: + #-dropdb pgq + #createdb pgq + #psql -f structure/pgq.sql pgq + make installcheck || { less regression.diffs; exit 1; } + +upload: dox + rsync -az docs/html structure functions data1:public_html/pgq/ + make cleandox + rsync -az catsql.py Makefile docs data1:public_html/pgq/ + make dox + diff --git a/sql/pgq/README.pgq b/sql/pgq/README.pgq new file mode 100644 index 00000000..b8757161 --- /dev/null +++ b/sql/pgq/README.pgq @@ -0,0 +1,19 @@ + +Schema overview +=============== + +pgq.consumer consumer name <> id mapping +pgq.queue queue information +pgq.subscription consumer registrations +pgq.tick snapshots that group events into batches +pgq.retry_queue events to be retried +pgq.failed_queue events that have failed +pgq.event_* data tables + +Random ideas +============ + +- all ticker logic in DB (plpython) +- more funcs in plpython +- insert_event in C (way to get rid of plpython) + diff --git a/sql/pgq/docs/Languages.txt b/sql/pgq/docs/Languages.txt new file mode 100644 index 00000000..aa9ce802 --- /dev/null +++ b/sql/pgq/docs/Languages.txt @@ -0,0 +1,113 @@ +Format: 1.35 + +# This is the Natural Docs languages file for this project. If you change +# anything here, it will apply to THIS PROJECT ONLY. If you'd like to change +# something for all your projects, edit the Languages.txt in Natural Docs' +# Config directory instead. + + +# You can prevent certain file extensions from being scanned like this: +# Ignore Extensions: [extension] [extension] ... + + +#------------------------------------------------------------------------------- +# SYNTAX: +# +# Unlike other Natural Docs configuration files, in this file all comments +# MUST be alone on a line. Some languages deal with the # character, so you +# cannot put comments on the same line as content. +# +# Also, all lists are separated with spaces, not commas, again because some +# languages may need to use them. +# +# Language: [name] +# Alter Language: [name] +# Defines a new language or alters an existing one. Its name can use any +# characters. If any of the properties below have an add/replace form, you +# must use that when using Alter Language. +# +# The language Shebang Script is special. It's entry is only used for +# extensions, and files with those extensions have their shebang (#!) lines +# read to determine the real language of the file. Extensionless files are +# always treated this way. +# +# The language Text File is also special. It's treated as one big comment +# so you can put Natural Docs content in them without special symbols. Also, +# if you don't specify a package separator, ignored prefixes, or enum value +# behavior, it will copy those settings from the language that is used most +# in the source tree. +# +# Extensions: [extension] [extension] ... +# [Add/Replace] Extensions: [extension] [extension] ... +# Defines the file extensions of the language's source files. You can +# redefine extensions found in the main languages file. You can use * to +# mean any undefined extension. +# +# Shebang Strings: [string] [string] ... +# [Add/Replace] Shebang Strings: [string] [string] ... +# Defines a list of strings that can appear in the shebang (#!) line to +# designate that it's part of the language. You can redefine strings found +# in the main languages file. +# +# Ignore Prefixes in Index: [prefix] [prefix] ... +# [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ... +# +# Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ... +# [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ... +# Specifies prefixes that should be ignored when sorting symbols in an +# index. Can be specified in general or for a specific topic type. +# +#------------------------------------------------------------------------------ +# For basic language support only: +# +# Line Comments: [symbol] [symbol] ... +# Defines a space-separated list of symbols that are used for line comments, +# if any. +# +# Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ... +# Defines a space-separated list of symbol pairs that are used for block +# comments, if any. +# +# Package Separator: [symbol] +# Defines the default package separator symbol. The default is a dot. +# +# [Topic Type] Prototype Enders: [symbol] [symbol] ... +# When defined, Natural Docs will attempt to get a prototype from the code +# immediately following the topic type. It stops when it reaches one of +# these symbols. Use \n for line breaks. +# +# Line Extender: [symbol] +# Defines the symbol that allows a prototype to span multiple lines if +# normally a line break would end it. +# +# Enum Values: [global|under type|under parent] +# Defines how enum values are referenced. The default is global. +# global - Values are always global, referenced as 'value'. +# under type - Values are under the enum type, referenced as +# 'package.enum.value'. +# under parent - Values are under the enum's parent, referenced as +# 'package.value'. +# +# Perl Package: [perl package] +# Specifies the Perl package used to fine-tune the language behavior in ways +# too complex to do in this file. +# +#------------------------------------------------------------------------------ +# For full language support only: +# +# Full Language Support: [perl package] +# Specifies the Perl package that has the parsing routines necessary for full +# language support. +# +#------------------------------------------------------------------------------- + +# The following languages are defined in the main file, if you'd like to alter +# them: +# +# Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python, +# PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile, +# ActionScript, ColdFusion, R, Fortran + +# If you add a language that you think would be useful to other developers +# and should be included in Natural Docs by default, please e-mail it to +# languages [at] naturaldocs [dot] org. diff --git a/sql/pgq/docs/Menu.txt b/sql/pgq/docs/Menu.txt new file mode 100644 index 00000000..c4b66684 --- /dev/null +++ b/sql/pgq/docs/Menu.txt @@ -0,0 +1,43 @@ +Format: 1.35 + + +Title: PgQ +SubTitle: Database API + +# You can add a footer to your documentation like this: +# Footer: [text] +# If you want to add a copyright notice, this would be the place to do it. + + +# -------------------------------------------------------------------------- +# +# Cut and paste the lines below to change the order in which your files +# appear on the menu. Don't worry about adding or removing files, Natural +# Docs will take care of that. +# +# You can further organize the menu by grouping the entries. Add a +# "Group: [name] {" line to start a group, and add a "}" to end it. +# +# You can add text and web links to the menu by adding "Text: [text]" and +# "Link: [name] ([URL])" lines, respectively. +# +# The formatting and comments are auto-generated, so don't worry about +# neatness when editing the file. Natural Docs will clean it up the next +# time it is run. When working with groups, just deal with the braces and +# forget about the indentation and comments. +# +# -------------------------------------------------------------------------- + + +File: Public Functions (external.sql) +File: Public Triggers (triggers.sql) +File: Internal Functions (internal.sql) +File: Internal Tables (schema.sql) + +Group: Index { + + Index: Everything + Database Table Index: Database Tables + Function Index: Functions + } # Group: Index + diff --git a/sql/pgq/docs/Topics.txt b/sql/pgq/docs/Topics.txt new file mode 100644 index 00000000..da6181df --- /dev/null +++ b/sql/pgq/docs/Topics.txt @@ -0,0 +1,107 @@ +Format: 1.35 + +# This is the Natural Docs topics file for this project. If you change anything +# here, it will apply to THIS PROJECT ONLY. If you'd like to change something +# for all your projects, edit the Topics.txt in Natural Docs' Config directory +# instead. + + +# If you'd like to prevent keywords from being recognized by Natural Docs, you +# can do it like this: +# Ignore Keywords: [keyword], [keyword], ... +# +# Or you can use the list syntax like how they are defined: +# Ignore Keywords: +# [keyword] +# [keyword], [plural keyword] +# ... + + +#------------------------------------------------------------------------------- +# SYNTAX: +# +# Topic Type: [name] +# Alter Topic Type: [name] +# Creates a new topic type or alters one from the main file. Each type gets +# its own index and behavior settings. Its name can have letters, numbers, +# spaces, and these charaters: - / . ' +# +# Plural: [name] +# Sets the plural name of the topic type, if different. +# +# Keywords: +# [keyword] +# [keyword], [plural keyword] +# ... +# Defines or adds to the list of keywords for the topic type. They may only +# contain letters, numbers, and spaces and are not case sensitive. Plural +# keywords are used for list topics. You can redefine keywords found in the +# main topics file. +# +# Index: [yes|no] +# Whether the topics get their own index. Defaults to yes. Everything is +# included in the general index regardless of this setting. +# +# Scope: [normal|start|end|always global] +# How the topics affects scope. Defaults to normal. +# normal - Topics stay within the current scope. +# start - Topics start a new scope for all the topics beneath it, +# like class topics. +# end - Topics reset the scope back to global for all the topics +# beneath it. +# always global - Topics are defined as global, but do not change the scope +# for any other topics. +# +# Class Hierarchy: [yes|no] +# Whether the topics are part of the class hierarchy. Defaults to no. +# +# Variable Type: [yes|no] +# Whether the topics can be a variable type. Defaults to no. +# +# Page Title If First: [yes|no] +# Whether the topic's title becomes the page title if it's the first one in +# a file. Defaults to no. +# +# Break Lists: [yes|no] +# Whether list topics should be broken into individual topics in the output. +# Defaults to no. +# +# Can Group With: [type], [type], ... +# Defines a list of topic types that this one can possibly be grouped with. +# Defaults to none. +#------------------------------------------------------------------------------- + +# The following topics are defined in the main file, if you'd like to alter +# their behavior or add keywords: +# +# Generic, Class, Interface, Section, File, Group, Function, Variable, +# Property, Type, Constant, Enumeration, Event, Delegate, Macro, +# Database, Database Table, Database View, Database Index, Database +# Cursor, Database Trigger, Cookie, Build Target + +# If you add something that you think would be useful to other developers +# and should be included in Natural Docs by default, please e-mail it to +# topics [at] naturaldocs [dot] org. + + +Topic Type: Schema + + Plural: Schemas + Index: No + Scope: Start + Class Hierarchy: Yes + + Keywords: + schema, schemas + + +Alter Topic Type: Function + + Add Keywords: + public function + internal function + + +Alter Topic Type: File + + Index: No diff --git a/sql/pgq/expected/logutriga.out b/sql/pgq/expected/logutriga.out new file mode 100644 index 00000000..6c7f9b14 --- /dev/null +++ b/sql/pgq/expected/logutriga.out @@ -0,0 +1,22 @@ +create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text) +returns bigint as $$ +begin + raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1; + return 1; +end; +$$ language plpgsql; +create table udata ( + id serial primary key, + txt text, + bin bytea +); +NOTICE: CREATE TABLE will create implicit sequence "udata_id_seq" for serial column "udata.id" +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "udata_pkey" for table "udata" +create trigger utest AFTER insert or update or delete ON udata +for each row execute procedure pgq.logutriga('udata_que'); +insert into udata (txt) values ('text1'); +ERROR: plpython: function "logutriga" failed +DETAIL: exceptions.NameError: global name 'rowdata' is not defined +insert into udata (bin) values (E'bi\tn\\000bin'); +ERROR: plpython: function "logutriga" failed +DETAIL: exceptions.NameError: global name 'rowdata' is not defined diff --git a/sql/pgq/expected/pgq_init.out b/sql/pgq/expected/pgq_init.out new file mode 100644 index 00000000..67b555ca --- /dev/null +++ b/sql/pgq/expected/pgq_init.out @@ -0,0 +1,253 @@ +\set ECHO none +select * from pgq.maint_tables_to_vacuum(); + maint_tables_to_vacuum +------------------------ + pgq.subscription + pgq.queue + pgq.tick + pgq.retry_queue +(4 rows) + +select * from pgq.maint_retry_events(); + maint_retry_events +-------------------- + 0 +(1 row) + +select pgq.create_queue('tmpqueue'); + create_queue +-------------- + 1 +(1 row) + +select pgq.register_consumer('tmpqueue', 'consumer'); + register_consumer +------------------- + 1 +(1 row) + +select pgq.unregister_consumer('tmpqueue', 'consumer'); + unregister_consumer +--------------------- + 1 +(1 row) + +select pgq.drop_queue('tmpqueue'); + drop_queue +------------ + 1 +(1 row) + +select pgq.create_queue('myqueue'); + create_queue +-------------- + 1 +(1 row) + +select pgq.register_consumer('myqueue', 'consumer'); + register_consumer +------------------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 1 +(1 row) + +select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); + queue_name | consumer_name | prev_tick_id | tick_id | lag +------------+---------------+--------------+---------+------------- + myqueue | consumer | 1 | 2 | @ 0.00 secs +(1 row) + +select queue_name from pgq.get_queue_info() limit 0; + queue_name +------------ +(0 rows) + +select queue_name, consumer_name from pgq.get_consumer_info() limit 0; + queue_name | consumer_name +------------+--------------- +(0 rows) + +select pgq.finish_batch(1); + finish_batch +-------------- + 1 +(1 row) + +select pgq.finish_batch(1); +WARNING: finish_batch: batch 1 not found + finish_batch +-------------- + 0 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 2 +(1 row) + +select * from pgq.batch_event_tables(2); + batch_event_tables +-------------------- + pgq.event_2_0 +(1 row) + +select * from pgq.get_batch_events(2); + ev_id | ev_time | ev_txid | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +-------+---------+---------+----------+---------+---------+-----------+-----------+-----------+----------- +(0 rows) + +select pgq.finish_batch(2); + finish_batch +-------------- + 1 +(1 row) + +select pgq.insert_event('myqueue', 'r1', 'data'); + insert_event +-------------- + 1 +(1 row) + +select pgq.insert_event('myqueue', 'r2', 'data'); + insert_event +-------------- + 2 +(1 row) + +select pgq.insert_event('myqueue', 'r3', 'data'); + insert_event +-------------- + 3 +(1 row) + +select pgq.current_event_table('myqueue'); + current_event_table +--------------------- + pgq.event_2_0 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + +select pgq.next_batch('myqueue', 'consumer'); + next_batch +------------ + 3 +(1 row) + +select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); + ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +-------+----------+---------+---------+-----------+-----------+-----------+----------- + 1 | | r1 | data | | | | + 2 | | r2 | data | | | | + 3 | | r3 | data | | | | +(3 rows) + +select * from pgq.failed_event_list('myqueue', 'consumer'); + ev_failed_reason | ev_failed_time | ev_id | ev_time | ev_txid | ev_owner | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 +------------------+----------------+-------+---------+---------+----------+----------+---------+---------+-----------+-----------+-----------+----------- +(0 rows) + +select pgq.event_failed(3, 1, 'failure test'); + event_failed +-------------- + 1 +(1 row) + +select pgq.event_failed(3, 1, 'failure test'); + event_failed +-------------- + 0 +(1 row) + +select pgq.event_retry(3, 2, 0); + event_retry +------------- + 1 +(1 row) + +select pgq.event_retry(3, 2, 0); + event_retry +------------- + 0 +(1 row) + +select pgq.finish_batch(3); + finish_batch +-------------- + 1 +(1 row) + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer'); + ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data +------------------+-------+---------+----------+---------+--------- + failure test | 1 | | 0 | r1 | data +(1 row) + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer', 0, 1); + ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data +------------------+-------+---------+----------+---------+--------- +(0 rows) + +select * from pgq.failed_event_count('myqueue', 'consumer'); + failed_event_count +-------------------- + 1 +(1 row) + +select * from pgq.failed_event_delete('myqueue', 'consumer', 0); +ERROR: event not found +select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, + 'rawtest', 'data', null, null, null, null); + event_retry_raw +----------------- + 666 +(1 row) + +select pgq.ticker(); + ticker +-------- + 1 +(1 row) + diff --git a/sql/pgq/expected/sqltriga.out b/sql/pgq/expected/sqltriga.out new file mode 100644 index 00000000..8e396212 --- /dev/null +++ b/sql/pgq/expected/sqltriga.out @@ -0,0 +1,86 @@ +-- start testing +create table rtest ( + id integer primary key, + dat text +); +NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest" +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que'); +-- simple test +insert into rtest values (1, 'value1'); +NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 'value2'; +NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 'value2'; +NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2'); +insert into rtest values (1, '666', 'newdat'); +NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 5, dat2 = 'newdat2'; +NOTICE: insert_event(que2, U, dat='5' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 6; +NOTICE: insert_event(que2, U, dat='6' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que2, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test hashed pkey +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)'); +insert into rtest values (1, '666', 'newdat'); +NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 5, dat2 = 'newdat2'; +NOTICE: insert_event(que2, U, dat='5' where dat='5' and hashtext(dat) = hashtext('5'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat = 6; +NOTICE: insert_event(que2, U, dat='6' where dat='6' and hashtext(dat) = hashtext('6'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest; +NOTICE: insert_event(que2, D, dat='6' and hashtext(dat) = hashtext('6'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que3'); +insert into rtest values (1, 0, 'non-null'); +NOTICE: insert_event(que3, I, (dat,id) values ('0','1'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +insert into rtest values (2, 0, NULL); +NOTICE: insert_event(que3, I, (dat,id) values ('0','2'), public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = 'non-null2' where id=1; +NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = NULL where id=1; +NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +update rtest set dat2 = 'new-nonnull' where id=2; +NOTICE: insert_event(que3, U, id='2' where id='2', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest where id=1; +NOTICE: insert_event(que3, D, id='1', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" +delete from rtest where id=2; +NOTICE: insert_event(que3, D, id='2', public.rtest) +CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" diff --git a/sql/pgq/functions/pgq.batch_event_sql.sql b/sql/pgq/functions/pgq.batch_event_sql.sql new file mode 100644 index 00000000..825de5b6 --- /dev/null +++ b/sql/pgq/functions/pgq.batch_event_sql.sql @@ -0,0 +1,106 @@ +create or replace function pgq.batch_event_sql(x_batch_id bigint) +returns text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.batch_event_sql(1) +-- Creates SELECT statement that fetches events for this batch. +-- +-- Parameters: +-- x_batch_id - ID of a active batch. +-- +-- Returns: +-- SQL statement +-- ---------------------------------------------------------------------- +declare + rec record; + sql text; + tbl text; + arr text; + part text; + select_fields text; + retry_expr text; + batch record; +begin + select s.sub_last_tick, s.sub_next_tick, s.sub_id, s.sub_queue, + get_snapshot_xmax(last.tick_snapshot) as tx_start, + get_snapshot_xmax(cur.tick_snapshot) as tx_end, + last.tick_snapshot as last_snapshot, + cur.tick_snapshot as cur_snapshot + into batch + from pgq.subscription s, pgq.tick last, pgq.tick cur + where s.sub_batch = x_batch_id + and last.tick_queue = s.sub_queue + and last.tick_id = s.sub_last_tick + and cur.tick_queue = s.sub_queue + and cur.tick_id = s.sub_next_tick; + if not found then + raise exception 'batch not found'; + end if; + + -- load older transactions + arr := ''; + for rec in + -- active tx-es in prev_snapshot that were committed in cur_snapshot + select id1 from + get_snapshot_active(batch.last_snapshot) id1 left join + get_snapshot_active(batch.cur_snapshot) id2 on (id1 = id2) + where id2 is null + order by 1 desc + loop + -- try to avoid big IN expression, so try to include nearby + -- tx'es into range + if batch.tx_start - 100 <= rec.id1 then + batch.tx_start := rec.id1; + else + if arr = '' then + arr := rec.id1; + else + arr := arr || ',' || rec.id1; + end if; + end if; + end loop; + + -- must match pgq.event_template + select_fields := 'select ev_id, ev_time, ev_txid, ev_retry, ev_type,' + || ' ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4'; + retry_expr := ' and (ev_owner is null or ev_owner = ' + || batch.sub_id || ')'; + + -- now generate query that goes over all potential tables + sql := ''; + for rec in + select xtbl from pgq.batch_event_tables(x_batch_id) xtbl + loop + tbl := rec.xtbl; + -- this gets newer queries that definitely are not in prev_snapshot + part := select_fields + || ' from pgq.tick cur, pgq.tick last, ' || tbl || ' ev ' + || ' where cur.tick_id = ' || batch.sub_next_tick + || ' and cur.tick_queue = ' || batch.sub_queue + || ' and last.tick_id = ' || batch.sub_last_tick + || ' and last.tick_queue = ' || batch.sub_queue + || ' and ev.ev_txid >= ' || batch.tx_start + || ' and ev.ev_txid <= ' || batch.tx_end + || ' and txid_in_snapshot(ev.ev_txid, cur.tick_snapshot)' + || ' and not txid_in_snapshot(ev.ev_txid, last.tick_snapshot)' + || retry_expr; + -- now include older tx-es, that were ongoing + -- at the time of prev_snapshot + if arr <> '' then + part := part || ' union all ' + || select_fields || ' from ' || tbl || ' ev ' + || ' where ev.ev_txid in (' || arr || ')' + || retry_expr; + end if; + if sql = '' then + sql := part; + else + sql := sql || ' union all ' || part; + end if; + end loop; + if sql = '' then + raise exception 'could not construct sql for batch %', x_batch_id; + end if; + return sql || ' order by 1'; +end; +$$ language plpgsql; -- no perms needed + diff --git a/sql/pgq/functions/pgq.batch_event_tables.sql b/sql/pgq/functions/pgq.batch_event_tables.sql new file mode 100644 index 00000000..f6bdc309 --- /dev/null +++ b/sql/pgq/functions/pgq.batch_event_tables.sql @@ -0,0 +1,67 @@ +create or replace function pgq.batch_event_tables(x_batch_id bigint) +returns setof text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.batch_event_tables(1) +-- +-- Returns set of table names where this batch events may reside. +-- +-- Parameters: +-- x_batch_id - ID of a active batch. +-- ---------------------------------------------------------------------- +declare + nr integer; + tbl text; + use_prev integer; + use_next integer; + batch record; +begin + select + get_snapshot_xmin(last.tick_snapshot) as tx_min, -- absolute minimum + get_snapshot_xmax(cur.tick_snapshot) as tx_max, -- absolute maximum + q.queue_data_pfx, q.queue_ntables, + q.queue_cur_table, q.queue_switch_step1, q.queue_switch_step2 + into batch + from pgq.tick last, pgq.tick cur, pgq.subscription s, pgq.queue q + where cur.tick_id = s.sub_next_tick + and cur.tick_queue = s.sub_queue + and last.tick_id = s.sub_last_tick + and last.tick_queue = s.sub_queue + and s.sub_batch = x_batch_id + and q.queue_id = s.sub_queue; + if not found then + raise exception 'Cannot find data for batch %', x_batch_id; + end if; + + -- if its definitely not in one or other, look into both + if batch.tx_max < batch.queue_switch_step1 then + use_prev := 1; + use_next := 0; + elsif batch.queue_switch_step2 is not null + and (batch.tx_min > batch.queue_switch_step2) + then + use_prev := 0; + use_next := 1; + else + use_prev := 1; + use_next := 1; + end if; + + if use_prev then + nr := batch.queue_cur_table - 1; + if nr < 0 then + nr := batch.queue_ntables - 1; + end if; + tbl := batch.queue_data_pfx || '_' || nr; + return next tbl; + end if; + + if use_next then + tbl := batch.queue_data_pfx || '_' || batch.queue_cur_table; + return next tbl; + end if; + + return; +end; +$$ language plpgsql; -- no perms needed + + diff --git a/sql/pgq/functions/pgq.create_queue.sql b/sql/pgq/functions/pgq.create_queue.sql new file mode 100644 index 00000000..927a48cb --- /dev/null +++ b/sql/pgq/functions/pgq.create_queue.sql @@ -0,0 +1,71 @@ +create or replace function pgq.create_queue(i_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.create_queue(1) +-- +-- Creates new queue with given name. +-- +-- Returns: +-- 0 - queue already exists +-- 1 - queue created +-- ---------------------------------------------------------------------- +declare + tblpfx text; + tblname text; + idxpfx text; + idxname text; + sql text; + id integer; + tick_seq text; + ev_seq text; + n_tables integer; +begin + if i_queue_name is null then + raise exception 'Invalid NULL value'; + end if; + + -- check if exists + perform 1 from pgq.queue where queue_name = i_queue_name; + if found then + return 0; + end if; + + -- insert event + id := nextval('pgq.queue_queue_id_seq'); + tblpfx := 'pgq.event_' || id; + idxpfx := 'event_' || id; + tick_seq := 'pgq.event_' || id || '_tick_seq'; + ev_seq := 'pgq.event_' || id || '_id_seq'; + insert into pgq.queue (queue_id, queue_name, + queue_data_pfx, queue_event_seq, queue_tick_seq) + values (id, i_queue_name, tblpfx, ev_seq, tick_seq); + + select queue_ntables into n_tables from pgq.queue + where queue_id = id; + + -- create seqs + execute 'CREATE SEQUENCE ' || tick_seq; + execute 'CREATE SEQUENCE ' || ev_seq; + + -- create data tables + execute 'CREATE TABLE ' || tblpfx || ' () ' + || ' INHERITS (pgq.event_template)'; + for i in 0 .. (n_tables - 1) loop + tblname := tblpfx || '_' || i; + idxname := idxpfx || '_' || i; + execute 'CREATE TABLE ' || tblname || ' () ' + || ' INHERITS (' || tblpfx || ')'; + execute 'ALTER TABLE ' || tblname || ' ALTER COLUMN ev_id ' + || ' SET DEFAULT nextval(' || quote_literal(ev_seq) || ')'; + execute 'create index ' || idxname || '_txid_idx on ' + || tblname || ' (ev_txid)'; + end loop; + + perform pgq.grant_perms(i_queue_name); + + perform pgq.ticker(i_queue_name); + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.current_event_table.sql b/sql/pgq/functions/pgq.current_event_table.sql new file mode 100644 index 00000000..b20dac03 --- /dev/null +++ b/sql/pgq/functions/pgq.current_event_table.sql @@ -0,0 +1,25 @@ +create or replace function pgq.current_event_table(x_queue_name text) +returns text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.current_event_table(1) +-- +-- Return active event table for particular queue. +-- +-- Note: +-- The result is valid only during current transaction. +-- +-- Parameters: +-- x_queue_name - Queue name. +-- ---------------------------------------------------------------------- +declare + res text; +begin + select queue_data_pfx || '_' || queue_cur_table into res + from pgq.queue where queue_name = x_queue_name; + if not found then + raise exception 'Event queue not found'; + end if; + return res; +end; +$$ language plpgsql; -- no perms needed + diff --git a/sql/pgq/functions/pgq.drop_queue.sql b/sql/pgq/functions/pgq.drop_queue.sql new file mode 100644 index 00000000..3819a914 --- /dev/null +++ b/sql/pgq/functions/pgq.drop_queue.sql @@ -0,0 +1,56 @@ +create or replace function pgq.drop_queue(x_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.drop_queue(1) +-- +-- Drop queue and all associated tables. +-- No consumers must be listening on the queue. +-- +-- ---------------------------------------------------------------------- +declare + tblname text; + q record; + num integer; +begin + -- check ares + if x_queue_name is null then + raise exception 'Invalid NULL value'; + end if; + + -- check if exists + select * into q from pgq.queue + where queue_name = x_queue_name; + if not found then + raise exception 'No such event queue'; + end if; + + -- check if no consumers + select count(*) into num from pgq.subscription + where sub_queue = q.queue_id; + if num > 0 then + raise exception 'cannot drop queue, consumers still attached'; + end if; + + -- drop data tables + for i in 0 .. (q.queue_ntables - 1) loop + tblname := q.queue_data_pfx || '_' || i; + execute 'DROP TABLE ' || tblname; + end loop; + execute 'DROP TABLE ' || q.queue_data_pfx; + + -- delete ticks + delete from pgq.tick where tick_queue = q.queue_id; + + -- drop seqs + -- FIXME: any checks needed here? + execute 'DROP SEQUENCE ' || q.queue_tick_seq; + execute 'DROP SEQUENCE ' || q.queue_event_seq; + + -- delete event + delete from pgq.queue + where queue_name = x_queue_name; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_failed.sql b/sql/pgq/functions/pgq.event_failed.sql new file mode 100644 index 00000000..fbcf3c86 --- /dev/null +++ b/sql/pgq/functions/pgq.event_failed.sql @@ -0,0 +1,41 @@ +create or replace function pgq.event_failed( + x_batch_id bigint, + x_event_id bigint, + x_reason text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_failed(3) +-- +-- Copies the event to failed queue. Can be looked later. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - Event id +-- x_reason - Text to associate with event. +-- +-- Returns: +-- 0 if event was already in queue, 1 otherwise. +-- ---------------------------------------------------------------------- +begin + insert into pgq.failed_queue (ev_failed_reason, ev_failed_time, + ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + select x_reason, now(), + ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0), + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 + from pgq.get_batch_events(x_batch_id), + pgq.subscription + where sub_batch = x_batch_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + return 1; + +-- dont worry if the event is already in queue +exception + when unique_violation then + return 0; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_retry.sql b/sql/pgq/functions/pgq.event_retry.sql new file mode 100644 index 00000000..c0259745 --- /dev/null +++ b/sql/pgq/functions/pgq.event_retry.sql @@ -0,0 +1,68 @@ +create or replace function pgq.event_retry( + x_batch_id bigint, + x_event_id bigint, + x_retry_time timestamptz) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry(3) +-- +-- Put the event into retry queue, to be processed later again. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - event id +-- x_retry_time - Time when the event should be put back into queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +begin + insert into pgq.retry_queue (ev_retry_after, + ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + select x_retry_time, + ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0) + 1, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4 + from pgq.get_batch_events(x_batch_id), + pgq.subscription + where sub_batch = x_batch_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + return 1; + +-- dont worry if the event is already in queue +exception + when unique_violation then + return 0; +end; +$$ language plpgsql security definer; + + +create or replace function pgq.event_retry( + x_batch_id bigint, + x_event_id bigint, + x_retry_seconds integer) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry(3) +-- +-- Put the event into retry queue, to be processed later again. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- x_event_id - event id +-- x_retry_seconds - Time when the event should be put back into queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + new_retry timestamptz; +begin + new_retry := current_timestamp + ((x_retry_seconds || ' seconds')::interval); + return pgq.event_retry(x_batch_id, x_event_id, new_retry); +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.event_retry_raw.sql b/sql/pgq/functions/pgq.event_retry_raw.sql new file mode 100644 index 00000000..3a2efb29 --- /dev/null +++ b/sql/pgq/functions/pgq.event_retry_raw.sql @@ -0,0 +1,66 @@ +create or replace function pgq.event_retry_raw( + x_queue text, + x_consumer text, + x_retry_after timestamptz, + x_ev_id bigint, + x_ev_time timestamptz, + x_ev_retry integer, + x_ev_type text, + x_ev_data text, + x_ev_extra1 text, + x_ev_extra2 text, + x_ev_extra3 text, + x_ev_extra4 text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.event_retry_raw(12) +-- +-- Allows full control over what goes to retry queue. +-- +-- Parameters: +-- x_queue - name of the queue +-- x_consumer - name of the consumer +-- x_retry_after - when the event should be processed again +-- x_ev_id - event id +-- x_ev_time - creation time +-- x_ev_retry - retry count +-- x_ev_type - user data +-- x_ev_data - user data +-- x_ev_extra1 - user data +-- x_ev_extra2 - user data +-- x_ev_extra3 - user data +-- x_ev_extra4 - user data +-- +-- Returns: +-- Event ID. +-- ---------------------------------------------------------------------- +declare + q record; + id bigint; +begin + select sub_id, queue_event_seq into q + from pgq.consumer, pgq.queue, pgq.subscription + where queue_name = x_queue + and co_name = x_consumer + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'consumer not registered'; + end if; + + id := x_ev_id; + if id is null then + id := nextval(q.queue_event_seq); + end if; + + insert into pgq.retry_queue (ev_retry_after, + ev_id, ev_time, ev_owner, ev_retry, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4) + values (x_retry_after, x_ev_id, x_ev_time, q.sub_id, x_ev_retry, + x_ev_type, x_ev_data, x_ev_extra1, x_ev_extra2, + x_ev_extra3, x_ev_extra4); + + return id; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.failed_queue.sql b/sql/pgq/functions/pgq.failed_queue.sql new file mode 100644 index 00000000..0ae02043 --- /dev/null +++ b/sql/pgq/functions/pgq.failed_queue.sql @@ -0,0 +1,201 @@ + +create or replace function pgq.failed_event_list( + x_queue_name text, + x_consumer_name text) +returns setof pgq.failed_queue as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_list(2) +-- +-- Get list of all failed events for one consumer. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- +-- Returns: +-- List of failed events. +-- ---------------------------------------------------------------------- +declare + rec pgq.failed_queue%rowtype; +begin + for rec in + select fq.* + from pgq.failed_queue fq, pgq.consumer, + pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id + and ev_owner = sub_id + order by ev_id + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_list( + x_queue_name text, + x_consumer_name text, + x_count integer, + x_offset integer) +returns setof pgq.failed_queue as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_list(4) +-- +-- Get list of failed events, from offset and specific count. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_count - Max amount of events to fetch +-- x_offset - From this offset +-- +-- Returns: +-- List of failed events. +-- ---------------------------------------------------------------------- +declare + rec pgq.failed_queue%rowtype; +begin + for rec in + select fq.* + from pgq.failed_queue fq, pgq.consumer, + pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id + and ev_owner = sub_id + order by ev_id + limit x_count + offset x_offset + loop + return next rec; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_count( + x_queue_name text, + x_consumer_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_count(2) +-- +-- Get size of failed event queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- +-- Returns: +-- Number of failed events in failed event queue. +-- ---------------------------------------------------------------------- +declare + ret integer; +begin + select count(1) into ret + from pgq.failed_queue, pgq.consumer, pgq.queue, pgq.subscription + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_queue = queue_id + and sub_consumer = co_id + and ev_owner = sub_id; + return ret; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_delete( + x_queue_name text, + x_consumer_name text, + x_event_id bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_delete(3) +-- +-- Delete specific event from failed event queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_event_id - Event ID +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'no such queue/consumer'; + end if; + + delete from pgq.failed_queue + where ev_owner = x_sub_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + + return 1; +end; +$$ language plpgsql security definer; + +create or replace function pgq.failed_event_retry( + x_queue_name text, + x_consumer_name text, + x_event_id bigint) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.failed_event_retry(3) +-- +-- Insert specific event from failed queue to main queue. +-- +-- Parameters: +-- x_queue_name - Queue name +-- x_consumer_name - Consumer name +-- x_event_id - Event ID +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + ret bigint; + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where queue_name = x_queue_name + and co_name = x_consumer_name + and sub_consumer = co_id + and sub_queue = queue_id; + if not found then + raise exception 'no such queue/consumer'; + end if; + + select pgq.insert_event_raw(x_queue_name, ev_id, ev_time, + ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4) + into ret + from pgq.failed_queue, pgq.consumer, pgq.queue + where ev_owner = x_sub_id + and ev_id = x_event_id; + if not found then + raise exception 'event not found'; + end if; + + perform pgq.failed_event_delete(x_queue_name, x_consumer_name, x_event_id); + + return ret; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.finish_batch.sql b/sql/pgq/functions/pgq.finish_batch.sql new file mode 100644 index 00000000..6ff4b28f --- /dev/null +++ b/sql/pgq/functions/pgq.finish_batch.sql @@ -0,0 +1,32 @@ + +create or replace function pgq.finish_batch( + x_batch_id bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.finish_batch(1) +-- +-- Closes a batch. No more operations can be done with events +-- of this batch. +-- +-- Parameters: +-- x_batch_id - id of batch. +-- +-- Returns: +-- If batch 1 if batch was found, 0 otherwise. +-- ---------------------------------------------------------------------- +begin + update pgq.subscription + set sub_active = now(), + sub_last_tick = sub_next_tick, + sub_next_tick = null, + sub_batch = null + where sub_batch = x_batch_id; + if not found then + raise warning 'finish_batch: batch % not found', x_batch_id; + return 0; + end if; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_batch_events.sql b/sql/pgq/functions/pgq.get_batch_events.sql new file mode 100644 index 00000000..8166d519 --- /dev/null +++ b/sql/pgq/functions/pgq.get_batch_events.sql @@ -0,0 +1,26 @@ +create or replace function pgq.get_batch_events(x_batch_id bigint) +returns setof pgq.ret_batch_event as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_batch_events(1) +-- +-- Get all events in batch. +-- +-- Parameters: +-- x_batch_id - ID of active batch. +-- +-- Returns: +-- List of events. +-- ---------------------------------------------------------------------- +declare + rec pgq.ret_batch_event%rowtype; + sql text; +begin + sql := pgq.batch_event_sql(x_batch_id); + for rec in execute sql loop + return next rec; + end loop; + return; +end; +$$ language plpgsql; -- no perms needed + + diff --git a/sql/pgq/functions/pgq.get_batch_info.sql b/sql/pgq/functions/pgq.get_batch_info.sql new file mode 100644 index 00000000..617e588d --- /dev/null +++ b/sql/pgq/functions/pgq.get_batch_info.sql @@ -0,0 +1,36 @@ + +create or replace function pgq.get_batch_info(x_batch_id bigint) +returns pgq.ret_batch_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_batch_info(1) +-- +-- Returns detailed info about a batch. +-- +-- Parameters: +-- x_batch_id - id of a active batch. +-- +-- Returns: +-- Info +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_batch_info%rowtype; +begin + select queue_name, co_name, + prev.tick_time as batch_start, + cur.tick_time as batch_end, + sub_last_tick, sub_next_tick, + current_timestamp - cur.tick_time as lag + into ret + from pgq.subscription, pgq.tick cur, pgq.tick prev, + pgq.queue, pgq.consumer + where sub_batch = x_batch_id + and prev.tick_id = sub_last_tick + and prev.tick_queue = sub_queue + and cur.tick_id = sub_next_tick + and cur.tick_queue = sub_queue + and queue_id = sub_queue + and co_id = sub_consumer; + return ret; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_consumer_info.sql b/sql/pgq/functions/pgq.get_consumer_info.sql new file mode 100644 index 00000000..3444421d --- /dev/null +++ b/sql/pgq/functions/pgq.get_consumer_info.sql @@ -0,0 +1,108 @@ + +------------------------------------------------------------------------- +create or replace function pgq.get_consumer_info() +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(0) +-- +-- Desc +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- desc +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; + i record; +begin + for i in select queue_name from pgq.queue order by 1 + loop + for ret in + select * from pgq.get_consumer_info(i.queue_name) + loop + return next ret; + end loop; + end loop; + return; +end; +$$ language plpgsql security definer; + + +------------------------------------------------------------------------- +create or replace function pgq.get_consumer_info(x_queue_name text) +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(1) +-- +-- Desc +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- desc +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; + tmp record; +begin + for tmp in + select queue_name, co_name + from pgq.queue, pgq.consumer, pgq.subscription + where queue_id = sub_queue + and co_id = sub_consumer + and queue_name = x_queue_name + order by 1, 2 + loop + for ret in + select * from pgq.get_consumer_info(tmp.queue_name, tmp.co_name) + loop + return next ret; + end loop; + end loop; + return; +end; +$$ language plpgsql security definer; + + +------------------------------------------------------------------------ +create or replace function pgq.get_consumer_info( + x_queue_name text, + x_consumer_name text) +returns setof pgq.ret_consumer_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_consumer_info(2) +-- +-- Get info about particular consumer on particular queue. +-- +-- Parameters: +-- x_queue_name - name of a queue. +-- x_consumer_name - name of a consumer +-- +-- Returns: +-- info +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_consumer_info%rowtype; +begin + for ret in + select queue_name, co_name, + current_timestamp - tick_time as lag, + current_timestamp - sub_active as last_seen + from pgq.subscription, pgq.tick, pgq.queue, pgq.consumer + where tick_id = sub_last_tick + and queue_id = sub_queue + and tick_queue = sub_queue + and co_id = sub_consumer + and queue_name = x_queue_name + and co_name = x_consumer_name + order by 1,2 + loop + return next ret; + end loop; + return; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.get_queue_info.sql b/sql/pgq/functions/pgq.get_queue_info.sql new file mode 100644 index 00000000..097d1a20 --- /dev/null +++ b/sql/pgq/functions/pgq.get_queue_info.sql @@ -0,0 +1,51 @@ +create or replace function pgq.get_queue_info() +returns setof pgq.ret_queue_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_queue_info(0) +-- +-- Get info about all queues. +-- +-- Returns: +-- List of pgq.ret_queue_info records. +-- ---------------------------------------------------------------------- +declare + qname text; + ret pgq.ret_queue_info%rowtype; +begin + for qname in + select queue_name from pgq.queue order by 1 + loop + select * into ret from pgq.get_queue_info(qname); + return next ret; + end loop; + return; +end; +$$ language plpgsql security definer; + +create or replace function pgq.get_queue_info(qname text) +returns pgq.ret_queue_info as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.get_queue_info(1) +-- +-- Get info about particular queue. +-- +-- Returns: +-- One pgq.ret_queue_info record. +-- ---------------------------------------------------------------------- +declare + ret pgq.ret_queue_info%rowtype; +begin + select queue_name, queue_ntables, queue_cur_table, + queue_rotation_period, queue_switch_time, + queue_external_ticker, + queue_ticker_max_count, queue_ticker_max_lag, + queue_ticker_idle_period, + (select current_timestamp - tick_time + from pgq.tick where tick_queue = queue_id + order by tick_queue desc, tick_id desc limit 1 + ) as ticker_lag + into ret from pgq.queue where queue_name = qname; + return ret; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.grant_perms.sql b/sql/pgq/functions/pgq.grant_perms.sql new file mode 100644 index 00000000..d2c00837 --- /dev/null +++ b/sql/pgq/functions/pgq.grant_perms.sql @@ -0,0 +1,37 @@ +create or replace function pgq.grant_perms(x_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.grant_perms(1) +-- +-- Make event tables readable by public. +-- +-- Parameters: +-- x_queue_name - Name of the queue. +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + q record; + i integer; +begin + select * from pgq.queue into q + where queue_name = x_queue_name; + if not found then + raise exception 'Queue not found'; + end if; + execute 'grant select, update on ' + || q.queue_event_seq || ',' || q.queue_tick_seq + || ' to public'; + execute 'grant select on ' + || q.queue_data_pfx + || ' to public'; + for i in 0 .. q.queue_ntables - 1 loop + execute 'grant select, insert on ' + || q.queue_data_pfx || '_' || i + || ' to public'; + end loop; + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.insert_event.sql b/sql/pgq/functions/pgq.insert_event.sql new file mode 100644 index 00000000..2adfcbc0 --- /dev/null +++ b/sql/pgq/functions/pgq.insert_event.sql @@ -0,0 +1,49 @@ +create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.insert_event(3) +-- +-- Insert a event into queue. +-- +-- Parameters: +-- queue_name - Name of the queue +-- ev_type - User-specified type for the event +-- ev_data - User data for the event +-- +-- Returns: +-- Event ID +-- ---------------------------------------------------------------------- +begin + return pgq.insert_event(queue_name, ev_type, ev_data, null, null, null, null); +end; +$$ language plpgsql; -- event inserting needs no special perms + + + +create or replace function pgq.insert_event( + queue_name text, ev_type text, ev_data text, + ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.insert_event(7) +-- +-- Insert a event into queue with all the extra fields. +-- +-- Parameters: +-- queue_name - Name of the queue +-- ev_type - User-specified type for the event +-- ev_data - User data for the event +-- ev_extra1 - Extra data field for the event +-- ev_extra2 - Extra data field for the event +-- ev_extra3 - Extra data field for the event +-- ev_extra4 - Extra data field for the event +-- +-- Returns: +-- Event ID +-- ---------------------------------------------------------------------- +begin + return pgq.insert_event_raw(queue_name, null, now(), null, null, + ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4); +end; +$$ language plpgsql; -- event inserting needs no special perms + diff --git a/sql/pgq/functions/pgq.insert_event_raw.sql b/sql/pgq/functions/pgq.insert_event_raw.sql new file mode 100644 index 00000000..faca04a2 --- /dev/null +++ b/sql/pgq/functions/pgq.insert_event_raw.sql @@ -0,0 +1,87 @@ +create or replace function pgq.insert_event_raw( + queue_name text, ev_id bigint, ev_time timestamptz, + ev_owner integer, ev_retry integer, ev_type text, ev_data text, + ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text) +returns bigint as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.insert_event_raw(11) +# -- +# -- Actual event insertion. Used also by retry queue maintenance. +# -- +# -- Parameters: +# -- queue_name - Name of the queue +# -- ev_id - Event ID. If NULL, will be taken from seq. +# -- ev_time - Event creation time. +# -- ev_type - user data +# -- ev_data - user data +# -- ev_extra1 - user data +# -- ev_extra2 - user data +# -- ev_extra3 - user data +# -- ev_extra4 - user data +# -- +# -- Returns: +# -- Event ID. +# -- ---------------------------------------------------------------------- + + # load args + queue_name = args[0] + ev_id = args[1] + ev_time = args[2] + ev_owner = args[3] + ev_retry = args[4] + ev_type = args[5] + ev_data = args[6] + ev_extra1 = args[7] + ev_extra2 = args[8] + ev_extra3 = args[9] + ev_extra4 = args[10] + + if not "cf_plan" in SD: + # get current event table + q = "select queue_data_pfx, queue_cur_table, queue_event_seq "\ + " from pgq.queue where queue_name = $1" + SD["cf_plan"] = plpy.prepare(q, ["text"]) + + # get next id + q = "select nextval($1) as id" + SD["seq_plan"] = plpy.prepare(q, ["text"]) + + # get queue config + res = plpy.execute(SD["cf_plan"], [queue_name]) + if len(res) != 1: + plpy.error("Unknown event queue: %s" % (queue_name)) + tbl_prefix = res[0]["queue_data_pfx"] + cur_nr = res[0]["queue_cur_table"] + id_seq = res[0]["queue_event_seq"] + + # get id - bump seq even if id is given + res = plpy.execute(SD['seq_plan'], [id_seq]) + if ev_id is None: + ev_id = res[0]["id"] + + # create plan for insertion + ins_plan = None + ins_key = "ins.%s" % (queue_name) + if ins_key in SD: + nr, ins_plan = SD[ins_key] + if nr != cur_nr: + ins_plan = None + if ins_plan == None: + q = "insert into %s_%s (ev_id, ev_time, ev_owner, ev_retry,"\ + " ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)"\ + " values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" % ( + tbl_prefix, cur_nr) + types = ["int8", "timestamptz", "int4", "int4", "text", + "text", "text", "text", "text", "text"] + ins_plan = plpy.prepare(q, types) + SD[ins_key] = (cur_nr, ins_plan) + + # insert the event + plpy.execute(ins_plan, [ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4]) + + # done + return ev_id + +$$ language plpythonu; -- event inserting needs no special perms + diff --git a/sql/pgq/functions/pgq.maint_retry_events.sql b/sql/pgq/functions/pgq.maint_retry_events.sql new file mode 100644 index 00000000..f3038b86 --- /dev/null +++ b/sql/pgq/functions/pgq.maint_retry_events.sql @@ -0,0 +1,42 @@ +create or replace function pgq.maint_retry_events() +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_retry_events(0) +-- +-- Moves retry events back to main queue. +-- +-- It moves small amount at a time. It should be called +-- until it returns 0 +-- +-- Parameters: +-- arg - desc +-- +-- Returns: +-- Number of events processed. +-- ---------------------------------------------------------------------- +declare + cnt integer; + rec record; +begin + cnt := 0; + for rec in + select pgq.insert_event_raw(queue_name, + ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4), + ev_owner, ev_id + from pgq.retry_queue, pgq.queue, pgq.subscription + where ev_retry_after <= current_timestamp + and sub_id = ev_owner + and queue_id = sub_queue + order by ev_retry_after + limit 10 + loop + cnt := cnt + 1; + delete from pgq.retry_queue + where ev_owner = rec.ev_owner + and ev_id = rec.ev_id; + end loop; + return cnt; +end; +$$ language plpgsql; -- need admin access + diff --git a/sql/pgq/functions/pgq.maint_rotate_tables.sql b/sql/pgq/functions/pgq.maint_rotate_tables.sql new file mode 100644 index 00000000..0195b605 --- /dev/null +++ b/sql/pgq/functions/pgq.maint_rotate_tables.sql @@ -0,0 +1,98 @@ +create or replace function pgq.maint_rotate_tables_step1(i_queue_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_rotate_tables_step1(1) +-- +-- Rotate tables for one queue. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + badcnt integer; + cf record; + nr integer; + tbl text; +begin + -- check if needed and load record + select * from pgq.queue into cf + where queue_name = i_queue_name + and queue_rotation_period is not null + and queue_switch_step2 is not null + and queue_switch_time + queue_rotation_period < current_timestamp + for update; + if not found then + return 0; + end if; + + -- check if any consumer is on previous table + select coalesce(count(*), 0) into badcnt + from pgq.subscription, pgq.tick + where get_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2 + and sub_queue = cf.queue_id + and tick_queue = cf.queue_id + and tick_id = (select tick_id from pgq.tick + where tick_id < sub_last_tick + and tick_queue = sub_queue + order by tick_queue desc, tick_id desc + limit 1); + if badcnt > 0 then + return 0; + end if; + + -- all is fine, calc next table number + nr := cf.queue_cur_table + 1; + if nr = cf.queue_ntables then + nr := 0; + end if; + tbl := cf.table_name || '_' || nr; + + -- there may be long lock on the table from pg_dump, + -- detect it and skip rotate then + begin + execute 'lock table ' || tbl || ' nowait'; + execute 'truncate ' || tbl; + exception + when lock_not_available then + raise warning 'truncate of % failed, skipping rotate', tbl; + return 0; + end; + + -- remember the moment + update pgq.queue + set queue_cur_table = nr, + queue_switch_time = current_timestamp, + queue_switch_step1 = get_current_txid(), + queue_switch_step2 = NULL + where queue_id = cf.queue_id; + + -- clean ticks - avoid partial batches + delete from pgq.tick + where tick_queue = cf.queue_id + and get_snapshot_xmin(snapshot) < cf.last_switch_step2; + + return 1; +end; +$$ language plpgsql; -- need admin access + +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_rotate_tables_step2(0) +-- +-- It tag rotation as finished where needed. It should be +-- called in separate transaction than pgq.maint_rotate_tables_step1() +-- ---------------------------------------------------------------------- +create or replace function pgq.maint_rotate_tables_step2() +returns integer as $$ +-- visibility tracking. this should run in separate +-- tranaction than step1 +begin + update pgq.queue + set queue_switch_step2 = get_current_txid() + where queue_switch_step2 is null; + return 1; +end; +$$ language plpgsql; -- need admin access + diff --git a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql new file mode 100644 index 00000000..920f68ec --- /dev/null +++ b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql @@ -0,0 +1,33 @@ +create or replace function pgq.maint_tables_to_vacuum() +returns setof text as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.maint_tables_to_vacuum(0) +-- +-- Returns list of tablenames that need frequent vacuuming. +-- +-- The goal is to avoid hardcoding them into maintenance process. +-- +-- Returns: +-- List of table names. +-- ---------------------------------------------------------------------- +begin + return next 'pgq.subscription'; + return next 'pgq.consumer'; + return next 'pgq.queue'; + return next 'pgq.tick'; + return next 'pgq.retry_queue'; + + -- vacuum also txid.epoch, if exists + perform 1 from pg_class t, pg_namespace n + where t.relname = 'epoch' + and n.nspname = 'txid' + and n.oid = t.relnamespace; + if found then + return next 'txid.epoch'; + end if; + + return; +end; +$$ language plpgsql; + + diff --git a/sql/pgq/functions/pgq.next_batch.sql b/sql/pgq/functions/pgq.next_batch.sql new file mode 100644 index 00000000..8d7d8f74 --- /dev/null +++ b/sql/pgq/functions/pgq.next_batch.sql @@ -0,0 +1,66 @@ +create or replace function pgq.next_batch(x_queue_name text, x_consumer_name text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.next_batch(2) +-- +-- Makes next block of events active. +-- +-- If it returns NULL, there is no events available in queue. +-- Consumer should sleep a bith then. +-- +-- Parameters: +-- x_queue_name - Name of the queue +-- x_consumer_name - Name of the consumer +-- +-- Returns: +-- Batch ID or NULL if there are no more events available. +-- ---------------------------------------------------------------------- +declare + next_tick bigint; + next_batch bigint; + errmsg text; + sub record; +begin + select sub_queue, sub_id, sub_last_tick, sub_batch into sub + from pgq.queue q, pgq.consumer c, pgq.subscription s + where q.queue_name = x_queue_name + and c.co_name = x_consumer_name + and s.sub_queue = q.queue_id + and s.sub_consumer = c.co_id; + if not found then + errmsg := 'Not subscriber to queue: ' + || coalesce(x_queue_name, 'NULL') + || '/' + || coalesce(x_consumer_name, 'NULL'); + raise exception '%', errmsg; + end if; + + -- has already active batch + if sub.sub_batch is not null then + return sub.sub_batch; + end if; + + -- find next tick + select tick_id into next_tick + from pgq.tick + where tick_id > sub.sub_last_tick + and tick_queue = sub.sub_queue + order by tick_queue asc, tick_id asc + limit 1; + if not found then + -- nothing to do + return null; + end if; + + -- get next batch + next_batch := nextval('pgq.batch_id_seq'); + update pgq.subscription + set sub_batch = next_batch, + sub_next_tick = next_tick, + sub_active = now() + where sub_id = sub.sub_id; + return next_batch; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.register_consumer.sql b/sql/pgq/functions/pgq.register_consumer.sql new file mode 100644 index 00000000..7d387dab --- /dev/null +++ b/sql/pgq/functions/pgq.register_consumer.sql @@ -0,0 +1,120 @@ +create or replace function pgq.register_consumer( + x_queue_name text, + x_consumer_id text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.register_consumer(2) +-- +-- Subscribe consumer on a queue. +-- +-- From this moment forward, consumer will see all events in the queue. +-- +-- Parameters: +-- x_queue_name - Name of queue +-- x_consumer_name - Name of consumer +-- +-- Returns: +-- 0 - if already registered +-- 1 - if new registration +-- ---------------------------------------------------------------------- +begin + return pgq.register_consumer(x_queue_name, x_consumer_id, NULL); +end; +$$ language plpgsql; -- no perms needed + + +create or replace function pgq.register_consumer( + x_queue_name text, + x_consumer_name text, + x_tick_pos bigint) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.register_consumer(3) +-- +-- Extended registration, allows to specify tick_id. +-- +-- Note: +-- For usage in special situations. +-- +-- Parameters: +-- x_queue_name - Name of a queue +-- x_consumer_name - Name of consumer +-- x_tick_pos - Tick ID +-- +-- Returns: +-- 0/1 whether consumer has already registered. +-- ---------------------------------------------------------------------- +declare + tmp text; + last_tick bigint; + x_queue_id integer; + x_consumer_id integer; + queue integer; + sub record; +begin + select queue_id into x_queue_id from pgq.queue + where queue_name = x_queue_name; + if not found then + raise exception 'Event queue not created yet'; + end if; + + -- get consumer and create if new + select co_id into x_consumer_id from pgq.consumer + where co_name = x_consumer_name; + if not found then + insert into pgq.consumer (co_name) values (x_consumer_name); + x_consumer_id := currval('pgq.consumer_co_id_seq'); + end if; + + -- if particular tick was requested, check if it exists + if x_tick_pos is not null then + perform 1 from pgq.tick + where tick_queue = x_queue_id + and tick_id = x_tick_pos; + if not found then + raise exception 'cannot reposition, tick not found: %', x_tick_pos; + end if; + end if; + + -- check if already registered + select sub_last_tick, sub_batch into sub + from pgq.subscription + where sub_consumer = x_consumer_id + and sub_queue = x_queue_id; + if found then + if sub.sub_batch is not null then + raise exception 'reposition while active not allowed'; + end if; + if x_tick_pos is not null then + -- update tick pos if requested + update pgq.subscription + set sub_last_tick = x_tick_pos + where sub_consumer = x_consumer_id + and sub_queue = x_queue_id; + end if; + -- already registered + return 0; + end if; + + -- new registration + if x_tick_pos is null then + -- start from current tick + select tick_id into last_tick from pgq.tick + where tick_queue = x_queue_id + order by tick_queue desc, tick_id desc + limit 1; + if not found then + raise exception 'No ticks for this queue. Please run ticker on database.'; + end if; + else + last_tick := x_tick_pos; + end if; + + -- register + insert into pgq.subscription (sub_queue, sub_consumer, sub_last_tick) + values (x_queue_id, x_consumer_id, last_tick); + return 1; +end; +$$ language plpgsql security definer; + + diff --git a/sql/pgq/functions/pgq.ticker.sql b/sql/pgq/functions/pgq.ticker.sql new file mode 100644 index 00000000..9489d195 --- /dev/null +++ b/sql/pgq/functions/pgq.ticker.sql @@ -0,0 +1,86 @@ +create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(2) +-- +-- Insert a tick with a particular tick_id. +-- +-- For external tickers. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- i_tick_id - Id of new tick. +-- +-- Returns: +-- Tick id. +-- ---------------------------------------------------------------------- +begin + insert into pgq.tick (tick_queue, tick_id) + select queue_id, i_tick_id + from pgq.queue + where queue_name = i_queue_name + and queue_external_ticker; + if not found then + raise exception 'queue not found'; + end if; + return i_tick_id; +end; +$$ language plpgsql security definer; -- unsure about access + +create or replace function pgq.ticker(i_queue_name text) +returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(1) +-- +-- Insert a tick with a tick_id from sequence. +-- +-- For pgqadm usage. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- +-- Returns: +-- Tick id. +-- ---------------------------------------------------------------------- +declare + res bigint; + ext boolean; + seq text; + q record; +begin + select queue_id, queue_tick_seq, queue_external_ticker into q + from pgq.queue where queue_name = i_queue_name; + if not found then + raise exception 'no such queue'; + end if; + + if q.queue_external_ticker then + raise exception 'This queue has external tick source.'; + end if; + + insert into pgq.tick (tick_queue, tick_id) + values (q.queue_id, nextval(q.queue_tick_seq)); + + res = currval(q.queue_tick_seq); + return res; +end; +$$ language plpgsql security definer; -- unsure about access + +create or replace function pgq.ticker() returns bigint as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.ticker(0) +-- +-- Creates ticks for all queues which dont have external ticker. +-- +-- Returns: +-- Number of queues that were processed. +-- ---------------------------------------------------------------------- +declare + res bigint; +begin + select count(pgq.ticker(queue_name)) into res + from pgq.queue where not queue_external_ticker; + return res; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.unregister_consumer.sql b/sql/pgq/functions/pgq.unregister_consumer.sql new file mode 100644 index 00000000..c97261d6 --- /dev/null +++ b/sql/pgq/functions/pgq.unregister_consumer.sql @@ -0,0 +1,44 @@ + +create or replace function pgq.unregister_consumer( + x_queue_name text, + x_consumer_name text) +returns integer as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.unregister_consumer(2) +-- +-- Unsubscriber consumer from the queue. Also consumer's failed +-- and retry events are deleted. +-- +-- Parameters: +-- x_queue_name - Name of the queue +-- x_consumer_name - Name of the consumer +-- +-- Returns: +-- nothing +-- ---------------------------------------------------------------------- +declare + x_sub_id integer; +begin + select sub_id into x_sub_id + from pgq.subscription, pgq.consumer, pgq.queue + where sub_queue = queue_id + and sub_consumer = co_id + and queue_name = x_queue_name + and co_name = x_consumer_name; + if not found then + raise exception 'consumer not registered on queue'; + end if; + + delete from pgq.retry_queue + where ev_owner = x_sub_id; + + delete from pgq.failed_queue + where ev_owner = x_sub_id; + + delete from pgq.subscription + where sub_id = x_sub_id; + + return 1; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq/functions/pgq.version.sql b/sql/pgq/functions/pgq.version.sql new file mode 100644 index 00000000..d48f9e18 --- /dev/null +++ b/sql/pgq/functions/pgq.version.sql @@ -0,0 +1,12 @@ +-- ---------------------------------------------------------------------- +-- Function: pgq.version(0) +-- +-- Returns verison string for pgq. +-- ---------------------------------------------------------------------- +create or replace function pgq.version() +returns text as $$ +begin + return '2.1'; +end; +$$ language plpgsql; + diff --git a/sql/pgq/sql/logutriga.sql b/sql/pgq/sql/logutriga.sql new file mode 100644 index 00000000..e2a90995 --- /dev/null +++ b/sql/pgq/sql/logutriga.sql @@ -0,0 +1,22 @@ + +create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text) +returns bigint as $$ +begin + raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1; + return 1; +end; +$$ language plpgsql; + +create table udata ( + id serial primary key, + txt text, + bin bytea +); + +create trigger utest AFTER insert or update or delete ON udata +for each row execute procedure pgq.logutriga('udata_que'); + +insert into udata (txt) values ('text1'); +insert into udata (bin) values (E'bi\tn\\000bin'); + + diff --git a/sql/pgq/sql/pgq_init.sql b/sql/pgq/sql/pgq_init.sql new file mode 100644 index 00000000..95f46459 --- /dev/null +++ b/sql/pgq/sql/pgq_init.sql @@ -0,0 +1,66 @@ + +\set ECHO none +\i ../txid/txid.sql +\i structure/install.sql + +\set ECHO all + +select * from pgq.maint_tables_to_vacuum(); +select * from pgq.maint_retry_events(); + +select pgq.create_queue('tmpqueue'); +select pgq.register_consumer('tmpqueue', 'consumer'); +select pgq.unregister_consumer('tmpqueue', 'consumer'); +select pgq.drop_queue('tmpqueue'); + +select pgq.create_queue('myqueue'); +select pgq.register_consumer('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.ticker(); +select pgq.next_batch('myqueue', 'consumer'); +select pgq.next_batch('myqueue', 'consumer'); + +select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); +select queue_name from pgq.get_queue_info() limit 0; +select queue_name, consumer_name from pgq.get_consumer_info() limit 0; + +select pgq.finish_batch(1); +select pgq.finish_batch(1); + +select pgq.ticker(); +select pgq.next_batch('myqueue', 'consumer'); +select * from pgq.batch_event_tables(2); +select * from pgq.get_batch_events(2); +select pgq.finish_batch(2); + +select pgq.insert_event('myqueue', 'r1', 'data'); +select pgq.insert_event('myqueue', 'r2', 'data'); +select pgq.insert_event('myqueue', 'r3', 'data'); +select pgq.current_event_table('myqueue'); +select pgq.ticker(); + +select pgq.next_batch('myqueue', 'consumer'); +select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3); + +select * from pgq.failed_event_list('myqueue', 'consumer'); + +select pgq.event_failed(3, 1, 'failure test'); +select pgq.event_failed(3, 1, 'failure test'); +select pgq.event_retry(3, 2, 0); +select pgq.event_retry(3, 2, 0); +select pgq.finish_batch(3); + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer'); + +select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data + from pgq.failed_event_list('myqueue', 'consumer', 0, 1); + +select * from pgq.failed_event_count('myqueue', 'consumer'); +select * from pgq.failed_event_delete('myqueue', 'consumer', 0); + +select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0, + 'rawtest', 'data', null, null, null, null); + +select pgq.ticker(); diff --git a/sql/pgq/sql/sqltriga.sql b/sql/pgq/sql/sqltriga.sql new file mode 100644 index 00000000..49b86ee7 --- /dev/null +++ b/sql/pgq/sql/sqltriga.sql @@ -0,0 +1,58 @@ + +-- start testing +create table rtest ( + id integer primary key, + dat text +); + +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que'); + +-- simple test +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; + +-- test new fields +alter table rtest add column dat2 text; +insert into rtest values (1, 'value1'); +update rtest set dat = 'value2'; +delete from rtest; + +-- test field ignore +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; + +-- test hashed pkey +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)'); + +insert into rtest values (1, '666', 'newdat'); +update rtest set dat = 5, dat2 = 'newdat2'; +update rtest set dat = 6; +delete from rtest; + + +-- test wrong key +drop trigger rtest_triga on rtest; +create trigger rtest_triga after insert or update or delete on rtest +for each row execute procedure pgq.sqltriga('que3'); + +insert into rtest values (1, 0, 'non-null'); +insert into rtest values (2, 0, NULL); +update rtest set dat2 = 'non-null2' where id=1; +update rtest set dat2 = NULL where id=1; +update rtest set dat2 = 'new-nonnull' where id=2; + +delete from rtest where id=1; +delete from rtest where id=2; + + + diff --git a/sql/pgq/structure/func_internal.sql b/sql/pgq/structure/func_internal.sql new file mode 100644 index 00000000..f84bb195 --- /dev/null +++ b/sql/pgq/structure/func_internal.sql @@ -0,0 +1,23 @@ +-- Section: Internal Functions + +-- Group: Low-level event handling + +\i functions/pgq.batch_event_sql.sql +\i functions/pgq.batch_event_tables.sql +\i functions/pgq.event_retry_raw.sql +\i functions/pgq.insert_event_raw.sql + +-- Group: Ticker + +\i functions/pgq.ticker.sql + +-- Group: Periodic maintenence + +\i functions/pgq.maint_retry_events.sql +\i functions/pgq.maint_rotate_tables.sql +\i functions/pgq.maint_tables_to_vacuum.sql + +-- Group: Random utility functions + +\i functions/pgq.grant_perms.sql + diff --git a/sql/pgq/structure/func_public.sql b/sql/pgq/structure/func_public.sql new file mode 100644 index 00000000..4440a22b --- /dev/null +++ b/sql/pgq/structure/func_public.sql @@ -0,0 +1,36 @@ +-- Section: Public Functions + +-- Group: Queue creation + +\i functions/pgq.create_queue.sql +\i functions/pgq.drop_queue.sql + +-- Group: Event publishing + +\i functions/pgq.insert_event.sql +\i functions/pgq.current_event_table.sql + +-- Group: Subscribing to queue + +\i functions/pgq.register_consumer.sql +\i functions/pgq.unregister_consumer.sql + +-- Group: Batch processing + +\i functions/pgq.next_batch.sql +\i functions/pgq.get_batch_events.sql +\i functions/pgq.event_failed.sql +\i functions/pgq.event_retry.sql +\i functions/pgq.finish_batch.sql + +-- Group: General info functions + +\i functions/pgq.get_queue_info.sql +\i functions/pgq.get_consumer_info.sql +\i functions/pgq.version.sql +\i functions/pgq.get_batch_info.sql + +-- Group: Failed queue browsing + +\i functions/pgq.failed_queue.sql + diff --git a/sql/pgq/structure/install.sql b/sql/pgq/structure/install.sql new file mode 100644 index 00000000..b3c77cb6 --- /dev/null +++ b/sql/pgq/structure/install.sql @@ -0,0 +1,7 @@ + +\i structure/tables.sql +\i structure/types.sql +\i structure/func_internal.sql +\i structure/func_public.sql +\i structure/triggers.sql + diff --git a/sql/pgq/structure/tables.sql b/sql/pgq/structure/tables.sql new file mode 100644 index 00000000..fc56cc81 --- /dev/null +++ b/sql/pgq/structure/tables.sql @@ -0,0 +1,217 @@ +-- ---------------------------------------------------------------------- +-- Section: Internal Tables +-- +-- Map to Slony-I: +-- sl_node - pgq.consumer +-- sl_set - pgq.queue +-- sl_subscriber + sl_confirm - pgq.subscription +-- sl_event - pgq.tick +-- sl_setsync - pgq_ext.completed_* +-- sl_log_* - slony1 has per-cluster data tables, +-- here we do redirection in pgq.queue +-- to have per-queue data tables. +-- ---------------------------------------------------------------------- + +set client_min_messages = 'warning'; + +-- drop schema if exists pgq cascade; +create schema pgq; +grant usage on schema pgq to public; + +-- ---------------------------------------------------------------------- +-- Table: pgq.consumer +-- +-- Name to id lookup for consumers +-- +-- Columns: +-- co_id - consumer's id for internal usage +-- co_name - consumer's id for external usage +-- ---------------------------------------------------------------------- +create table pgq.consumer ( + co_id serial, + co_name text not null default 'fooz', + + constraint consumer_pkey primary key (co_id), + constraint consumer_name_uq UNIQUE (co_name) +); + + +-- ---------------------------------------------------------------------- +-- Table: pgq.queue +-- +-- Information about available queues +-- +-- Columns: +-- queue_id - queue id for internal usage +-- queue_name - queue name visible outside +-- queue_data - parent table for actual data tables +-- queue_switch_step1 - tx when rotation happened +-- queue_switch_step2 - tx after rotation was committed +-- queue_switch_time - time when switch happened +-- queue_ticker_max_count - batch should not contain more events +-- queue_ticker_max_lag - events should not age more +-- queue_ticker_idle_period - how often to tick when no events happen +-- ---------------------------------------------------------------------- +create table pgq.queue ( + queue_id serial, + queue_name text not null, + + queue_ntables integer not null default 3, + queue_cur_table integer not null default 0, + queue_rotation_period interval not null default '2 hours', + queue_switch_step1 bigint not null default get_current_txid(), + queue_switch_step2 bigint default get_current_txid(), + queue_switch_time timestamptz not null default now(), + + queue_external_ticker boolean not null default false, + queue_ticker_max_count integer not null default 500, + queue_ticker_max_lag interval not null default '3 seconds', + queue_ticker_idle_period interval not null default '1 minute', + + queue_data_pfx text not null, + queue_event_seq text not null, + queue_tick_seq text not null, + + constraint queue_pkey primary key (queue_id), + constraint queue_name_uq unique (queue_name) +); + +-- ---------------------------------------------------------------------- +-- Table: pgq.tick +-- +-- Snapshots for event batching +-- +-- Columns: +-- tick_queue - queue id whose tick it is +-- tick_id - ticks id (per-queue) +-- tick_time - time when tick happened +-- tick_snapshot +-- ---------------------------------------------------------------------- +create table pgq.tick ( + tick_queue int4 not null, + tick_id bigint not null, + tick_time timestamptz not null default now(), + tick_snapshot txid_snapshot not null default get_current_snapshot(), + + constraint tick_pkey primary key (tick_queue, tick_id), + constraint tick_queue_fkey foreign key (tick_queue) + references pgq.queue (queue_id) +); + +-- ---------------------------------------------------------------------- +-- Sequence: pgq.batch_id_seq +-- +-- Sequence for batch id's. +-- ---------------------------------------------------------------------- + +create sequence pgq.batch_id_seq; +-- ---------------------------------------------------------------------- +-- Table: pgq.subscription +-- +-- Consumer registration on a queue +-- +-- Columns: +-- +-- sub_id - subscription id for internal usage +-- sub_queue - queue id +-- sub_consumer - consumer's id +-- sub_tick - last tick the consumer processed +-- sub_batch - shortcut for queue_id/consumer_id/tick_id +-- sub_next_tick - +-- ---------------------------------------------------------------------- +create table pgq.subscription ( + sub_id serial not null, + sub_queue int4 not null, + sub_consumer int4 not null, + sub_last_tick bigint not null, + sub_active timestamptz not null default now(), + sub_batch bigint, + sub_next_tick bigint, + + constraint subscription_pkey primary key (sub_id), + constraint sub_queue_fkey foreign key (sub_queue) + references pgq.queue (queue_id), + constraint sub_consumer_fkey foreign key (sub_consumer) + references pgq.consumer (co_id) +); + + +-- ---------------------------------------------------------------------- +-- Table: pgq.event_template +-- +-- Parent table for all event tables +-- +-- Columns: +-- ev_id - event's id, supposed to be unique per queue +-- ev_time - when the event was inserted +-- ev_txid - transaction id which inserted the event +-- ev_owner - subscription id that wanted to retry this +-- ev_retry - how many times the event has been retried, NULL for new events +-- ev_type - consumer/producer can specify what the data fields contain +-- ev_data - data field +-- ev_extra1 - extra data field +-- ev_extra2 - extra data field +-- ev_extra3 - extra data field +-- ev_extra4 - extra data field +-- ---------------------------------------------------------------------- +create table pgq.event_template ( + ev_id bigint not null, + ev_time timestamptz not null, + + ev_txid bigint not null default get_current_txid(), + ev_owner int4, + ev_retry int4, + + ev_type text, + ev_data text, + ev_extra1 text, + ev_extra2 text, + ev_extra3 text, + ev_extra4 text +); + +-- ---------------------------------------------------------------------- +-- Table: pgq.retry_queue +-- +-- Events to be retried +-- +-- Columns: +-- ev_retry_after - time when it should be re-inserted to main queue +-- ---------------------------------------------------------------------- +create table pgq.retry_queue ( + ev_retry_after timestamptz not null, + + like pgq.event_template, + + constraint rq_pkey primary key (ev_owner, ev_id), + constraint rq_owner_fkey foreign key (ev_owner) + references pgq.subscription (sub_id) +); +alter table pgq.retry_queue alter column ev_owner set not null; +alter table pgq.retry_queue alter column ev_txid drop not null; +create index rq_retry_idx on pgq.retry_queue (ev_retry_after); + +-- ---------------------------------------------------------------------- +-- Table: pgq.failed_queue +-- +-- Events whose processing failed +-- +-- Columns: +-- ev_failed_reason - consumer's excuse for not processing +-- ev_failed_time - when it was tagged failed +-- ---------------------------------------------------------------------- +create table pgq.failed_queue ( + ev_failed_reason text, + ev_failed_time timestamptz not null, + + -- all event fields + like pgq.event_template, + + constraint fq_pkey primary key (ev_owner, ev_id), + constraint fq_owner_fkey foreign key (ev_owner) + references pgq.subscription (sub_id) +); +alter table pgq.failed_queue alter column ev_owner set not null; +alter table pgq.failed_queue alter column ev_txid drop not null; + + diff --git a/sql/pgq/structure/triggers.sql b/sql/pgq/structure/triggers.sql new file mode 100644 index 00000000..e732347f --- /dev/null +++ b/sql/pgq/structure/triggers.sql @@ -0,0 +1,8 @@ + +-- Section: Public Triggers + +-- Group: Trigger Functions + +\i triggers/pgq.logutriga.sql +\i triggers/pgq.sqltriga.sql + diff --git a/sql/pgq/structure/types.sql b/sql/pgq/structure/types.sql new file mode 100644 index 00000000..c89ce500 --- /dev/null +++ b/sql/pgq/structure/types.sql @@ -0,0 +1,47 @@ + +create type pgq.ret_queue_info as ( + queue_name text, + queue_ntables integer, + queue_cur_table integer, + queue_rotation_period interval, + queue_switch_time timestamptz, + queue_external_ticker boolean, + queue_ticker_max_count integer, + queue_ticker_max_lag interval, + queue_ticker_idle_period interval, + ticker_lag interval +); + +create type pgq.ret_consumer_info as ( + queue_name text, + consumer_name text, + lag interval, + last_seen interval +); + +create type pgq.ret_batch_info as ( + queue_name text, + consumer_name text, + batch_start timestamptz, + batch_end timestamptz, + prev_tick_id bigint, + tick_id bigint, + lag interval +); + + +create type pgq.ret_batch_event as ( + ev_id bigint, + ev_time timestamptz, + + ev_txid bigint, + ev_retry int4, + + ev_type text, + ev_data text, + ev_extra1 text, + ev_extra2 text, + ev_extra3 text, + ev_extra4 text +); + diff --git a/sql/pgq/triggers/pgq.logutriga.sql b/sql/pgq/triggers/pgq.logutriga.sql new file mode 100644 index 00000000..d4cb0145 --- /dev/null +++ b/sql/pgq/triggers/pgq.logutriga.sql @@ -0,0 +1,103 @@ + +create or replace function pgq.logutriga() +returns trigger as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.logutriga() +# -- +# -- Trigger function that puts row data urlencoded into queue. +# -- +# -- Trigger parameters: +# -- arg1 - queue name +# -- arg2 - optionally 'SKIP' +# -- +# -- Queue event fields: +# -- ev_type - I/U/D +# -- ev_data - column values urlencoded +# -- ev_extra1 - table name +# -- ev_extra2 - primary key columns +# -- +# -- Regular listen trigger example: +# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer +# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname'); +# -- +# -- Redirect trigger example: +# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer +# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP'); +# -- ---------------------------------------------------------------------- + +# this triger takes 1 or 2 args: +# queue_name - destination queue +# option return code (OK, SKIP) SKIP means op won't happen +# copy-paste of db_urlencode from skytools.quoting +from urllib import quote_plus +def db_urlencode(dict): + elem_list = [] + for k, v in dict.items(): + if v is None: + elem = quote_plus(str(k)) + else: + elem = quote_plus(str(k)) + '=' + quote_plus(str(v)) + elem_list.append(elem) + return '&'.join(elem_list) + +# load args +queue_name = TD['args'][0] +if len(TD['args']) > 1: + ret_code = TD['args'][1] +else: + ret_code = 'OK' +table_oid = TD['relid'] + +# on first call init plans +if not 'init_done' in SD: + # find table name + q = "SELECT n.nspname || '.' || c.relname AS table_name"\ + " FROM pg_namespace n, pg_class c"\ + " WHERE n.oid = c.relnamespace AND c.oid = $1" + SD['name_plan'] = plpy.prepare(q, ['oid']) + + # find key columns + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + SD['key_plan'] = plpy.prepare(q, ['oid']) + + # insert data + q = "SELECT pgq.insert_event($1, $2, $3, $4, $5, null, null)" + SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text', 'text']) + + # shorter tags + SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'} + + # remember init + SD['init_done'] = 1 + +# load & cache table data +if table_oid in SD: + tbl_name, tbl_keys = SD[table_oid] +else: + res = plpy.execute(SD['name_plan'], [table_oid]) + tbl_name = res[0]['table_name'] + res = plpy.execute(SD['key_plan'], [table_oid]) + tbl_keys = ",".join(map(lambda x: x['attname'], res)) + + SD[table_oid] = (tbl_name, tbl_keys) + +# prepare args +if TD['event'] == 'DELETE': + data = db_urlencode(TD['old']) +else: + data = db_urlencode(TD['new']) + +# insert event +plpy.execute(SD['ins_plan'], [ + queue_name, + SD['op_map'][TD['event']], + data, tbl_name, tbl_keys]) + +# done +return ret_code + +$$ language plpythonu; + diff --git a/sql/pgq/triggers/pgq.sqltriga.sql b/sql/pgq/triggers/pgq.sqltriga.sql new file mode 100644 index 00000000..c978e19d --- /dev/null +++ b/sql/pgq/triggers/pgq.sqltriga.sql @@ -0,0 +1,195 @@ + +-- listen trigger: +-- create trigger triga_nimi after insert or update on customer +-- for each row execute procedure pgq.sqltriga('qname'); + +-- redirect trigger: +-- create trigger triga_nimi after insert or update on customer +-- for each row execute procedure pgq.sqltriga('qname', 'ret=SKIP'); + +create or replace function pgq.sqltriga() +returns trigger as $$ +# -- ---------------------------------------------------------------------- +# -- Function: pgq.sqltriga() +# -- +# -- Trigger function that puts row data in partial SQL form into queue. +# -- +# -- Parameters: +# -- arg1 - queue name +# -- arg2 - optional urlencoded options +# -- +# -- Extra options: +# -- +# -- ret - return value for function OK/SKIP +# -- pkey - override pkey fields, can be functions +# -- ignore - comma separated field names to ignore +# -- +# -- Queue event fields: +# -- ev_type - I/U/D +# -- ev_data - partial SQL statement +# -- ev_extra1 - table name +# -- +# -- ---------------------------------------------------------------------- +# this triger takes 1 or 2 args: +# queue_name - destination queue +# args - urlencoded dict of options: +# ret - return value: OK/SKIP +# pkey - comma-separated col names or funcs on cols +# simple: pkey=user,orderno +# hashed: pkey=user,hashtext(user) +# ignore - comma-separated col names to ignore + +# on first call init stuff +if not 'init_done' in SD: + # find table name plan + q = "SELECT n.nspname || '.' || c.relname AS table_name"\ + " FROM pg_namespace n, pg_class c"\ + " WHERE n.oid = c.relnamespace AND c.oid = $1" + SD['name_plan'] = plpy.prepare(q, ['oid']) + + # find key columns plan + q = "SELECT k.attname FROM pg_index i, pg_attribute k"\ + " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\ + " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\ + " ORDER BY k.attnum" + SD['key_plan'] = plpy.prepare(q, ['oid']) + + # data insertion + q = "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)" + SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text']) + + # shorter tags + SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'} + + # quoting + from psycopg import QuotedString + def quote(s): + if s is None: + return "null" + s = str(s) + return str(QuotedString(s)) + s = s.replace('\\', '\\\\') + s = s.replace("'", "''") + return "'%s'" % s + + # TableInfo class + import re, urllib + class TableInfo: + func_rc = re.compile("([^(]+) [(] ([^)]+) [)]", re.I | re.X) + def __init__(self, table_oid, options_txt): + res = plpy.execute(SD['name_plan'], [table_oid]) + self.name = res[0]['table_name'] + + self.parse_options(options_txt) + self.load_pkey() + + def recheck(self, options_txt): + if self.options_txt == options_txt: + return + self.parse_options(options_txt) + self.load_pkey() + + def parse_options(self, options_txt): + self.options = {'ret': 'OK'} + if options_txt: + for s in options_txt.split('&'): + k, v = s.split('=', 1) + self.options[k] = urllib.unquote_plus(v) + self.options_txt = options_txt + + def load_pkey(self): + self.pkey_list = [] + if not 'pkey' in self.options: + res = plpy.execute(SD['key_plan'], [table_oid]) + for krow in res: + col = krow['attname'] + expr = col + "=%s" + self.pkey_list.append( (col, expr) ) + else: + for a_pk in self.options['pkey'].split(','): + m = self.func_rc.match(a_pk) + if m: + col = m.group(2) + fn = m.group(1) + expr = "%s(%s) = %s(%%s)" % (fn, col, fn) + else: + # normal case + col = a_pk + expr = col + "=%s" + self.pkey_list.append( (col, expr) ) + if len(self.pkey_list) == 0: + plpy.error('sqltriga needs primary key on table') + + def get_insert_stmt(self, new): + col_list = [] + val_list = [] + for k, v in new.items(): + col_list.append(k) + val_list.append(quote(v)) + return "(%s) values (%s)" % (",".join(col_list), ",".join(val_list)) + + def get_update_stmt(self, old, new): + chg_list = [] + for k, v in new.items(): + ov = old[k] + if v == ov: + continue + chg_list.append("%s=%s" % (k, quote(v))) + if len(chg_list) == 0: + pk = self.pkey_list[0][0] + chg_list.append("%s=%s" % (pk, quote(new[pk]))) + return "%s where %s" % (",".join(chg_list), self.get_pkey_expr(new)) + + def get_pkey_expr(self, data): + exp_list = [] + for col, exp in self.pkey_list: + exp_list.append(exp % quote(data[col])) + return " and ".join(exp_list) + + SD['TableInfo'] = TableInfo + + # cache some functions + def proc_insert(tbl): + return tbl.get_insert_stmt(TD['new']) + def proc_update(tbl): + return tbl.get_update_stmt(TD['old'], TD['new']) + def proc_delete(tbl): + return tbl.get_pkey_expr(TD['old']) + SD['event_func'] = { + 'I': proc_insert, + 'U': proc_update, + 'D': proc_delete, + } + + # remember init + SD['init_done'] = 1 + + +# load args +table_oid = TD['relid'] +queue_name = TD['args'][0] +if len(TD['args']) > 1: + options_str = TD['args'][1] +else: + options_str = '' + +# load & cache table data +if table_oid in SD: + tbl = SD[table_oid] + tbl.recheck(options_str) +else: + tbl = SD['TableInfo'](table_oid, options_str) + SD[table_oid] = tbl + +# generate payload +op = SD['op_map'][TD['event']] +data = SD['event_func'][op](tbl) + +# insert event +plpy.execute(SD['ins_plan'], [queue_name, op, data, tbl.name]) + +# done +return tbl.options['ret'] + +$$ language plpythonu; + diff --git a/sql/pgq_ext/Makefile b/sql/pgq_ext/Makefile new file mode 100644 index 00000000..dc824924 --- /dev/null +++ b/sql/pgq_ext/Makefile @@ -0,0 +1,16 @@ + +DOCS = README.pgq_ext +DATA_built = pgq_ext.sql + +SRCS = structure/tables.sql functions/track_batch.sql functions/track_event.sql + +REGRESS = test_pgq_ext +REGRESS_OPTS = --load-language=plpgsql + +include ../../config.mak + +include $(PGXS) + +pgq_ext.sql: $(SRCS) + cat $(SRCS) > $@ + diff --git a/sql/pgq_ext/README.pgq_ext b/sql/pgq_ext/README.pgq_ext new file mode 100644 index 00000000..b2116b87 --- /dev/null +++ b/sql/pgq_ext/README.pgq_ext @@ -0,0 +1,52 @@ + +Track processed batches and events in target DB +================================================ + +Batch tracking is OK. + +Event tracking is OK if consumer does not use retry queue. + +Batch tracking +-------------- + +is_batch_done(consumer, batch) + +returns: + + true - batch is done already + false - batch is not done yet + +set_batch_done(consumer, batch) + +returns: + + true - tagging successful, batch was not done yet + false - batch was done already + +Event tracking +-------------- + +is_batch_done(consumer, batch, event) + +returns: + + true - event is done + false - event is not done yet + + +set_batch_done(consumer, batch, event) + +returns: + + true - tagging was successful, event was not done + false - event is done already + + +Fastvacuum +---------- + +pgq.ext.completed_batch +pgq.ext.completed_event +pgq.ext.completed_tick +pgq.ext.partial_batch + diff --git a/sql/pgq_ext/expected/test_pgq_ext.out b/sql/pgq_ext/expected/test_pgq_ext.out new file mode 100644 index 00000000..ccced856 --- /dev/null +++ b/sql/pgq_ext/expected/test_pgq_ext.out @@ -0,0 +1,85 @@ +\set ECHO off +-- +-- test batch tracking +-- +select pgq_ext.is_batch_done('c', 1); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 1); + set_batch_done +---------------- + t +(1 row) + +select pgq_ext.is_batch_done('c', 1); + is_batch_done +--------------- + t +(1 row) + +select pgq_ext.set_batch_done('c', 1); + set_batch_done +---------------- + f +(1 row) + +select pgq_ext.is_batch_done('c', 2); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 2); + set_batch_done +---------------- + t +(1 row) + +-- +-- test event tracking +-- +select pgq_ext.is_batch_done('c', 3); + is_batch_done +--------------- + f +(1 row) + +select pgq_ext.is_event_done('c', 3, 101); + is_event_done +--------------- + f +(1 row) + +select pgq_ext.set_event_done('c', 3, 101); + set_event_done +---------------- + t +(1 row) + +select pgq_ext.is_event_done('c', 3, 101); + is_event_done +--------------- + t +(1 row) + +select pgq_ext.set_event_done('c', 3, 101); + set_event_done +---------------- + f +(1 row) + +select pgq_ext.set_batch_done('c', 3); + set_batch_done +---------------- + t +(1 row) + +select * from pgq_ext.completed_event order by 1,2; + consumer_id | batch_id | event_id +-------------+----------+---------- + c | 3 | 101 +(1 row) + diff --git a/sql/pgq_ext/functions/track_batch.sql b/sql/pgq_ext/functions/track_batch.sql new file mode 100644 index 00000000..e77f590d --- /dev/null +++ b/sql/pgq_ext/functions/track_batch.sql @@ -0,0 +1,39 @@ + +create or replace function pgq_ext.is_batch_done( + a_consumer text, a_batch_id bigint) +returns boolean as $$ +declare + res boolean; +begin + select last_batch_id = a_batch_id + into res from pgq_ext.completed_batch + where consumer_id = a_consumer; + if not found then + return false; + end if; + return res; +end; +$$ language plpgsql security definer; + +create or replace function pgq_ext.set_batch_done( + a_consumer text, a_batch_id bigint) +returns boolean as $$ +begin + if pgq_ext.is_batch_done(a_consumer, a_batch_id) then + return false; + end if; + + if a_batch_id > 0 then + update pgq_ext.completed_batch + set last_batch_id = a_batch_id + where consumer_id = a_consumer; + if not found then + insert into pgq_ext.completed_batch (consumer_id, last_batch_id) + values (a_consumer, a_batch_id); + end if; + end if; + + return true; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq_ext/functions/track_event.sql b/sql/pgq_ext/functions/track_event.sql new file mode 100644 index 00000000..8e89f41e --- /dev/null +++ b/sql/pgq_ext/functions/track_event.sql @@ -0,0 +1,60 @@ + +create or replace function pgq_ext.is_event_done( + a_consumer text, + a_batch_id bigint, a_event_id bigint) +returns boolean as $$ +declare + res bigint; +begin + perform 1 from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = a_batch_id + and event_id = a_event_id; + return found; +end; +$$ language plpgsql security definer; + +create or replace function pgq_ext.set_event_done( + a_consumer text, a_batch_id bigint, a_event_id bigint) +returns boolean as $$ +declare + old_batch bigint; +begin + -- check if done + perform 1 from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = a_batch_id + and event_id = a_event_id; + if found then + return false; + end if; + + -- if batch changed, do cleanup + select cur_batch_id into old_batch + from pgq_ext.partial_batch + where consumer_id = a_consumer; + if not found then + -- first time here + insert into pgq_ext.partial_batch + (consumer_id, cur_batch_id) + values (a_consumer, a_batch_id); + elsif old_batch <> a_batch_id then + -- batch changed, that means old is finished on queue db + -- thus the tagged events are not needed anymore + delete from pgq_ext.completed_event + where consumer_id = a_consumer + and batch_id = old_batch; + -- remember current one + update pgq_ext.partial_batch + set cur_batch_id = a_batch_id + where consumer_id = a_consumer; + end if; + + -- tag as done + insert into pgq_ext.completed_event (consumer_id, batch_id, event_id) + values (a_consumer, a_batch_id, a_event_id); + + return true; +end; +$$ language plpgsql security definer; + diff --git a/sql/pgq_ext/sql/test_pgq_ext.sql b/sql/pgq_ext/sql/test_pgq_ext.sql new file mode 100644 index 00000000..d6f4eeaa --- /dev/null +++ b/sql/pgq_ext/sql/test_pgq_ext.sql @@ -0,0 +1,26 @@ +\set ECHO off +\i pgq_ext.sql +\set ECHO all + +-- +-- test batch tracking +-- +select pgq_ext.is_batch_done('c', 1); +select pgq_ext.set_batch_done('c', 1); +select pgq_ext.is_batch_done('c', 1); +select pgq_ext.set_batch_done('c', 1); +select pgq_ext.is_batch_done('c', 2); +select pgq_ext.set_batch_done('c', 2); + +-- +-- test event tracking +-- +select pgq_ext.is_batch_done('c', 3); +select pgq_ext.is_event_done('c', 3, 101); +select pgq_ext.set_event_done('c', 3, 101); +select pgq_ext.is_event_done('c', 3, 101); +select pgq_ext.set_event_done('c', 3, 101); +select pgq_ext.set_batch_done('c', 3); +select * from pgq_ext.completed_event order by 1,2; + + diff --git a/sql/pgq_ext/structure/tables.sql b/sql/pgq_ext/structure/tables.sql new file mode 100644 index 00000000..377353ba --- /dev/null +++ b/sql/pgq_ext/structure/tables.sql @@ -0,0 +1,48 @@ + +set client_min_messages = 'warning'; +set default_with_oids = 'off'; + +create schema pgq_ext; +grant usage on schema pgq_ext to public; + + +-- +-- batch tracking +-- +create table pgq_ext.completed_batch ( + consumer_id text not null, + last_batch_id bigint not null, + + primary key (consumer_id) +); + + +-- +-- event tracking +-- +create table pgq_ext.completed_event ( + consumer_id text not null, + batch_id bigint not null, + event_id bigint not null, + + primary key (consumer_id, batch_id, event_id) +); + +create table pgq_ext.partial_batch ( + consumer_id text not null, + cur_batch_id bigint not null, + + primary key (consumer_id) +); + +-- +-- tick tracking for SerialConsumer() +-- no access functions provided here +-- +create table pgq_ext.completed_tick ( + consumer_id text not null, + last_tick_id bigint not null, + + primary key (consumer_id) +); + diff --git a/sql/txid/Makefile b/sql/txid/Makefile new file mode 100644 index 00000000..b0c79925 --- /dev/null +++ b/sql/txid/Makefile @@ -0,0 +1,30 @@ + +MODULE_big = txid +SRCS = txid.c epoch.c +OBJS = $(SRCS:.c=.o) + +DATA_built = txid.sql +DATA = uninstall_txid.sql +DOCS = README.txid +EXTRA_CLEAN = txid.sql.in + +REGRESS = txid +REGRESS_OPTS = --load-language=plpgsql + +include ../../config.mak +include $(PGXS) + +# additional deps +txid.o: txid.h +epoch.o: txid.h + +# postgres >= manages epoch itself, so skip epoch tables +pgnew = $(shell test $(VERSION) "<" "8.2" && echo "false" || echo "true") +ifeq ($(pgnew),true) +TXID_SQL = txid.std.sql +else +TXID_SQL = txid.std.sql txid.schema.sql +endif +txid.sql.in: $(TXID_SQL) + cat $(TXID_SQL) > $@ + diff --git a/sql/txid/README.txid b/sql/txid/README.txid new file mode 100644 index 00000000..6cdac28a --- /dev/null +++ b/sql/txid/README.txid @@ -0,0 +1,65 @@ + +txid - 8 byte transaction ID's +============================== + +Based on xxid module from Slony-I. + +The goal is to make PostgreSQL internal transaction ID and snapshot +data usable externally. They cannot be used directly as the +internal 4-byte value wraps around and thus breaks indexing. + +This module extends the internal value with wraparound cound (epoch). +It uses relaxed method for wraparound check. There is a table +txid.epoch (epoch, last_value) which is used to check if the xid +is in current, next or previous epoch. It requires only occasional +read-write access - ca. after 100k - 500k transactions. + +Also it contains type 'txid_snapshot' and following functions: + + +get_current_txid() returns int8 + + Current transaction ID + +get_current_snapshot() returns txid_snapshot + + Current snapshot + +get_snapshot_xmin( snap ) returns int8 -- + + Smallest TXID in snapshot. TXID's smaller than this + are all visible in snapshot. + +get_snapshot_xmax( snap ) returns int8 + + Largest TXID in snapshot. TXID's starting from this one are + all invisible in snapshot. + +get_snapshot_values( snap ) setof int8 + + List of uncommitted TXID's in snapshot, that are invisible + in snapshot. Values are between xmin and xmax. + +txid_in_snapshot(id, snap) returns bool + + Is TXID visible in snapshot? + +txid_not_in_snapshot(id, snap) returns bool + + Is TXID invisible in snapshot? + + +Problems +-------- + +- it breaks when there are more than 2G tx'es between calls. + Fixed in 8.2 + +- functions that create new txid's should be 'security definers' + thus better protecting txid_epoch table. + +- After loading database from backup you should do: + + UPDATE txid.epoch SET epoch = epoch + 1, + last_value = (get_current_txid() & 4294967295); + diff --git a/sql/txid/epoch.c b/sql/txid/epoch.c new file mode 100644 index 00000000..a2cc28cd --- /dev/null +++ b/sql/txid/epoch.c @@ -0,0 +1,240 @@ +/*------------------------------------------------------------------------- + * epoch.c + * + * Detect current epoch. + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include + +#include "access/transam.h" +#include "executor/spi.h" +#include "miscadmin.h" +#include "catalog/pg_control.h" +#include "access/xlog.h" + +#include "txid.h" + +/* + * do a TransactionId -> txid conversion + */ +txid txid_convert_xid(TransactionId xid, TxidEpoch *state) +{ + uint64 epoch; + + /* avoid issues with the the special meaning of 0 */ + if (xid == InvalidTransactionId) + return MAX_INT64; + + /* return special xid's as-is */ + if (xid < FirstNormalTransactionId) + return xid; + + /* xid can on both sides on wrap-around */ + epoch = state->epoch; + if (TransactionIdPrecedes(xid, state->last_value)) { + if (xid > state->last_value) + epoch--; + } else if (TransactionIdFollows(xid, state->last_value)) { + if (xid < state->last_value) + epoch++; + } + return (epoch << 32) | xid; +} + +#if PG_CONTROL_VERSION >= 820 + +/* + * PostgreSQl 8.2 keeps track of epoch internally. + */ + +void txid_load_epoch(TxidEpoch *state, int try_write) +{ + TransactionId xid; + uint32 epoch; + + GetNextXidAndEpoch(&xid, &epoch); + + state->epoch = epoch; + state->last_value = xid; +} + +#else + +/* + * For older PostgreSQL keep epoch in table. + */ + +/* + * this caches the txid_epoch table. + * The struct should be updated only together with the table. + */ +static TxidEpoch epoch_state = { 0, 0 }; + +/* + * load values from txid_epoch table. + */ +static int load_epoch(void) +{ + HeapTuple row; + TupleDesc rdesc; + bool isnull = false; + Datum tmp; + int res; + uint64 db_epoch, db_value; + + res = SPI_connect(); + if (res < 0) + elog(ERROR, "cannot connect to SPI"); + + res = SPI_execute("select epoch, last_value from txid.epoch", true, 0); + if (res != SPI_OK_SELECT) + elog(ERROR, "load_epoch: select failed?"); + if (SPI_processed != 1) + elog(ERROR, "load_epoch: there must be exactly 1 row"); + + row = SPI_tuptable->vals[0]; + rdesc = SPI_tuptable->tupdesc; + + tmp = SPI_getbinval(row, rdesc, 1, &isnull); + if (isnull) + elog(ERROR, "load_epoch: epoch is NULL"); + db_epoch = DatumGetInt64(tmp); + + tmp = SPI_getbinval(row, rdesc, 2, &isnull); + if (isnull) + elog(ERROR, "load_epoch: last_value is NULL"); + db_value = DatumGetInt64(tmp); + + SPI_finish(); + + /* + * If the db has lesser values, then some updates were lost. + * + * Should that be special-cased? ATM just use db values. + * Thus immidiate update. + */ + epoch_state.epoch = db_epoch; + epoch_state.last_value = db_value; + return 1; +} + +/* + * updates last_value and epoch, if needed + */ +static void save_epoch(void) +{ + int res; + char qbuf[200]; + uint64 new_epoch, new_value; + TransactionId xid = GetTopTransactionId(); + TransactionId old_value; + + /* store old state */ + MemoryContext oldcontext = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; + + /* + * avoid changing internal values. + */ + new_value = xid; + new_epoch = epoch_state.epoch; + old_value = (TransactionId)epoch_state.last_value; + if (xid < old_value) { + if (TransactionIdFollows(xid, old_value)) + new_epoch++; + else + return; + } + sprintf(qbuf, "update txid.epoch set epoch = %llu, last_value = %llu", + (unsigned long long)new_epoch, + (unsigned long long)new_value); + + /* + * The update may fail in case of SERIALIZABLE transaction. + * Try to catch the error and hide it. + */ + BeginInternalSubTransaction(NULL); + PG_TRY(); + { + /* do the update */ + res = SPI_connect(); + if (res < 0) + elog(ERROR, "cannot connect to SPI"); + res = SPI_execute(qbuf, false, 0); + SPI_finish(); + + ReleaseCurrentSubTransaction(); + } + PG_CATCH(); + { + /* we expect rollback to clean up inner SPI call */ + RollbackAndReleaseCurrentSubTransaction(); + FlushErrorState(); + res = -1; /* remember failure */ + } + PG_END_TRY(); + + /* restore old state */ + MemoryContextSwitchTo(oldcontext); + CurrentResourceOwner = oldowner; + + if (res < 0) + return; + + /* + * Seems the update was successful, update internal state too. + * + * There is a chance that the TX will be rollbacked, but then + * another backend will do the update, or this one at next + * checkpoint. + */ + epoch_state.epoch = new_epoch; + epoch_state.last_value = new_value; +} + +static void check_epoch(int update_prio) +{ + TransactionId xid = GetTopTransactionId(); + TransactionId recheck, tx_next; + int ok = 1; + + /* should not happen, but just in case */ + if (xid == InvalidTransactionId) + return; + + /* new backend */ + if (epoch_state.last_value == 0) + load_epoch(); + + /* try to avoid concurrent access */ + if (update_prio) + recheck = 50000 + 100 * (MyProcPid & 0x1FF); + else + recheck = 300000 + 1000 * (MyProcPid & 0x1FF); + + /* read table */ + tx_next = (TransactionId)epoch_state.last_value + recheck; + if (TransactionIdFollows(xid, tx_next)) + ok = load_epoch(); + + /* + * check if save is needed. last_value may be updated above. + */ + tx_next = (TransactionId)epoch_state.last_value + recheck; + if (!ok || TransactionIdFollows(xid, tx_next)) + save_epoch(); +} + +void txid_load_epoch(TxidEpoch *state, int try_write) +{ + check_epoch(try_write); + + state->epoch = epoch_state.epoch; + state->last_value = epoch_state.last_value; +} + + +#endif diff --git a/sql/txid/expected/txid.out b/sql/txid/expected/txid.out new file mode 100644 index 00000000..400f88c2 --- /dev/null +++ b/sql/txid/expected/txid.out @@ -0,0 +1,88 @@ +-- init +\set ECHO none +-- i/o +select '12:13:'::txid_snapshot; + txid_snapshot +--------------- + 12:13: +(1 row) + +select '12:13:1,2'::txid_snapshot; +ERROR: illegal txid_snapshot input format +-- errors +select '31:12:'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '0:1:'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '12:13:0'::txid_snapshot; +ERROR: illegal txid_snapshot input format +select '12:13:2,1'::txid_snapshot; +ERROR: illegal txid_snapshot input format +create table snapshot_test ( + nr integer, + snap txid_snapshot +); +insert into snapshot_test values (1, '12:13:'); +insert into snapshot_test values (2, '12:20:13,15,18'); +insert into snapshot_test values (3, '100001:100009:100005,100007,100008'); +select snap from snapshot_test order by nr; + snap +------------------------------------ + 12:13: + 12:20:13,15,18 + 100001:100009:100005,100007,100008 +(3 rows) + +select get_snapshot_xmin(snap), + get_snapshot_xmax(snap), + get_snapshot_active(snap) +from snapshot_test order by nr; + get_snapshot_xmin | get_snapshot_xmax | get_snapshot_active +-------------------+-------------------+--------------------- + 12 | 20 | 13 + 12 | 20 | 15 + 12 | 20 | 18 + 100001 | 100009 | 100005 + 100001 | 100009 | 100007 + 100001 | 100009 | 100008 +(6 rows) + +select id, txid_in_snapshot(id, snap), + txid_not_in_snapshot(id, snap) +from snapshot_test, generate_series(11, 21) id +where nr = 2; + id | txid_in_snapshot | txid_not_in_snapshot +----+------------------+---------------------- + 11 | t | f + 12 | t | f + 13 | f | t + 14 | t | f + 15 | f | t + 16 | t | f + 17 | t | f + 18 | f | t + 19 | t | f + 20 | f | t + 21 | f | t +(11 rows) + +-- test current values also +select get_current_txid() >= get_snapshot_xmin(get_current_snapshot()); + ?column? +---------- + t +(1 row) + +select get_current_txid() < get_snapshot_xmax(get_current_snapshot()); + ?column? +---------- + t +(1 row) + +select txid_in_snapshot(get_current_txid(), get_current_snapshot()), + txid_not_in_snapshot(get_current_txid(), get_current_snapshot()); + txid_in_snapshot | txid_not_in_snapshot +------------------+---------------------- + t | f +(1 row) + diff --git a/sql/txid/sql/txid.sql b/sql/txid/sql/txid.sql new file mode 100644 index 00000000..6009944b --- /dev/null +++ b/sql/txid/sql/txid.sql @@ -0,0 +1,43 @@ +-- init +\set ECHO none +\i txid.sql +\set ECHO all + +-- i/o +select '12:13:'::txid_snapshot; +select '12:13:1,2'::txid_snapshot; + +-- errors +select '31:12:'::txid_snapshot; +select '0:1:'::txid_snapshot; +select '12:13:0'::txid_snapshot; +select '12:13:2,1'::txid_snapshot; + +create table snapshot_test ( + nr integer, + snap txid_snapshot +); + +insert into snapshot_test values (1, '12:13:'); +insert into snapshot_test values (2, '12:20:13,15,18'); +insert into snapshot_test values (3, '100001:100009:100005,100007,100008'); + +select snap from snapshot_test order by nr; + +select get_snapshot_xmin(snap), + get_snapshot_xmax(snap), + get_snapshot_active(snap) +from snapshot_test order by nr; + +select id, txid_in_snapshot(id, snap), + txid_not_in_snapshot(id, snap) +from snapshot_test, generate_series(11, 21) id +where nr = 2; + +-- test current values also +select get_current_txid() >= get_snapshot_xmin(get_current_snapshot()); +select get_current_txid() < get_snapshot_xmax(get_current_snapshot()); + +select txid_in_snapshot(get_current_txid(), get_current_snapshot()), + txid_not_in_snapshot(get_current_txid(), get_current_snapshot()); + diff --git a/sql/txid/txid.c b/sql/txid/txid.c new file mode 100644 index 00000000..256d3608 --- /dev/null +++ b/sql/txid/txid.c @@ -0,0 +1,364 @@ +/*------------------------------------------------------------------------- + * txid.c + * + * Safe handling of transaction ID's. + * + * Copyright (c) 2003-2004, PostgreSQL Global Development Group + * Author: Jan Wieck, Afilias USA INC. + * + * 64-bit output: Marko Kreen, Skype Technologies + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include + +#include "access/xact.h" +#include "funcapi.h" + +#include "txid.h" + +#ifdef INT64_IS_BUSTED +#error txid needs working int64 +#endif + +#ifdef PG_MODULE_MAGIC +PG_MODULE_MAGIC; +#endif + +/* + * public functions + */ + +PG_FUNCTION_INFO_V1(txid_current); +PG_FUNCTION_INFO_V1(txid_snapshot_in); +PG_FUNCTION_INFO_V1(txid_snapshot_out); +PG_FUNCTION_INFO_V1(txid_in_snapshot); +PG_FUNCTION_INFO_V1(txid_not_in_snapshot); +PG_FUNCTION_INFO_V1(txid_current_snapshot); +PG_FUNCTION_INFO_V1(txid_snapshot_xmin); +PG_FUNCTION_INFO_V1(txid_snapshot_xmax); +PG_FUNCTION_INFO_V1(txid_snapshot_active); + +/* + * utility functions + */ + +static int _cmp_txid(const void *aa, const void *bb) +{ + const uint64 *a = aa; + const uint64 *b = bb; + if (*a < *b) + return -1; + if (*a > *b) + return 1; + return 0; +} + +static void sort_snapshot(TxidSnapshot *snap) +{ + qsort(snap->xip, snap->nxip, sizeof(txid), _cmp_txid); +} + +static TxidSnapshot * +parse_snapshot(const char *str) +{ + int a_size; + txid *xip; + + int a_used = 0; + txid xmin; + txid xmax; + txid last_val = 0, val; + TxidSnapshot *snap; + int size; + + char *endp; + + a_size = 1024; + xip = (txid *) palloc(sizeof(txid) * a_size); + + xmin = (txid) strtoull(str, &endp, 0); + if (*endp != ':') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + + xmax = (txid) strtoull(str, &endp, 0); + if (*endp != ':') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + + /* it should look sane */ + if (xmin >= xmax || xmin > MAX_INT64 || xmax > MAX_INT64 + || xmin == 0 || xmax == 0) + elog(ERROR, "illegal txid_snapshot input format"); + + while (*str != '\0') + { + if (a_used >= a_size) + { + a_size *= 2; + xip = (txid *) repalloc(xip, sizeof(txid) * a_size); + } + + /* read next value */ + if (*str == '\'') + { + str++; + val = (txid) strtoull(str, &endp, 0); + if (*endp != '\'') + elog(ERROR, "illegal txid_snapshot input format"); + str = endp + 1; + } + else + { + val = (txid) strtoull(str, &endp, 0); + str = endp; + } + + /* require the input to be in order */ + if (val < xmin || val <= last_val || val >= xmax) + elog(ERROR, "illegal txid_snapshot input format"); + + xip[a_used++] = val; + last_val = val; + + if (*str == ',') + str++; + else + { + if (*str != '\0') + elog(ERROR, "illegal txid_snapshot input format"); + } + } + + size = offsetof(TxidSnapshot, xip) + sizeof(txid) * a_used; + snap = (TxidSnapshot *) palloc(size); + snap->varsz = size; + snap->xmin = xmin; + snap->xmax = xmax; + snap->nxip = a_used; + if (a_used > 0) + memcpy(&(snap->xip[0]), xip, sizeof(txid) * a_used); + pfree(xip); + + return snap; +} + +/* + * Public functions + */ + +/* + * txid_current - Return the current transaction ID as txid + */ +Datum +txid_current(PG_FUNCTION_ARGS) +{ + txid val; + TxidEpoch state; + + txid_load_epoch(&state, 0); + + val = txid_convert_xid(GetTopTransactionId(), &state); + + PG_RETURN_INT64(val); +} + +/* + * txid_current_snapshot - return current snapshot + */ +Datum +txid_current_snapshot(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap; + unsigned num, i, size; + TxidEpoch state; + + if (SerializableSnapshot == NULL) + elog(ERROR, "get_current_snapshot: SerializableSnapshot == NULL"); + + txid_load_epoch(&state, 1); + + num = SerializableSnapshot->xcnt; + size = offsetof(TxidSnapshot, xip) + sizeof(txid) * num; + snap = palloc(size); + snap->varsz = size; + snap->xmin = txid_convert_xid(SerializableSnapshot->xmin, &state); + snap->xmax = txid_convert_xid(SerializableSnapshot->xmax, &state); + snap->nxip = num; + for (i = 0; i < num; i++) + snap->xip[i] = txid_convert_xid(SerializableSnapshot->xip[i], &state); + + /* we want then guaranteed ascending order */ + sort_snapshot(snap); + + PG_RETURN_POINTER(snap); +} + +/* + * txid_snapshot_in - input function for type txid_snapshot + */ +Datum +txid_snapshot_in(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap; + char *str = PG_GETARG_CSTRING(0); + + snap = parse_snapshot(str); + PG_RETURN_POINTER(snap); +} + +/* + * txid_snapshot_out - output function for type txid_snapshot + */ +Datum +txid_snapshot_out(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + + char *str = palloc(60 + snap->nxip * 30); + char *cp = str; + int i; + + snprintf(str, 60, "%llu:%llu:", + (unsigned long long)snap->xmin, + (unsigned long long)snap->xmax); + cp = str + strlen(str); + + for (i = 0; i < snap->nxip; i++) + { + snprintf(cp, 30, "%llu%s", + (unsigned long long)snap->xip[i], + (i < snap->nxip - 1) ? "," : ""); + cp += strlen(cp); + } + + PG_RETURN_CSTRING(str); +} + + +/* + * txid_in_snapshot - is txid visible in snapshot ? + */ +Datum +txid_in_snapshot(PG_FUNCTION_ARGS) +{ + txid value = PG_GETARG_INT64(0); + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1); + int i; + int res = true; + + if (value < snap->xmin) + res = true; + else if (value >= snap->xmax) + res = false; + else + { + for (i = 0; i < snap->nxip; i++) + if (value == snap->xip[i]) + { + res = false; + break; + } + } + PG_FREE_IF_COPY(snap, 1); + PG_RETURN_BOOL(res); +} + + +/* + * txid_not_in_snapshot - is txid invisible in snapshot ? + */ +Datum +txid_not_in_snapshot(PG_FUNCTION_ARGS) +{ + txid value = PG_GETARG_INT64(0); + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1); + int i; + int res = false; + + if (value < snap->xmin) + res = false; + else if (value >= snap->xmax) + res = true; + else + { + for (i = 0; i < snap->nxip; i++) + if (value == snap->xip[i]) + { + res = true; + break; + } + } + PG_FREE_IF_COPY(snap, 1); + PG_RETURN_BOOL(res); +} + +/* + * txid_snapshot_xmin - return snapshot's xmin + */ +Datum +txid_snapshot_xmin(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + txid res = snap->xmin; + PG_FREE_IF_COPY(snap, 0); + PG_RETURN_INT64(res); +} + +/* + * txid_snapshot_xmin - return snapshot's xmax + */ +Datum +txid_snapshot_xmax(PG_FUNCTION_ARGS) +{ + TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + txid res = snap->xmax; + PG_FREE_IF_COPY(snap, 0); + PG_RETURN_INT64(res); +} + +/* remember state between function calls */ +struct snap_state { + int pos; + TxidSnapshot *snap; +}; + +/* + * txid_snapshot_active - returns uncommitted TXID's in snapshot. + */ +Datum +txid_snapshot_active(PG_FUNCTION_ARGS) +{ + FuncCallContext *fctx; + struct snap_state *state; + + if (SRF_IS_FIRSTCALL()) { + TxidSnapshot *snap; + int statelen; + + snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0); + + fctx = SRF_FIRSTCALL_INIT(); + statelen = sizeof(*state) + snap->varsz; + state = MemoryContextAlloc(fctx->multi_call_memory_ctx, statelen); + state->pos = 0; + state->snap = (TxidSnapshot *)((char *)state + sizeof(*state)); + memcpy(state->snap, snap, snap->varsz); + fctx->user_fctx = state; + + PG_FREE_IF_COPY(snap, 0); + } + fctx = SRF_PERCALL_SETUP(); + state = fctx->user_fctx; + if (state->pos < state->snap->nxip) { + Datum res = Int64GetDatum(state->snap->xip[state->pos]); + state->pos++; + SRF_RETURN_NEXT(fctx, res); + } else { + SRF_RETURN_DONE(fctx); + } +} + diff --git a/sql/txid/txid.h b/sql/txid/txid.h new file mode 100644 index 00000000..8c648754 --- /dev/null +++ b/sql/txid/txid.h @@ -0,0 +1,43 @@ +#ifndef _TXID_H_ +#define _TXID_H_ + +#define MAX_INT64 0x7FFFFFFFFFFFFFFFLL + +/* Use unsigned variant internally */ +typedef uint64 txid; + +typedef struct +{ + int32 varsz; + uint32 nxip; + txid xmin; + txid xmax; + txid xip[1]; +} TxidSnapshot; + + +typedef struct { + uint64 last_value; + uint64 epoch; +} TxidEpoch; + +/* internal functions */ +void txid_load_epoch(TxidEpoch *state, int try_write); +txid txid_convert_xid(TransactionId xid, TxidEpoch *state); + +/* public functions */ +Datum txid_current(PG_FUNCTION_ARGS); +Datum txid_current_snapshot(PG_FUNCTION_ARGS); + +Datum txid_snapshot_in(PG_FUNCTION_ARGS); +Datum txid_snapshot_out(PG_FUNCTION_ARGS); + +Datum txid_in_snapshot(PG_FUNCTION_ARGS); +Datum txid_not_in_snapshot(PG_FUNCTION_ARGS); +Datum txid_snapshot_xmin(PG_FUNCTION_ARGS); +Datum txid_snapshot_xmax(PG_FUNCTION_ARGS); +Datum txid_snapshot_active(PG_FUNCTION_ARGS); + + +#endif /* _TXID_H_ */ + diff --git a/sql/txid/txid.schema.sql b/sql/txid/txid.schema.sql new file mode 100644 index 00000000..b0a5b5a1 --- /dev/null +++ b/sql/txid/txid.schema.sql @@ -0,0 +1,50 @@ +-- ---------- +-- txid.sql +-- +-- SQL script for loading the transaction ID compatible datatype +-- +-- Copyright (c) 2003-2004, PostgreSQL Global Development Group +-- Author: Jan Wieck, Afilias USA INC. +-- +-- ---------- + +-- +-- now the epoch storage +-- + +CREATE SCHEMA txid; + +-- remember txid settings +-- use bigint so we can do arithmetic with it +create table txid.epoch ( + epoch bigint, + last_value bigint +); + +-- make sure there exist exactly one row +insert into txid.epoch values (0, 1); + + +-- then protect it +create function txid.epoch_guard() +returns trigger as $$ +begin + if TG_OP = 'UPDATE' then + -- epoch: allow only small increase + if NEW.epoch > OLD.epoch and NEW.epoch < (OLD.epoch + 3) then + return NEW; + end if; + -- last_value: allow only increase + if NEW.epoch = OLD.epoch and NEW.last_value > OLD.last_value then + return NEW; + end if; + end if; + raise exception 'bad operation on txid.epoch'; +end; +$$ language plpgsql; + +-- the trigger +create trigger epoch_guard_trigger +before insert or update or delete on txid.epoch +for each row execute procedure txid.epoch_guard(); + diff --git a/sql/txid/txid.std.sql b/sql/txid/txid.std.sql new file mode 100644 index 00000000..8ba34cbc --- /dev/null +++ b/sql/txid/txid.std.sql @@ -0,0 +1,77 @@ +-- ---------- +-- txid.sql +-- +-- SQL script for loading the transaction ID compatible datatype +-- +-- Copyright (c) 2003-2004, PostgreSQL Global Development Group +-- Author: Jan Wieck, Afilias USA INC. +-- +-- ---------- + +set client_min_messages = 'warning'; + +CREATE DOMAIN txid AS bigint CHECK (value > 0); + +-- +-- A special transaction snapshot data type for faster visibility checks +-- +CREATE OR REPLACE FUNCTION txid_snapshot_in(cstring) + RETURNS txid_snapshot + AS 'MODULE_PATHNAME' LANGUAGE C + IMMUTABLE STRICT; +CREATE OR REPLACE FUNCTION txid_snapshot_out(txid_snapshot) + RETURNS cstring + AS 'MODULE_PATHNAME' LANGUAGE C + IMMUTABLE STRICT; + +-- +-- The data type itself +-- +CREATE TYPE txid_snapshot ( + INPUT = txid_snapshot_in, + OUTPUT = txid_snapshot_out, + INTERNALLENGTH = variable, + STORAGE = extended, + ALIGNMENT = double +); + +CREATE OR REPLACE FUNCTION get_current_txid() + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_current' LANGUAGE C + SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION get_current_snapshot() + RETURNS txid_snapshot + AS 'MODULE_PATHNAME', 'txid_current_snapshot' LANGUAGE C + SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION get_snapshot_xmin(txid_snapshot) + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_xmin' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION get_snapshot_xmax(txid_snapshot) + RETURNS bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_xmax' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION get_snapshot_active(txid_snapshot) + RETURNS setof bigint + AS 'MODULE_PATHNAME', 'txid_snapshot_active' LANGUAGE C + IMMUTABLE STRICT; + + +-- +-- Special comparision functions used by the remote worker +-- for sync chunk selection +-- +CREATE OR REPLACE FUNCTION txid_in_snapshot(bigint, txid_snapshot) + RETURNS boolean + AS 'MODULE_PATHNAME', 'txid_in_snapshot' LANGUAGE C + IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION txid_not_in_snapshot(bigint, txid_snapshot) + RETURNS boolean + AS 'MODULE_PATHNAME', 'txid_not_in_snapshot' LANGUAGE C + IMMUTABLE STRICT; + diff --git a/sql/txid/uninstall_txid.sql b/sql/txid/uninstall_txid.sql new file mode 100644 index 00000000..17a88045 --- /dev/null +++ b/sql/txid/uninstall_txid.sql @@ -0,0 +1,10 @@ + +DROP DOMAIN txid; +DROP TYPE txid_snapshot cascade; +DROP SCHEMA txid CASCADE; +DROP FUNCTION get_current_txid(); +DROP FUNCTION get_snapshot_xmin(); +DROP FUNCTION get_snapshot_xmax(); +DROP FUNCTION get_snapshot_active(); + + diff --git a/tests/env.sh b/tests/env.sh new file mode 100644 index 00000000..05f61e8d --- /dev/null +++ b/tests/env.sh @@ -0,0 +1,6 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + + diff --git a/tests/londiste/conf/fread.ini b/tests/londiste/conf/fread.ini new file mode 100644 index 00000000..f3595d7f --- /dev/null +++ b/tests/londiste/conf/fread.ini @@ -0,0 +1,17 @@ + +[londiste] +job_name = fread_test + +method = file_read + +provider_db = dbname=provider +subscriber_db = dbname=file_subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_test_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +file_src = ./file_logs + diff --git a/tests/londiste/conf/fwrite.ini b/tests/londiste/conf/fwrite.ini new file mode 100644 index 00000000..ecba2379 --- /dev/null +++ b/tests/londiste/conf/fwrite.ini @@ -0,0 +1,17 @@ + +[londiste] +job_name = fwrite_test + +method = file_write + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +file_dst = ./file_logs + diff --git a/tests/londiste/conf/linkticker.ini b/tests/londiste/conf/linkticker.ini new file mode 100644 index 00000000..a228bb83 --- /dev/null +++ b/tests/londiste/conf/linkticker.ini @@ -0,0 +1,17 @@ +[pgqadm] + +job_name = link_ticker + +db = dbname=subscriber + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/londiste/conf/replic.ini b/tests/londiste/conf/replic.ini new file mode 100644 index 00000000..e34e4adf --- /dev/null +++ b/tests/londiste/conf/replic.ini @@ -0,0 +1,15 @@ + +[londiste] +job_name = replic + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +loop_delay = 0.3 + diff --git a/tests/londiste/conf/tester.ini b/tests/londiste/conf/tester.ini new file mode 100644 index 00000000..8293ee1b --- /dev/null +++ b/tests/londiste/conf/tester.ini @@ -0,0 +1,16 @@ + +[londiste] +job_name = replic_test + +provider_db = dbname=provider +subscriber_db = dbname=subscriber + +# it will be used as sql ident so no dots/spaces +pgq_queue_name = londiste_test_replic + +pidfile = sys/pid.%(job_name)s +logfile = sys/log.%(job_name)s + +# should be in right sequence +table_list = data1, data2 + diff --git a/tests/londiste/conf/ticker.ini b/tests/londiste/conf/ticker.ini new file mode 100644 index 00000000..5a35c85f --- /dev/null +++ b/tests/londiste/conf/ticker.ini @@ -0,0 +1,17 @@ +[pgqadm] + +job_name = ticker + +db = dbname=provider + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/londiste/data.sql b/tests/londiste/data.sql new file mode 100644 index 00000000..9bf6e819 --- /dev/null +++ b/tests/londiste/data.sql @@ -0,0 +1,24 @@ + +set client_min_messages = 'warning'; + +create table data1 ( + id serial primary key, + data text +); + +create unique index idx_data1_uq on data1 (data); + +create index idx_data1_rand on data1 (id, data); + +create table data2 ( + id serial primary key, + data text, + -- old_id integer references data1, + constraint uq_data2 unique (data) +); + +create index idx_data2_rand on data2 (id, data); + + +create sequence test_seq; +select setval('test_seq', 50); diff --git a/tests/londiste/env.sh b/tests/londiste/env.sh new file mode 100644 index 00000000..45c82d89 --- /dev/null +++ b/tests/londiste/env.sh @@ -0,0 +1,7 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + +#. /opt/apps/pgsql-dev/env + diff --git a/tests/londiste/gendb.sh b/tests/londiste/gendb.sh new file mode 100755 index 00000000..6effbe6d --- /dev/null +++ b/tests/londiste/gendb.sh @@ -0,0 +1,47 @@ +#! /bin/sh + +. ../env.sh + +contrib=/usr/share/postgresql/8.1/contrib +contrib=/opt/apps/pgsql-dev/share/contrib +contrib=/opt/pgsql/share/contrib + +db=provider + + +mkdir -p file_logs sys +./stop.sh +sleep 1 + +rm -rf file_logs sys +mkdir -p file_logs sys + +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +pgqadm.py conf/ticker.ini install +psql -q $db -f data.sql + +db=subscriber +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +pgqadm.py conf/linkticker.ini install +psql -q $db -f data.sql + +db=file_subscriber +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +createlang plpgsql $db +createlang plpythonu $db +psql -q $db -f data.sql + +echo "done, testing" + +#pgqmgr.py -d conf/ticker.ini ticker +#./run-tests.sh + diff --git a/tests/londiste/run-tests.sh b/tests/londiste/run-tests.sh new file mode 100755 index 00000000..2e6fa8a8 --- /dev/null +++ b/tests/londiste/run-tests.sh @@ -0,0 +1,58 @@ +#! /bin/sh + +. ./env.sh + +script=londiste.py + +set -e + +$script conf/replic.ini provider install + +psql -c "update pgq.queue set queue_ticker_idle_period = '3', queue_ticker_max_lag = '2'" provider + +pgqadm.py -d conf/ticker.ini ticker + +$script conf/replic.ini subscriber install + +$script conf/replic.ini subscriber register +$script conf/replic.ini subscriber unregister + +$script -v -d conf/replic.ini replay +$script -v -d conf/fwrite.ini replay + +sleep 2 + +$script conf/replic.ini provider add data1 +$script conf/replic.ini subscriber add data1 + +sleep 2 + +$script conf/replic.ini provider add data2 +$script conf/replic.ini subscriber add data2 + +sleep 2 + +$script conf/replic.ini provider tables +$script conf/replic.ini provider remove data2 + +sleep 2 + +$script conf/replic.ini provider add data2 + +$script conf/replic.ini provider add-seq data1_id_seq +$script conf/replic.ini provider add-seq test_seq +$script conf/replic.ini subscriber add-seq data1_id_seq +$script conf/replic.ini subscriber add-seq test_seq + +sleep 2 + +$script conf/replic.ini subscriber tables +$script conf/replic.ini subscriber missing + +$script conf/replic.ini subscriber remove data2 +sleep 2 +$script conf/replic.ini subscriber add data2 +sleep 2 + +./testing.py conf/tester.ini + diff --git a/tests/londiste/stop.sh b/tests/londiste/stop.sh new file mode 100755 index 00000000..b6b951e4 --- /dev/null +++ b/tests/londiste/stop.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +. ../env.sh +./testing.py -s conf/tester.ini +londiste.py -s conf/fwrite.ini +londiste.py -s conf/replic.ini + +sleep 1 + +pgqadm.py -s conf/ticker.ini +pgqadm.py -s conf/linkticker.ini + diff --git a/tests/londiste/testing.py b/tests/londiste/testing.py new file mode 100755 index 00000000..6f62ae1b --- /dev/null +++ b/tests/londiste/testing.py @@ -0,0 +1,80 @@ +#! /usr/bin/env python + +"""Londiste tester. +""" + +import sys, os, skytools + + +class Tester(skytools.DBScript): + test_pos = 0 + nr = 1 + def __init__(self, args): + skytools.DBScript.__init__(self, 'londiste', args) + self.log.info('start testing') + + def reload(self): + skytools.DBScript.reload(self) + self.loop_delay = 0.1 + + def work(self): + + src_db = self.get_database('provider_db') + dst_db = self.get_database('subscriber_db') + src_curs = src_db.cursor() + dst_curs = dst_db.cursor() + src_curs.execute("insert into data1 (data) values ('foo%d')" % self.nr) + src_curs.execute("insert into data2 (data) values ('foo%d')" % self.nr) + src_db.commit() + self.nr += 1 + + if self.bad_state(dst_db, dst_curs): + return + + if self.test_pos == 0: + self.resync_table(dst_db, dst_curs) + self.test_pos += 1 + elif self.test_pos == 1: + self.run_compare() + self.test_pos += 1 + + def bad_state(self, db, curs): + q = "select * from londiste.subscriber_table" + curs.execute(q) + db.commit() + ok = 0 + bad = 0 + cnt = 0 + for row in curs.dictfetchall(): + cnt += 1 + if row['merge_state'] == 'ok': + ok += 1 + else: + bad += 1 + + if cnt < 2: + return 1 + if bad > 0: + return 1 + + if ok > 0: + return 0 + + return 1 + + def resync_table(self, db, curs): + self.log.info('trying to remove table') + curs.execute("update londiste.subscriber_table"\ + " set merge_state = null" + " where table_name='public.data1'") + db.commit() + + def run_compare(self): + args = ["londiste.py", "conf/replic.ini", "compare"] + err = os.spawnvp(os.P_WAIT, "londiste.py", args) + self.log.info("Compare result=%d" % err) + +if __name__ == '__main__': + script = Tester(sys.argv[1:]) + script.start() + diff --git a/tests/scripts/conf/cube.ini b/tests/scripts/conf/cube.ini new file mode 100644 index 00000000..9b31b41c --- /dev/null +++ b/tests/scripts/conf/cube.ini @@ -0,0 +1,18 @@ +[cube_dispatcher] +job_name = cube_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptdst + +pgq_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + +# how many rows are kept: keep_latest, keep_all +mode = keep_latest + +part_template = + create table _DEST_TABLE (like _PARENT); + alter table only _DEST_TABLE add primary key (_PKEY); + diff --git a/tests/scripts/conf/mover.ini b/tests/scripts/conf/mover.ini new file mode 100644 index 00000000..5ad7cd7f --- /dev/null +++ b/tests/scripts/conf/mover.ini @@ -0,0 +1,13 @@ +[queue_mover] +job_name = queue_mover_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptsrc + +pgq_queue_name = data.src + +dst_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + diff --git a/tests/scripts/conf/table.ini b/tests/scripts/conf/table.ini new file mode 100644 index 00000000..c7165ea3 --- /dev/null +++ b/tests/scripts/conf/table.ini @@ -0,0 +1,29 @@ +[table_dispatcher] +job_name = table_test + +src_db = dbname=scriptsrc +dst_db = dbname=scriptdst + +pgq_queue_name = data.middle + +logfile = sys/%(job_name)s.log +pidfile = sys/%(job_name)s.pid + +# where to put data. when partitioning, will be used as base name +dest_table = data2 + +# date field with will be used for partitioning +# special value: _EVTIME - event creation time +part_column = start_date + +#fields = * +#fields = id_pstn_cdr, foo, bar +#fields = id_pstn_cdr:my_id, foo, bar:baz + + +# template used for creating partition tables +# _DEST_TABLE +part_template = + create table _DEST_TABLE () inherits (data2); + + diff --git a/tests/scripts/conf/ticker.ini b/tests/scripts/conf/ticker.ini new file mode 100644 index 00000000..8e2ee9b2 --- /dev/null +++ b/tests/scripts/conf/ticker.ini @@ -0,0 +1,26 @@ +[pgqadm] + +job_name = ticker + +db = dbname=scriptsrc + +# how often to run maintenance [minutes] +maint_delay_min = 1 + +# how often to check fot activity [secs] +loop_delay = 0.1 + +# if there's no events, how often to tick (to show liveness) [secs] +idle_interval = 1 + +# if there's this much events available, tick +max_events = 10 + +# if there is not many events, then don't let them stay more than [secs] +max_lag = 0.5 + +logfile = sys/log.%(job_name)s +pidfile = sys/pid.%(job_name)s + +use_skylog = 0 + diff --git a/tests/scripts/data.sql b/tests/scripts/data.sql new file mode 100644 index 00000000..84572549 --- /dev/null +++ b/tests/scripts/data.sql @@ -0,0 +1,22 @@ + +set client_min_messages = 'warning'; + +create table data1 ( + id serial primary key, + data text +); + +create unique index idx_data1_uq on data1 (data); + +create index idx_data1_rand on data1 (id, data); + +create table data2 ( + id serial primary key, + data text, + -- old_id integer references data1, + constraint uq_data2 unique (data) +); + +create index idx_data2_rand on data2 (id, data); + + diff --git a/tests/scripts/env.sh b/tests/scripts/env.sh new file mode 100644 index 00000000..1ab42fa8 --- /dev/null +++ b/tests/scripts/env.sh @@ -0,0 +1,7 @@ + +PYTHONPATH=../../python:$PYTHONPATH +PATH=../../python:../../scripts:$PATH +export PYTHONPATH PATH + +#. /opt/pgsql/env + diff --git a/tests/scripts/gendb.sh b/tests/scripts/gendb.sh new file mode 100755 index 00000000..8f329980 --- /dev/null +++ b/tests/scripts/gendb.sh @@ -0,0 +1,45 @@ +#! /bin/sh + +. ./env.sh + +contrib=/usr/share/postgresql/8.1/contrib +contrib=/opt/pgsql/share/contrib + +mkdir -p sys +./stop.sh +sleep 1 + +rm -f sys/* + + +db=scriptsrc +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db + +pgqadm.py conf/ticker.ini install + +#createlang plpgsql $db +#createlang plpythonu $db +#psql -q $db -f $contrib/txid.sql +#psql -q $db -f $contrib/pgq.sql +psql -q $db -f $contrib/pgq_ext.sql +psql -q $db -f $contrib/logutriga.sql +psql -q $db -f data.sql +psql -q $db -f install.sql + +db=scriptdst +echo "creating database: $db" +dropdb $db +sleep 1 +createdb $db +createlang plpgsql $db +psql -q $db -f data.sql +psql -q $db -f $contrib/pgq_ext.sql + +echo "done, testing" + +#pgqmgr.py -d conf/ticker.ini ticker +#./run-tests.sh + diff --git a/tests/scripts/install.sql b/tests/scripts/install.sql new file mode 100644 index 00000000..392980bf --- /dev/null +++ b/tests/scripts/install.sql @@ -0,0 +1,7 @@ + +select pgq.create_queue('data.src'); +select pgq.create_queue('data.middle'); + +create trigger test_logger after insert or update or delete +on data1 for each row execute procedure pgq.logutriga('data.src'); + diff --git a/tests/scripts/run-tests.sh b/tests/scripts/run-tests.sh new file mode 100755 index 00000000..b9223e18 --- /dev/null +++ b/tests/scripts/run-tests.sh @@ -0,0 +1,15 @@ +#! /bin/sh + +. ./env.sh + +pgqadm.py -d conf/ticker.ini ticker +queue_mover.py -d conf/mover.ini +cube_dispatcher.py -d conf/cube.ini +table_dispatcher.py -d conf/table.ini + +sleep 1 +psql scriptsrc < ' +#log_statement = 'none' # none, mod, ddl, all +#log_hostname = off + + +#--------------------------------------------------------------------------- +# RUNTIME STATISTICS +#--------------------------------------------------------------------------- + +# - Statistics Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + +# - Query/Index Statistics Collector - + +#stats_start_collector = on +#stats_command_string = off +#stats_block_level = off +#stats_row_level = off +#stats_reset_on_server_start = off + + +#--------------------------------------------------------------------------- +# AUTOVACUUM PARAMETERS +#--------------------------------------------------------------------------- + +#autovacuum = off # enable autovacuum subprocess? +#autovacuum_naptime = 60 # time between autovacuum runs, in secs +#autovacuum_vacuum_threshold = 1000 # min # of tuple updates before + # vacuum +#autovacuum_analyze_threshold = 500 # min # of tuple updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.4 # fraction of rel size before + # vacuum +#autovacuum_analyze_scale_factor = 0.2 # fraction of rel size before + # analyze +#autovacuum_vacuum_cost_delay = -1 # default vacuum cost delay for + # autovac, -1 means use + # vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovac, -1 means use + # vacuum_cost_limit + + +#--------------------------------------------------------------------------- +# CLIENT CONNECTION DEFAULTS +#--------------------------------------------------------------------------- + +# - Statement Behavior - + +#search_path = '$user,public' # schema names +#default_tablespace = '' # a tablespace name, '' uses + # the default +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#statement_timeout = 0 # 0 is disabled, in milliseconds + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#timezone = unknown # actually, defaults to TZ + # environment setting +#australian_timezones = off +#extra_float_digits = 0 # min -15, max 2 +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb -- they might be changed +lc_messages = 'C' # locale for system error message + # strings +lc_monetary = 'C' # locale for monetary formatting +lc_numeric = 'C' # locale for number formatting +lc_time = 'C' # locale for time formatting + +# - Other Defaults - + +#explain_pretty_print = on +#dynamic_library_path = '$libdir' + + +#--------------------------------------------------------------------------- +# LOCK MANAGEMENT +#--------------------------------------------------------------------------- + +#deadlock_timeout = 1000 # in milliseconds +#max_locks_per_transaction = 64 # min 10 +# note: each lock table slot uses ~220 bytes of shared memory, and there are +# max_locks_per_transaction * (max_connections + max_prepared_transactions) +# lock table slots. + + +#--------------------------------------------------------------------------- +# VERSION/PLATFORM COMPATIBILITY +#--------------------------------------------------------------------------- + +# - Previous Postgres Versions - + +#add_missing_from = off +#backslash_quote = safe_encoding # on, off, or safe_encoding +#default_with_oids = off +#escape_string_warning = off +#regex_flavor = advanced # advanced, extended, or basic +#sql_inheritance = on + +# - Other Platforms & Clients - + +#transform_null_equals = off + + +#--------------------------------------------------------------------------- +# CUSTOMIZED OPTIONS +#--------------------------------------------------------------------------- + +#custom_variable_classes = '' # list of custom variable class names diff --git a/tests/walmgr/run-test.sh b/tests/walmgr/run-test.sh new file mode 100755 index 00000000..555ef282 --- /dev/null +++ b/tests/walmgr/run-test.sh @@ -0,0 +1,85 @@ +#! /bin/sh + +set -e + +tmp=/tmp/waltest +src=$PWD +walmgr=$src/../../python/walmgr.py + +test -f $tmp/data.master/postmaster.pid \ +&& kill `head -1 $tmp/data.master/postmaster.pid` || true +test -f $tmp/data.slave/postmaster.pid \ +&& kill `head -1 $tmp/data.slave/postmaster.pid` || true + +rm -rf $tmp +mkdir -p $tmp +cd $tmp + +LANG=C +PATH=/usr/lib/postgresql/8.1/bin:$PATH +export PATH LANG + +mkdir log slave + +# +# Prepare configs +# + +### wal.master.ini ### +cat > wal.master.ini < wal.slave.ini < rc.slave < log/initdb.log 2>&1 +cp $src/conf.master/* data.master/ +pg_ctl -D data.master -l log/pg.master.log start +sleep 4 + +echo '####' $walmgr $tmp/wal.master.ini setup +$walmgr wal.master.ini setup +echo '####' $walmgr $tmp/wal.master.ini backup +$walmgr wal.master.ini backup + +echo '####' $walmgr $tmp/wal.slave.ini restore +$walmgr $tmp/wal.slave.ini restore +sleep 10 +echo '####' $walmgr $tmp/wal.slave.ini boot +$walmgr $tmp/wal.slave.ini boot + -- 2.39.5