--- /dev/null
+
+Marko Kreen <marko.kreen@skype.net> - main coder
+Martin Pihlak <martin.pihlak@skype.net> - walmgr
+
--- /dev/null
+SkyTools - tool collection for PostgreSQL
+
+Copyright (c) 2007 Marko Kreen, Skype Technologies OÜ
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
--- /dev/null
+
+-include config.mak
+
+PYTHON ?= python
+
+pyver = $(shell $(PYTHON) -V 2>&1 | sed 's/^[^ ]* \([0-9]*\.[0-9]*\).*/\1/')
+
+SUBDIRS = sql
+
+all: python-all modules-all
+
+modules-all: config.mak
+ make -C sql all
+
+python-all: config.mak
+ $(PYTHON) setup.py build
+
+clean:
+ make -C sql clean
+ make -C doc clean
+ $(PYTHON) setup.py clean
+ rm -rf build
+ find python -name '*.py[oc]' -print | xargs rm -f
+
+install: python-install modules-install
+
+installcheck:
+ make -C sql installcheck
+
+modules-install: config.mak
+ make -C sql install DESTDIR=$(DESTDIR)
+ test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR)
+
+python-install: config.mak
+ $(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR)/
+ test \! -d compat || make -C compat $@ DESTDIR=$(DESTDIR)
+
+distclean: clean
+ for dir in $(SUBDIRS); do make -C $$dir $@ || exit 1; done
+ make -C doc $@
+ rm -rf source.list dist skytools-*
+ find python -name '*.pyc' | xargs rm -f
+ rm -rf dist build
+ rm -rf autom4te.cache config.log config.status config.mak
+
+deb80:
+ ./configure
+ sed -e s/PGVER/8.0/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages
+ yada rebuild
+ debuild -uc -us -b
+
+deb81:
+ ./configure
+ sed -e s/PGVER/8.1/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages
+ yada rebuild
+ debuild -uc -us -b
+
+deb82:
+ ./configure
+ sed -e s/PGVER/8.2/g -e s/PYVER/$(pyver)/g < debian/packages.in > debian/packages
+ yada rebuild
+ debuild -uc -us -b
+
+tgz: config.mak
+ $(PYTHON) setup.py sdist -t source.cfg -m source.list
+
+debclean: distclean
+ rm -rf debian/tmp-* debian/build* debian/control debian/packages-tmp*
+ rm -f debian/files debian/rules debian/sub* debian/packages
+
+boot: configure
+
+configure: configure.ac
+ autoconf
+
+
+.PHONY: all clean distclean install deb debclean tgz
+.PHONY: python-all python-clean python-install
+
--- /dev/null
+
+2007-03-xx ver 2.1 "Radioactive Candy"
+
+ * Final public release.
+
--- /dev/null
+
+SkyTools - tools for PostgreSQL
+===============================
+
+This is a package of tools in use in Skype for replication and
+failover. Also it includes a generic queuing mechanism PgQ and
+utility library for Python scripts.
+
+It contains following tools:
+
+PgQ
+---
+
+This is the queue machanism we use. Consists of PL/pgsql, PL/python
+and C code in database, with Python framework on top of it. It is
+based on snapshot based event handling ideas from Slony-I,
+written for general usage.
+
+Features:
+
+ * There can be several queues in database.
+ * There can be several producers than can insert into any queue.
+ * There can be several consumers on one queue and all consumers
+ see all events.
+
+
+Londiste
+--------
+
+Replication tool written in Python, using PgQ as event transport.
+
+Features:
+- Tables can be added one-by-one into set.
+- Initial COPY for one table does not block event replay
+ for other tables.
+- Can compare tables on both sides.
+
+
+walmgr
+------
+
+This script will setup WAL archiving, does initial backup and
+runtime WAL archive and restore.
+
+
+
+
--- /dev/null
+
+prefix = @prefix@
+
+override PYTHON = @PYTHON@
+override PG_CONFIG = @PG_CONFIG@
+
+PGXS = $(shell $(PG_CONFIG) --pgxs)
+
+DESTDIR = /
+
--- /dev/null
+dnl Process this file with autoconf to produce a configure script.
+
+AC_INIT(skytools, 2.1)
+AC_CONFIG_SRCDIR(python/pgqadm.py)
+
+dnl Find Python interpreter
+AC_ARG_WITH(python, [ --with-python=PYTHON name of the Python executable (default: python)],
+[ AC_MSG_CHECKING(for python)
+ PYTHON=$withval
+ AC_MSG_RESULT($PYTHON)],
+[ AC_PATH_PROGS(PYTHON, python) ])
+test -n "$PYTHON" || AC_MSG_ERROR([Cannot continue without Python])
+
+dnl Find PostgreSQL pg_config
+AC_ARG_WITH(pgconfig, [ --with-pgconfig=PG_CONFIG path to pg_config (default: pg_config)],
+[ AC_MSG_CHECKING(for pg_config)
+ PG_CONFIG=$withval
+ AC_MSG_RESULT($PG_CONFIG)],
+[ AC_PATH_PROGS(PG_CONFIG, pg_config) ])
+test -n "$PG_CONFIG" || AC_MSG_ERROR([Cannot continue without pg_config])
+
+dnl Write result
+AC_OUTPUT([config.mak])
+
--- /dev/null
+skytools (2.1) unstable; urgency=low
+
+ * cleanup
+
+ -- Marko Kreen <marko.kreen@skype.net> Fri, 02 Feb 2007 12:38:17 +0200
+
--- /dev/null
+## debian/packages for skytools
+
+Source: skytools
+Section: contrib/misc
+Priority: extra
+Maintainer: Marko Kreen <marko.kreen@skype.net>
+Standards-Version: 3.6.2
+Description: PostgreSQL
+Copyright: BSD
+ Copyright 2006 Marko Kreen
+Build: sh
+ PG_CONFIG=/usr/lib/postgresql/PGVER/bin/pg_config \
+ ./configure --prefix=/usr
+ make DESTDIR=$ROOT
+Clean: sh
+ make distclean || make clean || true
+Build-Depends: python-dev, postgresql-server-dev-PGVER
+
+Package: skytools
+Architecture: any
+Depends: python-psycopg | pythonPYVER-psycopg, skytools-modules-8.1 | skytools-modules-8.0, []
+Description: Skype database tools - Python parts
+ .
+ londiste - replication
+ pgqadm - generic event queue
+ walmgr - failover server scripts
+Install: sh
+ make python-install DESTDIR=$ROOT prefix=/usr
+
+Package: skytools-modules-PGVER
+Architecture: any
+Depends: postgresql-PGVER, []
+Conflicts: postgresql-extras-PGVER
+Description: Extra modules for PostgreSQL
+ It includes various extra modules for PostgreSQL:
+ .
+ txid - 8-byte transaction id's
+ logtriga - Trigger function to log change in SQL format.
+ logutriga - Trigger function to log change in urlencoded format.
+ londiste - Database parts of replication engine.
+ pgq - Generic queue in database.
+Install: sh
+ make modules-install DESTDIR=$ROOT
+
--- /dev/null
+
+wiki = https://developer.skype.com/SkypeGarage/DbProjects/SkyTools
+
+web = mkz@shell.pgfoundry.org:/home/pgfoundry.org/groups/skytools/htdocs/
+
+EPYARGS = --no-private -u "http://pgfoundry.org/projects/skytools/" \
+ -n "Skytools"
+
+all:
+
+upload:
+ devupload.sh overview.txt $(wiki)
+ devupload.sh londiste.txt $(wiki)/LondisteUsage
+ devupload.sh pgq-sql.txt $(wiki)/PgQdocs
+ devupload.sh pgq-nodupes.txt $(wiki)/PgqNoDupes
+ devupload.sh walmgr.txt $(wiki)/WalMgr
+ devupload.sh pgq-admin.txt $(wiki)/PgqAdm
+
+PY_PKGS = skytools skytools.config skytools.dbstruct skytools.gzlog \
+ skytools.quoting skytools.scripting skytools.sqltools \
+ pgq pgq.consumer pgq.event pgq.maint pgq.producer pgq.status pgq.ticker \
+ londiste londiste.compare londiste.file_read londiste.file_write \
+ londiste.installer londiste.playback londiste.repair londiste.setup \
+ londiste.syncer londiste.table_copy
+
+apidoc:
+ rm -rf api
+ mkdir -p api
+ cd ../python && epydoc3 -o ../doc/api --html --no-private $(PY_PKGS)
+
+apiupload: apidoc
+ cd ../sql/pgq && rm -rf docs/pgq && make dox && mv docs/html docs/pgq
+ rsync -rtlz api $(web)
+ rsync -rtlz ../sql/pgq/docs/pgq $(web)
+
+clean:
+ rm -rf api
+
+distclean:
+ rm -rf ../sql/pgq/docs/pgq api
+
--- /dev/null
+
+web:
+ - walmgr
+ - pgqadm
+ - todo
+
+
+londiste link <qname>
+londiste unlink <qname>
+
+Immidiate
+=========
+
+* londiste swithcover support / deny triggers
+* deb: /etc/skylog.ini should be conffile
+* RemoteConsumer/SerialConsumer/pgq_ext sanity, too much duplication
+
+Near future
+============
+
+* londiste: create tables on subscriber
+* skytools: switch for silence for cron scripts
+* docs: londiste, pgq/python, pgq/sql, skytools
+* txid: decide on renaming functions
+* logtriga: way to switch off logging for some connection
+* pgq: separately installable fkeys for all tables for testing
+* logdb: hostname
+* pgq_ext: solve event tracking
+* contrib/*.sql loading from python - need to check db version
+* ideas from SlonyI:
+ - force timestamps to ISO
+ - when buffering queries, check their size
+* DBScript: failure to write pidfile should be logged (crontscripts)
+
+Just ideas
+===========
+
+* logtriga: use pgq.insert_event_directly?
+* pgq/sql: rewrite pgq.insert_event in C or logtriga in plpython?
+* skytools: config-less operation?
+* skytools: config from database?
+* skytools: partial sql parser for log processing
+* pgqadm ticker logic into db, to make easier other implementations?
+
--- /dev/null
+
+
+== Config file ==
+
+{{{
+[londiste]
+job_name = test_to_subcriber
+
+# source database, where the queue resides
+provider_db = dbname=provider port=6000 host=127.0.0.1
+
+# destination database
+subscriber_db = dbname=subscriber port=6000 host=127.0.0.1
+
+# the queue where to listen on
+pgq_queue_name = londiste.replika
+
+# where to log
+logfile = ~/log/%(job_name)s.log
+
+# pidfile is used for avoiding duplicate processes
+pidfile = ~/pid/%(job_name)s.pid
+
+}}}
+
+== Command line overview ==
+
+{{{
+$ londiste.py --help
+usage: londiste.py [options] INI CMD [subcmd args]
+
+commands:
+ provider install installs modules, creates queue
+ provider add TBL ... add table to queue
+ provider remove TBL ... remove table from queue
+ provider tables show all tables linked to queue
+ subscriber install installs schema
+ subscriber register attaches subscriber to queue (also done by replay)
+ subscriber unregister detach subscriber from queue
+ subscriber add TBL ... add table to subscriber
+ subscriber remove TBL ... remove table from subscriber
+ subscriber resync TBL ... do full copy again
+ subscriber tables list tables subscriber has attached to
+ subscriber missing list tables subscriber has not yet attached to
+ subscriber check compare table structure on both sides
+ subscriber fkeys print out fkey drop/create commands
+ replay replay events to subscriber
+ copy full copy of table, internal cmd
+ compare [TBL ...] compare table contents on both sides
+ repair [TBL ...] repair data on subscriber
+
+options:
+ -h, --help show this help message and exit
+ -q, --quiet make program silent
+ -v, --verbose make program verbose
+ -d, --daemon go background
+ --expect-sync no copy needed (for add command)
+ --force ignore some warnings
+
+ control running process:
+ -r, --reload reload config (send SIGHUP)
+ -s, --stop stop program safely (send SIGINT)
+ -k, --kill kill program immidiately (send SIGTERM)
+}}}
--- /dev/null
+#pragma section-numbers 2
+
+= SkyTools =
+
+[[TableOfContents]]
+
+== Intro ==
+
+This is package of tools we use at Skype to manage our cluster of [http://www.postgresql.org PostgreSQL]
+servers. They are put together for our own convinience and also because they build on each other,
+so managing them separately is pain.
+
+The code is hosted at [http://pgfoundry.org PgFoundry] site:
+
+ http://pgfoundry.org/projects/skytools/
+
+There are our [http://pgfoundry.org/frs/?group_id=1000206 downloads] and
+[http://lists.pgfoundry.org/mailman/listinfo/skytools-users mailing list].
+Also [http://pgfoundry.org/scm/?group_id=1000206 CVS]
+and [http://pgfoundry.org/tracker/?group_id=1000206 bugtracker].
+
+== High-level tools ==
+
+Those are script that are meant for end-user.
+In our case that means database administrators.
+
+=== Londiste ===
+
+Replication engine written in Python. It uses PgQ as transport mechanism.
+Its main goals are robustness and easy usage. Thus its not as complete
+and featureful as Slony-I.
+
+Docs: ./LondisteUsage
+
+''' Features '''
+
+ * Tables can be added one-by-one into set.
+ * Initial COPY for one table does not block event replay for other tables.
+ * Can compare tables on both sides.
+ * Easy installation.
+
+''' Missing features '''
+
+ * No support for sequences. Thus its not possible to use it for keeping
+ failover server up-to-date. We use WalMgr for that.
+
+ * Does not understand cascaded replication, when one subscriber acts
+ as provider to another one and it dies, the last one loses sync with the first one.
+ In other words - it understands only pair of servers.
+
+''' Sample usage '''
+{{{
+$ londiste.py replic.ini provider install
+$ londiste.py replic.ini subscriber install
+$ londiste.py replic.ini replay -d
+$ londiste.py replic.ini provider add users orders
+$ londiste.py replic.ini subscriber add users
+}}}
+
+=== PgQ ===
+
+Generic queue implementation. Based on ideas from [http://www.slony1.info/ Slony-I] -
+snapshot based event batching.
+
+''' Features '''
+
+ * Generic multi-consumer, multi-producer queue.
+ * There can be several consumers on one queue.
+ * It is guaranteed that each of them sees a event at least once.
+ But it's not guaranteed that it sees it only once.
+ * The goal is to provide a clean API as SQL functions. The frameworks
+ on top of that don't need to understand internal details.
+
+''' Technical design '''
+
+ * Events are batched using snapshots (like Slony-I).
+ * Consumers are poll-only, they don't need to do any administrative work.
+ * Queue administration is separate process from consumers.
+ * Tolerant of long transactions.
+ * Easy to monitor.
+
+''' Docs '''
+
+ * SQL API overview: ./PgQdocs
+ * SQL API detailed docs: http://skytools.projects.postgresql.org/pgq/
+ * Administrative tool usage: ./PgqAdm
+
+=== WalMgr ===
+
+Python script for hot failover. Tries to make setup
+initial copy and later switch easy for admins.
+
+ * Docs: ./WalMgr
+
+Sample:
+
+{{{
+ [ .. prepare config .. ]
+
+ master$ walmgr setup
+ master$ walmgr backup
+ slave$ walmgr restore
+
+ [ .. main server down, switch failover server to normal mode: ]
+
+ slave$ walmgr restore
+}}}
+
+== Low-level tools ==
+
+Those are building blocks for the PgQ and Londiste.
+Useful for database developers.
+
+=== txid ===
+
+ Provides 8-byte transaction id-s for external usage.
+
+=== logtriga ===
+
+ Trigger function for table event logging in "partial SQL" format.
+ Based on Slony-I logtrigger. Used in londiste for replication.
+
+=== logutriga ===
+
+ Trigger function for table event logging in urlencoded format.
+ Written in PL/Python. For cases where data manipulation is necessary.
+
+== Developement frameworks ==
+
+=== skytools - Python framework for database scripting ===
+
+This collect various utilities for Python scripts for databases.
+
+''' Topics '''
+
+ * Daemonization
+ * Logging
+ * Configuration.
+ * Skeleton class for scripts.
+ * Quoting (SQL/COPY)
+ * COPY helpers.
+ * Database object lookup.
+ * Table structure detection.
+
+Documentation: http://skytools.projects.postgresql.org/api/
+
+=== pgq - Python framework for PgQ consumers ===
+
+This builds on scripting framework above.
+
+Documentation: http://skytools.projects.postgresql.org/api/
+
+== Sample scripts ==
+
+Those are specialized script that are based on skytools/pgq framework.
+Can be considered examples, although they are used in production in Skype.
+
+=== Special data moving scripts ===
+
+There are couple of scripts for situations where regular replication
+does not fit. They all operate on `logutriga()` urlencoded queues.
+
+ * `cube_dispatcher`: Multi-table partitioning on change date, with optional keep-all-row-versions mode.
+ * `table_dispatcher`: configurable partitioning for one table.
+ * `bulk_loader`: aggregates changes for slow databases. Instead of each change in separate statement,
+ does minimal amount of DELETE-s and then big COPY.
+
+|| Script || Supported operations || Number of tables || Partitioning ||
+|| table_dispatcher || INSERT || 1 || any ||
+|| cube_dispatcher || INSERT/UPDATE || any || change time ||
+|| bulk_loader || INSERT/UPDATE/DELETE || any || none ||
+
+=== queue_mover ===
+
+Simply copies all events from one queue to another.
+
+=== scriptmgr ===
+
+Allows to start and stop several scripts together.
--- /dev/null
+= PgqAdm =
+
+== Config ==
+
+{{{
+[pgqadm]
+job_name = pgqadm_somedb
+
+db = dbname=somedb
+
+# how often to run maintenance [minutes]
+maint_delay_min = 5
+
+# how often to check for activity [secs]
+loop_delay = 0.1
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+}}}
+
+== Command line usage ==
+
+{{{
+$ pgqadm.py --help
+usage: pgqadm.py [options] INI CMD [subcmd args]
+
+commands:
+ ticker start ticking & maintenance process
+ status show overview of queue healt
+
+options:
+ -h, --help show this help message and exit
+ -q, --quiet make program silent
+ -v, --verbose make program verbose
+ -d, --daemon go background
+
+ control running process:
+ -r, --reload reload config (send SIGHUP)
+ -s, --stop stop program safely (send SIGINT)
+ -k, --kill kill program immidiately (send SIGTERM)
+}}}
--- /dev/null
+= Avoiding duplicate events =\r
+\r
+It is pretty burdensome to check if event is already processed,\r
+especially on bulk data moving. Here's a way how this can be avoided.\r
+\r
+First, consumer must guarantee that it processes all events in one tx.\r
+\r
+Consumer itself can tag events for retry, but then it must be able to handle them later.\r
+\r
+ * If the PgQ queue and event data handling happen in same database,\r
+ the consumer must simply call pgq.finish_batch() inside the event-processing\r
+ transaction.\r
+\r
+ * If the event processing happens in different database, the consumer\r
+ must store the batch_id into destination database, inside the same\r
+ transaction as the event processing happens.\r
+\r
+ Only after committing it, consumer can call pgq.finish_batch() in queue database\r
+ and commit that.\r
+\r
+ As the batches come in sequence, there's no need to remember full log of batch_id's,\r
+ it's enough to keep the latest batch_id.\r
+\r
+ Then at the start of every batch, consumer can check if the batch_id already\r
+ exists in destination database, and if it does, then just tag batch done,\r
+ without processing.\r
+\r
+With this, there's no need for consumer to check for already processed\r
+events.\r
+\r
+NB: This assumes the event processing is transaction-able - failures\r
+will be rollbacked. If event processing includes communication with\r
+world outside database, eg. sending email, such handling won't work.\r
--- /dev/null
+= PgQ - queue for PostgreSQL =
+
+== Queue creation ==
+
+{{{
+ pgq.create_queue(queue_name text)
+}}}
+
+Initialize event queue.
+
+Returns 0 if event queue already exists, 1 otherwise.
+
+== Producer ==
+
+{{{
+ pgq.insert_event(queue_name text, ev_type, ev_data)
+ pgq.insert_event(queue_name text, ev_type, ev_data, extra1, extra2, extra3, extra4)
+}}}
+
+Generate new event. This should be called inside main tx - thus
+rollbacked with it if needed.
+
+
+== Consumer ==
+
+{{{
+ pgq.register_consumer(queue_name text, consumer_id text)
+}}}
+
+Attaches this consumer to particular event queue.
+
+Returns 0 if the consumer was already attached, 1 otherwise.
+
+{{{
+ pgq.unregister_consumer(queue_name text, consumer_id text)
+}}}
+
+Unregister and drop resources allocated to customer.
+
+
+{{{
+ pgq.next_batch(queue_name text, consumer_id text)
+}}}
+
+Allocates next batch of events to consumer.
+
+Returns batch id (int8), to be used in processing functions. If no batches
+are available, returns NULL. That means that the ticker has not cut them yet.
+This is the appropriate moment for consumer to sleep.
+
+{{{
+ pgq.get_batch_events(batch_id int8)
+}}}
+
+`pgq.get_batch_events()` returns a set of events in this batch.
+
+There may be no events in the batch. This is normal. The batch must still be closed
+with pgq.finish_batch().
+
+Event fields: (ev_id int8, ev_time timestamptz, ev_txid int8, ev_retry int4, ev_type text,
+ ev_data text, ev_extra1, ev_extra2, ev_extra3, ev_extra4)
+
+{{{
+ pgq.event_failed(batch_id int8, event_id int8, reason text)
+}}}
+
+Tag event as 'failed' - it will be stored, but not further processing is done.
+
+{{{
+ pgq.event_retry(batch_id int8, event_id int8, retry_seconds int4)
+}}}
+
+Tag event for 'retry' - after x seconds the event will be re-inserted
+into main queue.
+
+{{{
+ pgq.finish_batch(batch_id int8)
+}}}
+
+Tag batch as finished. Until this is not done, the consumer will get
+same batch again.
+
+After calling finish_batch consumer cannot do any operations with events
+of that batch. All operations must be done before.
+
+== Failed queue operation ==
+
+Events tagged as failed just stay on their queue. Following
+functions can be used to manage them.
+
+{{{
+ pgq.failed_event_list(queue_name, consumer)
+ pgq.failed_event_list(queue_name, consumer, cnt, offset)
+ pgq.failed_event_count(queue_name, consumer)
+}}}
+
+Get info about the queue.
+
+Event fields are same as for pgq.get_batch_events()
+
+{{{
+ pgq.failed_event_delete(queue_name, consumer, event_id)
+ pgq.failed_event_retry(queue_name, consumer, event_id)
+}}}
+
+Remove an event from queue, or retry it.
+
+== Info operations ==
+
+{{{
+ pgq.get_queue_info()
+}}}
+
+Get list of queues.
+
+Result: ()
+
+{{{
+ pgq.get_consumer_info()
+ pgq.get_consumer_info(queue_name)
+ pgq.get_consumer_info(queue_name, consumer)
+}}}
+
+Get list of active consumers.
+
+Result: ()
+
+{{{
+ pgq.get_batch_info(batch_id)
+}}}
+
+Get info about batch.
+
+Result fields: ()
+
+== Notes ==
+
+Consumer '''must''' be able to process same event several times.
+
+== Example ==
+
+First, create event queue:
+
+{{{
+ select pgq.create_queue('LogEvent');
+}}}
+
+Then, producer side can do whenever it wishes:
+
+{{{
+ select pgq.insert_event('LogEvent', 'data', 'DataFor123');
+}}}
+
+First step for consumer is to register:
+
+{{{
+ select pgq.register_consumer('LogEvent', 'TestConsumer');
+}}}
+
+Then it can enter into consuming loop:
+
+{{{
+ begin;
+ select pgq.next_batch('LogEvent', 'TestConsumer'); [into batch_id]
+ commit;
+}}}
+
+That will reserve a batch of events for this consumer.
+
+To see the events in batch:
+
+{{{
+ select * from pgq.get_batch_events(batch_id);
+}}}
+
+That will give all events in batch. The processing does not need to be happen
+all in one transaction, framework can split the work how it wants.
+
+If a events failed or needs to be tried again, framework can call:
+
+{{{
+ select pgq.event_retry(batch_id, event_id, 60);
+ select pgq.event_failed(batch_id, event_id, 'Record deleted');
+}}}
+
+When all done, notify core about it:
+
+{{{
+ select pgq.finish_batch(batch_id)
+}}}
+
--- /dev/null
+#pragma section-numbers 2
+
+= WalMgr =
+
+[[TableOfContents]]
+
+== Step-by-step instructions ==
+
+=== no-password ssh access from one to other ===
+
+ master$ test -f ~/.ssh/id_dsa.pub || ssh-keygen -t dsa
+ master$ scp .ssh/id_dsa.pub slave:
+ slave$ cat id_dsa.pub >> ~/.ssh/authorized_keys
+
+=== Configure paths ===
+
+ master$ edit master.ini
+ slave$ edit slave.ini
+ slave$ mkdir data.master logs.full logs.partial
+
+=== Start archival process ===
+
+ master$ ./walmgr.py setup
+
+=== Do full backup+restore ===
+
+ master$ ./walmgr.py backup
+ slave$ ./walmgr.py restore
+
+ 'walmgr.py restore' moves data in place and starts postmaster,
+ that starts replaying logs as they appear.
+
+=== In-progress WAL segments can be backup by command: ===
+
+ master$ ./walmgr.py sync
+
+=== If need to stop replay on slave and boot into normal mode, do: ===
+
+ slave$ ./walmgr.py boot
+
+== Configuration ==
+
+=== master.ini ===
+
+{{{
+[wal-master]
+logfile = master.log
+use_skylog = 0
+
+master_db = dbname=template1
+master_data = /var/lib/postgresql/8.0/main
+master_config = /etc/postgresql/8.0/main/postgresql.conf
+
+slave = slave:/var/lib/postgresql/walshipping
+
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+
+# syncdaemon update frequency
+loop_delay = 10.0
+
+}}}
+
+=== slave.ini ===
+
+{{{
+[wal-slave]
+logfile = slave.log
+use_skylog = 0
+
+slave_data = /var/lib/postgresql/8.0/main
+slave_stop_cmd = /etc/init.d/postgresql-8.0 stop
+slave_start_cmd = /etc/init.d/postgresql-8.0 start
+
+slave = /var/lib/postgresql/walshipping
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+
+keep_old_logs = 0
+}}}
--- /dev/null
+
+[londiste]
+job_name = test_to_subcriber
+
+provider_db = dbname=provider port=6000 host=127.0.0.1
+subscriber_db = dbname=subscriber port=6000 host=127.0.0.1
+
+# it will be used as sql ident so no dots/spaces
+pgq_queue_name = londiste.replika
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+# both events and ticks will be copied there
+#mirror_queue = replika_mirror
+
--- /dev/null
+
+[pgqadm]
+
+job_name = pgqadm_somedb
+
+db = dbname=provider port=6000 host=127.0.0.1
+
+# how often to run maintenance [minutes]
+maint_delay_min = 5
+
+# how often to check for activity [secs]
+loop_delay = 0.1
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+use_skylog = 0
+
--- /dev/null
+; notes:
+; - 'args' is mandatory in [handler_*] sections
+; - in lists there must not be spaces
+
+;
+; top-level config
+;
+
+; list of all loggers
+[loggers]
+keys=root
+; root logger sees everything. there can be per-job configs by
+; specifing loggers with job_name of the script
+
+; list of all handlers
+[handlers]
+;; seems logger module immidiately initalized all handlers,
+;; whether they are actually used or not. so better
+;; keep this list in sync with actual handler list
+;keys=stderr,logdb,logsrv,logfile
+keys=stderr
+
+; list of all formatters
+[formatters]
+keys=short,long,none
+
+;
+; map specific loggers to specifig handlers
+;
+[logger_root]
+level=DEBUG
+;handlers=stderr,logdb,logsrv,logfile
+handlers=stderr
+
+;
+; configure formatters
+;
+[formatter_short]
+format=%(asctime)s %(levelname)s %(message)s
+datefmt=%H:%M
+
+[formatter_long]
+format=%(asctime)s %(process)s %(levelname)s %(message)s
+
+[formatter_none]
+format=%(message)s
+
+;
+; configure handlers
+;
+
+; file. args: stream
+[handler_stderr]
+class=StreamHandler
+args=(sys.stderr,)
+formatter=short
+
+; log into db. args: conn_string
+[handler_logdb]
+class=skylog.LogDBHandler
+args=("host=127.0.0.1 port=5432 user=logger dbname=logdb",)
+formatter=none
+level=INFO
+
+; JSON messages over UDP. args: host, port
+[handler_logsrv]
+class=skylog.UdpLogServerHandler
+args=('127.0.0.1', 6666)
+formatter=none
+
+; rotating logfile. args: filename, maxsize, maxcount
+[handler_logfile]
+class=skylog.EasyRotatingFileHandler
+args=('~/log/%(job_name)s.log', 100*1024*1024, 3)
+formatter=long
+
--- /dev/null
+[wal-master]
+logfile = master.log
+use_skylog = 0
+
+master_db = dbname=template1
+master_data = /var/lib/postgresql/8.0/main
+master_config = /etc/postgresql/8.0/main/postgresql.conf
+
+
+slave = slave:/var/lib/postgresql/walshipping
+
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+
+# syncdaemon update frequency
+loop_delay = 10.0
+
--- /dev/null
+[wal-slave]
+logfile = slave.log
+use_skylog = 0
+
+slave_data = /var/lib/postgresql/8.0/main
+slave_stop_cmd = /etc/init.d/postgresql-8.0 stop
+slave_start_cmd = /etc/init.d/postgresql-8.0 start
+
+slave = /var/lib/postgresql/walshipping
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+
+keep_old_logs = 0
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Londiste launcher.
+"""
+
+import sys, os, optparse, skytools
+
+# python 2.3 will try londiste.py first...
+import sys, os.path
+if os.path.exists(os.path.join(sys.path[0], 'londiste.py')) \
+ and not os.path.exists(os.path.join(sys.path[0], 'londiste')):
+ del sys.path[0]
+
+from londiste import *
+
+command_usage = """
+%prog [options] INI CMD [subcmd args]
+
+commands:
+ provider install installs modules, creates queue
+ provider add TBL ... add table to queue
+ provider remove TBL ... remove table from queue
+ provider tables show all tables linked to queue
+ provider seqs show all sequences on provider
+
+ subscriber install installs schema
+ subscriber add TBL ... add table to subscriber
+ subscriber remove TBL ... remove table from subscriber
+ subscriber tables list tables subscriber has attached to
+ subscriber seqs list sequences subscriber is interested
+ subscriber missing list tables subscriber has not yet attached to
+ subscriber link QUE create mirror queue
+ subscriber unlink dont mirror queue
+
+ replay replay events to subscriber
+
+ switchover switch the roles between provider & subscriber
+ compare [TBL ...] compare table contents on both sides
+ repair [TBL ...] repair data on subscriber
+ copy full copy of table, internal cmd
+ subscriber check compare table structure on both sides
+ subscriber fkeys print out fkey drop/create commands
+ subscriber resync TBL ... do full copy again
+ subscriber register attaches subscriber to queue (also done by replay)
+ subscriber unregister detach subscriber from queue
+"""
+
+"""switchover:
+goal is to launch a replay with reverse config
+
+- should link be required? (link should guarantee all tables/seqs same?)
+- should link auto-add tables for subscriber
+
+1. lock all tables on provider, in order specified by 'nr'
+2. wait until old replay is past the point
+3. sync seq
+4. replace queue triggers on provider with deny triggers
+5. replace deny triggers on subscriber with queue triggers
+6. sync pgq tick seqs? change pgq config?
+
+"""
+
+class Londiste(skytools.DBScript):
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'londiste', args)
+
+ if self.options.rewind or self.options.reset:
+ self.script = Replicator(args)
+ return
+
+ if len(self.args) < 2:
+ print "need command"
+ sys.exit(1)
+ cmd = self.args[1]
+
+ if cmd =="provider":
+ script = ProviderSetup(args)
+ elif cmd == "subscriber":
+ script = SubscriberSetup(args)
+ elif cmd == "replay":
+ method = self.cf.get('method', 'direct')
+ if method == 'direct':
+ script = Replicator(args)
+ elif method == 'file_write':
+ script = FileWrite(args)
+ elif method == 'file_write':
+ script = FileWrite(args)
+ else:
+ print "unknown method, quitting"
+ sys.exit(1)
+ elif cmd == "copy":
+ script = CopyTable(args)
+ elif cmd == "compare":
+ script = Comparator(args)
+ elif cmd == "repair":
+ script = Repairer(args)
+ elif cmd == "upgrade":
+ script = UpgradeV2(args)
+ else:
+ print "Unknown command '%s', use --help for help" % cmd
+ sys.exit(1)
+
+ self.script = script
+
+ def start(self):
+ self.script.start()
+
+ def init_optparse(self, parser=None):
+ p = skytools.DBScript.init_optparse(self, parser)
+ p.set_usage(command_usage.strip())
+
+ g = optparse.OptionGroup(p, "expert options")
+ g.add_option("--force", action="store_true",
+ help = "add: ignore table differences, repair: ignore lag")
+ g.add_option("--expect-sync", action="store_true", dest="expect_sync",
+ help = "add: no copy needed", default=False)
+ g.add_option("--skip-truncate", action="store_true", dest="skip_truncate",
+ help = "copy: keep old data", default=False)
+ g.add_option("--rewind", action="store_true",
+ help = "replay: sync queue pos with subscriber")
+ g.add_option("--reset", action="store_true",
+ help = "replay: forget queue pos on subscriber")
+ p.add_option_group(g)
+
+ return p
+
+if __name__ == '__main__':
+ script = Londiste(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+"""Replication on top of PgQ."""
+
+from playback import *
+from compare import *
+from file_read import *
+from file_write import *
+from setup import *
+from table_copy import *
+from installer import *
+from repair import *
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Compares tables in replication set.
+
+Currently just does count(1) on both sides.
+"""
+
+import sys, os, time, skytools
+
+__all__ = ['Comparator']
+
+from syncer import Syncer
+
+class Comparator(Syncer):
+ def process_sync(self, tbl, src_db, dst_db):
+ """Actual comparision."""
+
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+
+ self.log.info('Counting %s' % tbl)
+
+ q = "select count(1) from only _TABLE_"
+ q = self.cf.get('compare_sql', q)
+ q = q.replace('_TABLE_', tbl)
+
+ self.log.debug("srcdb: " + q)
+ src_curs.execute(q)
+ src_row = src_curs.fetchone()
+ src_str = ", ".join(map(str, src_row))
+ self.log.info("srcdb: res = %s" % src_str)
+
+ self.log.debug("dstdb: " + q)
+ dst_curs.execute(q)
+ dst_row = dst_curs.fetchone()
+ dst_str = ", ".join(map(str, dst_row))
+ self.log.info("dstdb: res = %s" % dst_str)
+
+ if src_str != dst_str:
+ self.log.warning("%s: Results do not match!" % tbl)
+
+if __name__ == '__main__':
+ script = Comparator(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+"""Reads events from file instead of db queue."""
+
+import sys, os, re, skytools
+
+from playback import *
+from table_copy import *
+
+__all__ = ['FileRead']
+
+file_regex = r"^tick_0*([0-9]+)\.sql$"
+file_rc = re.compile(file_regex)
+
+
+class FileRead(CopyTable):
+ """Reads events from file instead of db queue.
+
+ Incomplete implementation.
+ """
+
+ def __init__(self, args, log = None):
+ CopyTable.__init__(self, args, log, copy_thread = 0)
+
+ def launch_copy(self, tbl):
+ # copy immidiately
+ self.do_copy(t)
+
+ def work(self):
+ last_batch = self.get_last_batch(curs)
+ list = self.get_file_list()
+
+ def get_list(self):
+ """Return list of (first_batch, full_filename) pairs."""
+
+ src_dir = self.cf.get('file_src')
+ list = os.listdir(src_dir)
+ list.sort()
+ res = []
+ for fn in list:
+ m = file_rc.match(fn)
+ if not m:
+ self.log.debug("Ignoring file: %s" % fn)
+ continue
+ full = os.path.join(src_dir, fn)
+ batch_id = int(m.group(1))
+ res.append((batch_id, full))
+ return res
+
+if __name__ == '__main__':
+ script = Replicator(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+"""Writes events into file."""
+
+import sys, os, skytools
+from cStringIO import StringIO
+from playback import *
+
+__all__ = ['FileWrite']
+
+class FileWrite(Replicator):
+ """Writes events into file.
+
+ Incomplete implementation.
+ """
+
+ last_successful_batch = None
+
+ def load_state(self, batch_id):
+ # maybe check if batch exists on filesystem?
+ self.cur_tick = self.cur_batch_info['tick_id']
+ self.prev_tick = self.cur_batch_info['prev_tick_id']
+ return 1
+
+ def process_batch(self, db, batch_id, ev_list):
+ pass
+
+ def save_state(self, do_commit):
+ # nothing to save
+ pass
+
+ def sync_tables(self, dst_db):
+ # nothing to sync
+ return 1
+
+ def interesting(self, ev):
+ # wants all of them
+ return 1
+
+ def handle_data_event(self, ev):
+ fmt = self.sql_command[ev.type]
+ sql = fmt % (ev.ev_extra1, ev.data)
+ row = "%s -- txid:%d" % (sql, ev.txid)
+ self.sql_list.append(row)
+ ev.tag_done()
+
+ def handle_system_event(self, ev):
+ row = "-- sysevent:%s txid:%d data:%s" % (
+ ev.type, ev.txid, ev.data)
+ self.sql_list.append(row)
+ ev.tag_done()
+
+ def flush_sql(self):
+ self.sql_list.insert(0, "-- tick:%d prev:%s" % (
+ self.cur_tick, self.prev_tick))
+ self.sql_list.append("-- end_tick:%d\n" % self.cur_tick)
+ # store result
+ dir = self.cf.get("file_dst")
+ fn = os.path.join(dir, "tick_%010d.sql" % self.cur_tick)
+ f = open(fn, "w")
+ buf = "\n".join(self.sql_list)
+ f.write(buf)
+ f.close()
+
+if __name__ == '__main__':
+ script = Replicator(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+"""Functions to install londiste and its depentencies into database."""
+
+import os, skytools
+
+__all__ = ['install_provider', 'install_subscriber']
+
+provider_object_list = [
+ skytools.DBFunction('logtriga', 0, sql_file = "logtriga.sql"),
+ skytools.DBFunction('get_current_snapshot', 0, sql_file = "txid.sql"),
+ skytools.DBSchema('pgq', sql_file = "pgq.sql"),
+ skytools.DBSchema('londiste', sql_file = "londiste.sql")
+]
+
+subscriber_object_list = [
+ skytools.DBSchema('londiste', sql_file = "londiste.sql")
+]
+
+def install_provider(curs, log):
+ """Installs needed code into provider db."""
+ skytools.db_install(curs, provider_object_list, log)
+
+def install_subscriber(curs, log):
+ """Installs needed code into subscriber db."""
+ skytools.db_install(curs, subscriber_object_list, log)
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Basic replication core."""
+
+import sys, os, time
+import skytools, pgq
+
+__all__ = ['Replicator', 'TableState',
+ 'TABLE_MISSING', 'TABLE_IN_COPY', 'TABLE_CATCHING_UP',
+ 'TABLE_WANNA_SYNC', 'TABLE_DO_SYNC', 'TABLE_OK']
+
+# state # owner - who is allowed to change
+TABLE_MISSING = 0 # main
+TABLE_IN_COPY = 1 # copy
+TABLE_CATCHING_UP = 2 # copy
+TABLE_WANNA_SYNC = 3 # main
+TABLE_DO_SYNC = 4 # copy
+TABLE_OK = 5 # setup
+
+SYNC_OK = 0 # continue with batch
+SYNC_LOOP = 1 # sleep, try again
+SYNC_EXIT = 2 # nothing to do, exit skript
+
+class Counter(object):
+ """Counts table statuses."""
+
+ missing = 0
+ copy = 0
+ catching_up = 0
+ wanna_sync = 0
+ do_sync = 0
+ ok = 0
+
+ def __init__(self, tables):
+ """Counts and sanity checks."""
+ for t in tables:
+ if t.state == TABLE_MISSING:
+ self.missing += 1
+ elif t.state == TABLE_IN_COPY:
+ self.copy += 1
+ elif t.state == TABLE_CATCHING_UP:
+ self.catching_up += 1
+ elif t.state == TABLE_WANNA_SYNC:
+ self.wanna_sync += 1
+ elif t.state == TABLE_DO_SYNC:
+ self.do_sync += 1
+ elif t.state == TABLE_OK:
+ self.ok += 1
+ # only one table is allowed to have in-progress copy
+ if self.copy + self.catching_up + self.wanna_sync + self.do_sync > 1:
+ raise Exception('Bad table state')
+
+class TableState(object):
+ """Keeps state about one table."""
+ def __init__(self, name, log):
+ self.name = name
+ self.log = log
+ self.forget()
+ self.changed = 0
+
+ def forget(self):
+ self.state = TABLE_MISSING
+ self.str_snapshot = None
+ self.from_snapshot = None
+ self.sync_tick_id = None
+ self.ok_batch_count = 0
+ self.last_tick = 0
+ self.changed = 1
+
+ def change_snapshot(self, str_snapshot, tag_changed = 1):
+ if self.str_snapshot == str_snapshot:
+ return
+ self.log.debug("%s: change_snapshot to %s" % (self.name, str_snapshot))
+ self.str_snapshot = str_snapshot
+ if str_snapshot:
+ self.from_snapshot = skytools.Snapshot(str_snapshot)
+ else:
+ self.from_snapshot = None
+
+ if tag_changed:
+ self.ok_batch_count = 0
+ self.last_tick = None
+ self.changed = 1
+
+ def change_state(self, state, tick_id = None):
+ if self.state == state and self.sync_tick_id == tick_id:
+ return
+ self.state = state
+ self.sync_tick_id = tick_id
+ self.changed = 1
+ self.log.debug("%s: change_state to %s" % (self.name,
+ self.render_state()))
+
+ def render_state(self):
+ """Make a string to be stored in db."""
+
+ if self.state == TABLE_MISSING:
+ return None
+ elif self.state == TABLE_IN_COPY:
+ return 'in-copy'
+ elif self.state == TABLE_CATCHING_UP:
+ return 'catching-up'
+ elif self.state == TABLE_WANNA_SYNC:
+ return 'wanna-sync:%d' % self.sync_tick_id
+ elif self.state == TABLE_DO_SYNC:
+ return 'do-sync:%d' % self.sync_tick_id
+ elif self.state == TABLE_OK:
+ return 'ok'
+
+ def parse_state(self, merge_state):
+ """Read state from string."""
+
+ state = -1
+ if merge_state == None:
+ state = TABLE_MISSING
+ elif merge_state == "in-copy":
+ state = TABLE_IN_COPY
+ elif merge_state == "catching-up":
+ state = TABLE_CATCHING_UP
+ elif merge_state == "ok":
+ state = TABLE_OK
+ elif merge_state == "?":
+ state = TABLE_OK
+ else:
+ tmp = merge_state.split(':')
+ if len(tmp) == 2:
+ self.sync_tick_id = int(tmp[1])
+ if tmp[0] == 'wanna-sync':
+ state = TABLE_WANNA_SYNC
+ elif tmp[0] == 'do-sync':
+ state = TABLE_DO_SYNC
+
+ if state < 0:
+ raise Exception("Bad table state: %s" % merge_state)
+
+ return state
+
+ def loaded_state(self, merge_state, str_snapshot):
+ self.log.debug("loaded_state: %s: %s / %s" % (
+ self.name, merge_state, str_snapshot))
+ self.change_snapshot(str_snapshot, 0)
+ self.state = self.parse_state(merge_state)
+ self.changed = 0
+ if merge_state == "?":
+ self.changed = 1
+
+ def interesting(self, ev, tick_id, copy_thread):
+ """Check if table wants this event."""
+
+ if copy_thread:
+ if self.state not in (TABLE_CATCHING_UP, TABLE_DO_SYNC):
+ return False
+ else:
+ if self.state != TABLE_OK:
+ return False
+
+ # if no snapshot tracking, then accept always
+ if not self.from_snapshot:
+ return True
+
+ # uninteresting?
+ if self.from_snapshot.contains(ev.txid):
+ return False
+
+ # after couple interesting batches there no need to check snapshot
+ # as there can be only one partially interesting batch
+ if tick_id != self.last_tick:
+ self.last_tick = tick_id
+ self.ok_batch_count += 1
+
+ # disable batch tracking
+ if self.ok_batch_count > 3:
+ self.change_snapshot(None)
+ return True
+
+class SeqCache(object):
+ def __init__(self):
+ self.seq_list = []
+ self.val_cache = {}
+
+ def set_seq_list(self, seq_list):
+ self.seq_list = seq_list
+ new_cache = {}
+ for seq in seq_list:
+ val = self.val_cache.get(seq)
+ if val:
+ new_cache[seq] = val
+ self.val_cache = new_cache
+
+ def resync(self, src_curs, dst_curs):
+ if len(self.seq_list) == 0:
+ return
+ dat = ".last_value, ".join(self.seq_list)
+ dat += ".last_value"
+ q = "select %s from %s" % (dat, ",".join(self.seq_list))
+ src_curs.execute(q)
+ row = src_curs.fetchone()
+ for i in range(len(self.seq_list)):
+ seq = self.seq_list[i]
+ cur = row[i]
+ old = self.val_cache.get(seq)
+ if old != cur:
+ q = "select setval(%s, %s)"
+ dst_curs.execute(q, [seq, cur])
+ self.val_cache[seq] = cur
+
+class Replicator(pgq.SerialConsumer):
+ """Replication core."""
+
+ sql_command = {
+ 'I': "insert into %s %s;",
+ 'U': "update only %s set %s;",
+ 'D': "delete from only %s where %s;",
+ }
+
+ # batch info
+ cur_tick = 0
+ prev_tick = 0
+
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, 'londiste', 'provider_db', 'subscriber_db', args)
+
+ # tick table in dst for SerialConsumer(). keep londiste stuff under one schema
+ self.dst_completed_table = "londiste.completed"
+
+ self.table_list = []
+ self.table_map = {}
+
+ self.copy_thread = 0
+ self.maint_time = 0
+ self.seq_cache = SeqCache()
+ self.maint_delay = self.cf.getint('maint_delay', 600)
+ self.mirror_queue = self.cf.get('mirror_queue', '')
+
+ def process_remote_batch(self, src_db, batch_id, ev_list, dst_db):
+ "All work for a batch. Entry point from SerialConsumer."
+
+ # this part can play freely with transactions
+
+ dst_curs = dst_db.cursor()
+
+ self.cur_tick = self.cur_batch_info['tick_id']
+ self.prev_tick = self.cur_batch_info['prev_tick_id']
+
+ self.load_table_state(dst_curs)
+ self.sync_tables(dst_db)
+
+ # now the actual event processing happens.
+ # they must be done all in one tx in dst side
+ # and the transaction must be kept open so that
+ # the SerialConsumer can save last tick and commit.
+
+ self.handle_seqs(dst_curs)
+ self.handle_events(dst_curs, ev_list)
+ self.save_table_state(dst_curs)
+
+ def handle_seqs(self, dst_curs):
+ if self.copy_thread:
+ return
+
+ q = "select * from londiste.subscriber_get_seq_list(%s)"
+ dst_curs.execute(q, [self.pgq_queue_name])
+ seq_list = []
+ for row in dst_curs.fetchall():
+ seq_list.append(row[0])
+
+ self.seq_cache.set_seq_list(seq_list)
+
+ src_curs = self.get_database('provider_db').cursor()
+ self.seq_cache.resync(src_curs, dst_curs)
+
+ def sync_tables(self, dst_db):
+ """Table sync loop.
+
+ Calls appropriate handles, which is expected to
+ return one of SYNC_* constants."""
+
+ self.log.debug('Sync tables')
+ while 1:
+ cnt = Counter(self.table_list)
+ if self.copy_thread:
+ res = self.sync_from_copy_thread(cnt, dst_db)
+ else:
+ res = self.sync_from_main_thread(cnt, dst_db)
+
+ if res == SYNC_EXIT:
+ self.log.debug('Sync tables: exit')
+ self.detach()
+ sys.exit(0)
+ elif res == SYNC_OK:
+ return
+ elif res != SYNC_LOOP:
+ raise Exception('Program error')
+
+ self.log.debug('Sync tables: sleeping')
+ time.sleep(3)
+ dst_db.commit()
+ self.load_table_state(dst_db.cursor())
+ dst_db.commit()
+
+ def sync_from_main_thread(self, cnt, dst_db):
+ "Main thread sync logic."
+
+ #
+ # decide what to do - order is imortant
+ #
+ if cnt.do_sync:
+ # wait for copy thread to catch up
+ return SYNC_LOOP
+ elif cnt.wanna_sync:
+ # copy thread wants sync, if not behind, do it
+ t = self.get_table_by_state(TABLE_WANNA_SYNC)
+ if self.cur_tick >= t.sync_tick_id:
+ self.change_table_state(dst_db, t, TABLE_DO_SYNC, self.cur_tick)
+ return SYNC_LOOP
+ else:
+ return SYNC_OK
+ elif cnt.catching_up:
+ # active copy, dont worry
+ return SYNC_OK
+ elif cnt.copy:
+ # active copy, dont worry
+ return SYNC_OK
+ elif cnt.missing:
+ # seems there is no active copy thread, launch new
+ t = self.get_table_by_state(TABLE_MISSING)
+ self.change_table_state(dst_db, t, TABLE_IN_COPY)
+
+ # the copy _may_ happen immidiately
+ self.launch_copy(t)
+
+ # there cannot be interesting events in current batch
+ # but maybe there's several tables, lets do them in one go
+ return SYNC_LOOP
+ else:
+ # seems everything is in sync
+ return SYNC_OK
+
+ def sync_from_copy_thread(self, cnt, dst_db):
+ "Copy thread sync logic."
+
+ #
+ # decide what to do - order is imortant
+ #
+ if cnt.do_sync:
+ # main thread is waiting, catch up, then handle over
+ t = self.get_table_by_state(TABLE_DO_SYNC)
+ if self.cur_tick == t.sync_tick_id:
+ self.change_table_state(dst_db, t, TABLE_OK)
+ return SYNC_EXIT
+ elif self.cur_tick < t.sync_tick_id:
+ return SYNC_OK
+ else:
+ self.log.error("copy_sync: cur_tick=%d sync_tick=%d" % (
+ self.cur_tick, t.sync_tick_id))
+ raise Exception('Invalid table state')
+ elif cnt.wanna_sync:
+ # wait for main thread to react
+ return SYNC_LOOP
+ elif cnt.catching_up:
+ # is there more work?
+ if self.work_state:
+ return SYNC_OK
+
+ # seems we have catched up
+ t = self.get_table_by_state(TABLE_CATCHING_UP)
+ self.change_table_state(dst_db, t, TABLE_WANNA_SYNC, self.cur_tick)
+ return SYNC_LOOP
+ elif cnt.copy:
+ # table is not copied yet, do it
+ t = self.get_table_by_state(TABLE_IN_COPY)
+ self.do_copy(t)
+
+ # forget previous value
+ self.work_state = 1
+
+ return SYNC_LOOP
+ else:
+ # nothing to do
+ return SYNC_EXIT
+
+ def handle_events(self, dst_curs, ev_list):
+ "Actual event processing happens here."
+
+ ignored_events = 0
+ self.sql_list = []
+ mirror_list = []
+ for ev in ev_list:
+ if not self.interesting(ev):
+ ignored_events += 1
+ ev.tag_done()
+ continue
+
+ if ev.type in ('I', 'U', 'D'):
+ self.handle_data_event(ev, dst_curs)
+ else:
+ self.handle_system_event(ev, dst_curs)
+
+ if self.mirror_queue:
+ mirror_list.append(ev)
+
+ # finalize table changes
+ self.flush_sql(dst_curs)
+ self.stat_add('ignored', ignored_events)
+
+ # put events into mirror queue if requested
+ if self.mirror_queue:
+ self.fill_mirror_queue(mirror_list, dst_curs)
+
+ def handle_data_event(self, ev, dst_curs):
+ fmt = self.sql_command[ev.type]
+ sql = fmt % (ev.extra1, ev.data)
+ self.sql_list.append(sql)
+ if len(self.sql_list) > 200:
+ self.flush_sql(dst_curs)
+ ev.tag_done()
+
+ def flush_sql(self, dst_curs):
+ if len(self.sql_list) == 0:
+ return
+
+ buf = "\n".join(self.sql_list)
+ self.sql_list = []
+
+ dst_curs.execute(buf)
+
+ def interesting(self, ev):
+ if ev.type not in ('I', 'U', 'D'):
+ return 1
+ t = self.get_table_by_name(ev.extra1)
+ if t:
+ return t.interesting(ev, self.cur_tick, self.copy_thread)
+ else:
+ return 0
+
+ def handle_system_event(self, ev, dst_curs):
+ "System event."
+
+ if ev.type == "T":
+ self.log.info("got new table event: "+ev.data)
+ # check tables to be dropped
+ name_list = []
+ for name in ev.data.split(','):
+ name_list.append(name.strip())
+
+ del_list = []
+ for tbl in self.table_list:
+ if tbl.name in name_list:
+ continue
+ del_list.append(tbl)
+
+ # separate loop to avoid changing while iterating
+ for tbl in del_list:
+ self.log.info("Removing table %s from set" % tbl.name)
+ self.remove_table(tbl, dst_curs)
+
+ ev.tag_done()
+ else:
+ self.log.warning("Unknows op %s" % ev.type)
+ ev.tag_failed("Unknown operation")
+
+ def remove_table(self, tbl, dst_curs):
+ del self.table_map[tbl.name]
+ self.table_list.remove(tbl)
+ q = "select londiste.subscriber_remove_table(%s, %s)"
+ dst_curs.execute(q, [self.pgq_queue_name, tbl.name])
+
+ def load_table_state(self, curs):
+ """Load table state from database.
+
+ Todo: if all tables are OK, there is no need
+ to load state on every batch.
+ """
+
+ q = """select table_name, snapshot, merge_state
+ from londiste.subscriber_get_table_list(%s)
+ """
+ curs.execute(q, [self.pgq_queue_name])
+
+ new_list = []
+ new_map = {}
+ for row in curs.dictfetchall():
+ t = self.get_table_by_name(row['table_name'])
+ if not t:
+ t = TableState(row['table_name'], self.log)
+ t.loaded_state(row['merge_state'], row['snapshot'])
+ new_list.append(t)
+ new_map[t.name] = t
+
+ self.table_list = new_list
+ self.table_map = new_map
+
+ def save_table_state(self, curs):
+ """Store changed table state in database."""
+
+ for t in self.table_list:
+ if not t.changed:
+ continue
+ merge_state = t.render_state()
+ self.log.info("storing state of %s: copy:%d new_state:%s" % (
+ t.name, self.copy_thread, merge_state))
+ q = "select londiste.subscriber_set_table_state(%s, %s, %s, %s)"
+ curs.execute(q, [self.pgq_queue_name,
+ t.name, t.str_snapshot, merge_state])
+ t.changed = 0
+
+ def change_table_state(self, dst_db, tbl, state, tick_id = None):
+ tbl.change_state(state, tick_id)
+ self.save_table_state(dst_db.cursor())
+ dst_db.commit()
+
+ self.log.info("Table %s status changed to '%s'" % (
+ tbl.name, tbl.render_state()))
+
+ def get_table_by_state(self, state):
+ "get first table with specific state"
+
+ for t in self.table_list:
+ if t.state == state:
+ return t
+ raise Exception('No table was found with state: %d' % state)
+
+ def get_table_by_name(self, name):
+ if name.find('.') < 0:
+ name = "public.%s" % name
+ if name in self.table_map:
+ return self.table_map[name]
+ return None
+
+ def fill_mirror_queue(self, ev_list, dst_curs):
+ # insert events
+ rows = []
+ fields = ['ev_type', 'ev_data', 'ev_extra1']
+ for ev in mirror_list:
+ rows.append((ev.type, ev.data, ev.extra1))
+ pgq.bulk_insert_events(dst_curs, rows, fields, self.mirror_queue)
+
+ # create tick
+ q = "select pgq.ticker(%s, %s)"
+ dst_curs.execute(q, [self.mirror_queue, self.cur_tick])
+
+ def launch_copy(self, tbl_stat):
+ self.log.info("Launching copy process")
+ script = sys.argv[0]
+ conf = self.cf.filename
+ if self.options.verbose:
+ cmd = "%s -d -v %s copy"
+ else:
+ cmd = "%s -d %s copy"
+ cmd = cmd % (script, conf)
+ self.log.debug("Launch args: "+repr(cmd))
+ res = os.system(cmd)
+ self.log.debug("Launch result: "+repr(res))
+
+if __name__ == '__main__':
+ script = Replicator(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+"""Repair data on subscriber.
+
+Walks tables by primary key and searcher
+missing inserts/updates/deletes.
+"""
+
+import sys, os, time, psycopg, skytools
+
+from syncer import Syncer
+
+__all__ = ['Repairer']
+
+def unescape(s):
+ return skytools.unescape_copy(s)
+
+def get_pkey_list(curs, tbl):
+ """Get list of pkey fields in right order."""
+
+ oid = skytools.get_table_oid(curs, tbl)
+ q = """SELECT k.attname FROM pg_index i, pg_attribute k
+ WHERE i.indrelid = %s AND k.attrelid = i.indexrelid
+ AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped
+ ORDER BY k.attnum"""
+ curs.execute(q, [oid])
+ list = []
+ for row in curs.fetchall():
+ list.append(row[0])
+ return list
+
+def get_column_list(curs, tbl):
+ """Get list of columns in right order."""
+
+ oid = skytools.get_table_oid(curs, tbl)
+ q = """SELECT a.attname FROM pg_attribute a
+ WHERE a.attrelid = %s
+ AND a.attnum > 0 AND NOT a.attisdropped
+ ORDER BY a.attnum"""
+ curs.execute(q, [oid])
+ list = []
+ for row in curs.fetchall():
+ list.append(row[0])
+ return list
+
+class Repairer(Syncer):
+ """Walks tables in primary key order and checks if data matches."""
+
+
+ def process_sync(self, tbl, src_db, dst_db):
+ """Actual comparision."""
+
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+
+ self.log.info('Checking %s' % tbl)
+
+ self.common_fields = []
+ self.pkey_list = []
+ copy_tbl = self.gen_copy_tbl(tbl, src_curs, dst_curs)
+
+ dump_src = tbl + ".src"
+ dump_dst = tbl + ".dst"
+
+ self.log.info("Dumping src table: %s" % tbl)
+ self.dump_table(tbl, copy_tbl, src_curs, dump_src)
+ src_db.commit()
+ self.log.info("Dumping dst table: %s" % tbl)
+ self.dump_table(tbl, copy_tbl, dst_curs, dump_dst)
+ dst_db.commit()
+
+ self.log.info("Sorting src table: %s" % tbl)
+
+ s_in, s_out = os.popen4("sort --version")
+ s_ver = s_out.read()
+ del s_in, s_out
+ if s_ver.find("coreutils") > 0:
+ args = "-S 30%"
+ else:
+ args = ""
+ os.system("sort %s -T . -o %s.sorted %s" % (args, dump_src, dump_src))
+ self.log.info("Sorting dst table: %s" % tbl)
+ os.system("sort %s -T . -o %s.sorted %s" % (args, dump_dst, dump_dst))
+
+ self.dump_compare(tbl, dump_src + ".sorted", dump_dst + ".sorted")
+
+ os.unlink(dump_src)
+ os.unlink(dump_dst)
+ os.unlink(dump_src + ".sorted")
+ os.unlink(dump_dst + ".sorted")
+
+ def gen_copy_tbl(self, tbl, src_curs, dst_curs):
+ self.pkey_list = get_pkey_list(src_curs, tbl)
+ dst_pkey = get_pkey_list(dst_curs, tbl)
+ if dst_pkey != self.pkey_list:
+ self.log.error('pkeys do not match')
+ sys.exit(1)
+
+ src_cols = get_column_list(src_curs, tbl)
+ dst_cols = get_column_list(dst_curs, tbl)
+ field_list = []
+ for f in self.pkey_list:
+ field_list.append(f)
+ for f in src_cols:
+ if f in self.pkey_list:
+ continue
+ if f in dst_cols:
+ field_list.append(f)
+
+ self.common_fields = field_list
+
+ tbl_expr = "%s (%s)" % (tbl, ",".join(field_list))
+
+ self.log.debug("using copy expr: %s" % tbl_expr)
+
+ return tbl_expr
+
+ def dump_table(self, tbl, copy_tbl, curs, fn):
+ f = open(fn, "w", 64*1024)
+ curs.copy_to(f, copy_tbl)
+ size = f.tell()
+ f.close()
+ self.log.info('Got %d bytes' % size)
+
+ def get_row(self, ln):
+ t = ln[:-1].split('\t')
+ row = {}
+ for i in range(len(self.common_fields)):
+ row[self.common_fields[i]] = t[i]
+ return row
+
+ def dump_compare(self, tbl, src_fn, dst_fn):
+ self.log.info("Comparing dumps: %s" % tbl)
+ self.cnt_insert = 0
+ self.cnt_update = 0
+ self.cnt_delete = 0
+ self.total_src = 0
+ self.total_dst = 0
+ f1 = open(src_fn, "r", 64*1024)
+ f2 = open(dst_fn, "r", 64*1024)
+ src_ln = f1.readline()
+ dst_ln = f2.readline()
+ if src_ln: self.total_src += 1
+ if dst_ln: self.total_dst += 1
+
+ fix = "fix.%s.sql" % tbl
+ if os.path.isfile(fix):
+ os.unlink(fix)
+
+ while src_ln or dst_ln:
+ keep_src = keep_dst = 0
+ if src_ln != dst_ln:
+ src_row = self.get_row(src_ln)
+ dst_row = self.get_row(dst_ln)
+
+ cmp = self.cmp_keys(src_row, dst_row)
+ if cmp > 0:
+ # src > dst
+ self.got_missed_delete(tbl, dst_row)
+ keep_src = 1
+ elif cmp < 0:
+ # src < dst
+ self.got_missed_insert(tbl, src_row)
+ keep_dst = 1
+ else:
+ if self.cmp_data(src_row, dst_row) != 0:
+ self.got_missed_update(tbl, src_row, dst_row)
+
+ if not keep_src:
+ src_ln = f1.readline()
+ if src_ln: self.total_src += 1
+ if not keep_dst:
+ dst_ln = f2.readline()
+ if dst_ln: self.total_dst += 1
+
+ self.log.info("finished %s: src: %d rows, dst: %d rows,"\
+ " missed: %d inserts, %d updates, %d deletes" % (
+ tbl, self.total_src, self.total_dst,
+ self.cnt_insert, self.cnt_update, self.cnt_delete))
+
+ def got_missed_insert(self, tbl, src_row):
+ self.cnt_insert += 1
+ fld_list = self.common_fields
+ val_list = []
+ for f in fld_list:
+ v = unescape(src_row[f])
+ val_list.append(skytools.quote_literal(v))
+ q = "insert into %s (%s) values (%s);" % (
+ tbl, ", ".join(fld_list), ", ".join(val_list))
+ self.show_fix(tbl, q, 'insert')
+
+ def got_missed_update(self, tbl, src_row, dst_row):
+ self.cnt_update += 1
+ fld_list = self.common_fields
+ set_list = []
+ whe_list = []
+ for f in self.pkey_list:
+ self.addcmp(whe_list, f, unescape(src_row[f]))
+ for f in fld_list:
+ v1 = src_row[f]
+ v2 = dst_row[f]
+ if self.cmp_value(v1, v2) == 0:
+ continue
+
+ self.addeq(set_list, f, unescape(v1))
+ self.addcmp(whe_list, f, unescape(v2))
+
+ q = "update only %s set %s where %s;" % (
+ tbl, ", ".join(set_list), " and ".join(whe_list))
+ self.show_fix(tbl, q, 'update')
+
+ def got_missed_delete(self, tbl, dst_row, pkey_list):
+ self.cnt_delete += 1
+ whe_list = []
+ for f in self.pkey_list:
+ self.addcmp(whe_list, f, unescape(dst_row[f]))
+ q = "delete from only %s where %s;" % (tbl, " and ".join(whe_list))
+ self.show_fix(tbl, q, 'delete')
+
+ def show_fix(self, tbl, q, desc):
+ #self.log.warning("missed %s: %s" % (desc, q))
+ fn = "fix.%s.sql" % tbl
+ open(fn, "a").write("%s\n" % q)
+
+ def addeq(self, list, f, v):
+ vq = skytools.quote_literal(v)
+ s = "%s = %s" % (f, vq)
+ list.append(s)
+
+ def addcmp(self, list, f, v):
+ if v is None:
+ s = "%s is null" % f
+ else:
+ vq = skytools.quote_literal(v)
+ s = "%s = %s" % (f, vq)
+ list.append(s)
+
+ def cmp_data(self, src_row, dst_row):
+ for k in self.common_fields:
+ v1 = src_row[k]
+ v2 = dst_row[k]
+ if self.cmp_value(v1, v2) != 0:
+ return -1
+ return 0
+
+ def cmp_value(self, v1, v2):
+ if v1 == v2:
+ return 0
+
+ # try to work around tz vs. notz
+ z1 = len(v1)
+ z2 = len(v2)
+ if z1 == z2 + 3 and z2 >= 19 and v1[z2] == '+':
+ v1 = v1[:-3]
+ if v1 == v2:
+ return 0
+ elif z1 + 3 == z2 and z1 >= 19 and v2[z1] == '+':
+ v2 = v2[:-3]
+ if v1 == v2:
+ return 0
+
+ return -1
+
+ def cmp_keys(self, src_row, dst_row):
+ """Compare primary keys of the rows.
+
+ Returns 1 if src > dst, -1 if src < dst and 0 if src == dst"""
+
+ # None means table is done. tag it larger than any existing row.
+ if src_row is None:
+ if dst_row is None:
+ return 0
+ return 1
+ elif dst_row is None:
+ return -1
+
+ for k in self.pkey_list:
+ v1 = src_row[k]
+ v2 = dst_row[k]
+ if v1 < v2:
+ return -1
+ elif v1 > v2:
+ return 1
+ return 0
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Londiste setup and sanity checker.
+
+"""
+import sys, os, skytools
+from installer import *
+
+__all__ = ['ProviderSetup', 'SubscriberSetup']
+
+def find_column_types(curs, table):
+ table_oid = skytools.get_table_oid(curs, table)
+ if table_oid == None:
+ return None
+
+ key_sql = """
+ SELECT k.attname FROM pg_index i, pg_attribute k
+ WHERE i.indrelid = %d AND k.attrelid = i.indexrelid
+ AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped
+ """ % table_oid
+
+ # find columns
+ q = """
+ SELECT a.attname as name,
+ CASE WHEN k.attname IS NOT NULL
+ THEN 'k' ELSE 'v' END AS type
+ FROM pg_attribute a LEFT JOIN (%s) k ON (k.attname = a.attname)
+ WHERE a.attrelid = %d AND a.attnum > 0 AND NOT a.attisdropped
+ ORDER BY a.attnum
+ """ % (key_sql, table_oid)
+ curs.execute(q)
+ rows = curs.dictfetchall()
+ return rows
+
+def make_type_string(col_rows):
+ res = map(lambda x: x['type'], col_rows)
+ return "".join(res)
+
+class CommonSetup(skytools.DBScript):
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'londiste', args)
+ self.set_single_loop(1)
+ self.pidfile = self.pidfile + ".setup"
+
+ self.pgq_queue_name = self.cf.get("pgq_queue_name")
+ self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name)
+ self.fake = self.cf.getint('fake', 0)
+
+ if len(self.args) < 3:
+ self.log.error("need subcommand")
+ sys.exit(1)
+
+ def run(self):
+ self.admin()
+
+ def fetch_provider_table_list(self, curs):
+ q = """select table_name, trigger_name
+ from londiste.provider_get_table_list(%s)"""
+ curs.execute(q, [self.pgq_queue_name])
+ return curs.dictfetchall()
+
+ def get_provider_table_list(self):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ list = self.fetch_provider_table_list(src_curs)
+ src_db.commit()
+ res = []
+ for row in list:
+ res.append(row['table_name'])
+ return res
+
+ def get_provider_seqs(self, curs):
+ q = """SELECT * from londiste.provider_get_seq_list(%s)"""
+ curs.execute(q, [self.pgq_queue_name])
+ res = []
+ for row in curs.fetchall():
+ res.append(row[0])
+ return res
+
+ def get_all_seqs(self, curs):
+ q = """SELECT n.nspname || '.'|| c.relname
+ from pg_class c, pg_namespace n
+ where n.oid = c.relnamespace
+ and c.relkind = 'S'
+ order by 1"""
+ curs.execute(q)
+ res = []
+ for row in curs.fetchall():
+ res.append(row[0])
+ return res
+
+ def check_provider_queue(self):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ q = "select count(1) from pgq.get_queue_info(%s)"
+ src_curs.execute(q, [self.pgq_queue_name])
+ ok = src_curs.fetchone()[0]
+ src_db.commit()
+
+ if not ok:
+ self.log.error('Event queue does not exist yet')
+ sys.exit(1)
+
+ def fetch_subscriber_tables(self, curs):
+ q = "select * from londiste.subscriber_get_table_list(%s)"
+ curs.execute(q, [self.pgq_queue_name])
+ return curs.dictfetchall()
+
+ def get_subscriber_table_list(self):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ list = self.fetch_subscriber_tables(dst_curs)
+ dst_db.commit()
+ res = []
+ for row in list:
+ res.append(row['table_name'])
+ return res
+
+ def init_optparse(self, parser=None):
+ p = skytools.DBScript.init_optparse(self, parser)
+ p.add_option("--expect-sync", action="store_true", dest="expect_sync",
+ help = "no copy needed", default=False)
+ p.add_option("--force", action="store_true",
+ help="force", default=False)
+ return p
+
+
+#
+# Provider commands
+#
+
+class ProviderSetup(CommonSetup):
+
+ def admin(self):
+ cmd = self.args[2]
+ if cmd == "tables":
+ self.provider_show_tables()
+ elif cmd == "add":
+ self.provider_add_tables(self.args[3:])
+ elif cmd == "remove":
+ self.provider_remove_tables(self.args[3:])
+ elif cmd == "add-seq":
+ for seq in self.args[3:]:
+ self.provider_add_seq(seq)
+ self.provider_notify_change()
+ elif cmd == "remove-seq":
+ for seq in self.args[3:]:
+ self.provider_remove_seq(seq)
+ self.provider_notify_change()
+ elif cmd == "install":
+ self.provider_install()
+ elif cmd == "seqs":
+ self.provider_list_seqs()
+ else:
+ self.log.error('bad subcommand')
+ sys.exit(1)
+
+ def provider_list_seqs(self):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ list = self.get_provider_seqs(src_curs)
+ src_db.commit()
+
+ for seq in list:
+ print seq
+
+ def provider_install(self):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ install_provider(src_curs, self.log)
+
+ # create event queue
+ q = "select pgq.create_queue(%s)"
+ self.exec_provider(q, [self.pgq_queue_name])
+
+ def provider_add_tables(self, table_list):
+ self.check_provider_queue()
+
+ cur_list = self.get_provider_table_list()
+ for tbl in table_list:
+ if tbl.find('.') < 0:
+ tbl = "public." + tbl
+ if tbl not in cur_list:
+ self.log.info('Adding %s' % tbl)
+ self.provider_add_table(tbl)
+ else:
+ self.log.info("Table %s already added" % tbl)
+ self.provider_notify_change()
+
+ def provider_remove_tables(self, table_list):
+ self.check_provider_queue()
+
+ cur_list = self.get_provider_table_list()
+ for tbl in table_list:
+ if tbl.find('.') < 0:
+ tbl = "public." + tbl
+ if tbl not in cur_list:
+ self.log.info('%s already removed' % tbl)
+ else:
+ self.log.info("Removing %s" % tbl)
+ self.provider_remove_table(tbl)
+ self.provider_notify_change()
+
+ def provider_add_table(self, tbl):
+ q = "select londiste.provider_add_table(%s, %s)"
+ self.exec_provider(q, [self.pgq_queue_name, tbl])
+
+ def provider_remove_table(self, tbl):
+ q = "select londiste.provider_remove_table(%s, %s)"
+ self.exec_provider(q, [self.pgq_queue_name, tbl])
+
+ def provider_show_tables(self):
+ self.check_provider_queue()
+ list = self.get_provider_table_list()
+ for tbl in list:
+ print tbl
+
+ def provider_notify_change(self):
+ q = "select londiste.provider_notify_change(%s)"
+ self.exec_provider(q, [self.pgq_queue_name])
+
+ def provider_add_seq(self, seq):
+ seq = skytools.fq_name(seq)
+ q = "select londiste.provider_add_seq(%s, %s)"
+ self.exec_provider(q, [self.pgq_queue_name, seq])
+
+ def provider_remove_seq(self, seq):
+ seq = skytools.fq_name(seq)
+ q = "select londiste.provider_remove_seq(%s, %s)"
+ self.exec_provider(q, [self.pgq_queue_name, seq])
+
+ def exec_provider(self, sql, args):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+
+ src_curs.execute(sql, args)
+
+ if self.fake:
+ src_db.rollback()
+ else:
+ src_db.commit()
+
+#
+# Subscriber commands
+#
+
+class SubscriberSetup(CommonSetup):
+
+ def admin(self):
+ cmd = self.args[2]
+ if cmd == "tables":
+ self.subscriber_show_tables()
+ elif cmd == "missing":
+ self.subscriber_missing_tables()
+ elif cmd == "add":
+ self.subscriber_add_tables(self.args[3:])
+ elif cmd == "remove":
+ self.subscriber_remove_tables(self.args[3:])
+ elif cmd == "resync":
+ self.subscriber_resync_tables(self.args[3:])
+ elif cmd == "register":
+ self.subscriber_register()
+ elif cmd == "unregister":
+ self.subscriber_unregister()
+ elif cmd == "install":
+ self.subscriber_install()
+ elif cmd == "check":
+ self.check_tables(self.get_provider_table_list())
+ elif cmd == "fkeys":
+ self.collect_fkeys(self.get_provider_table_list())
+ elif cmd == "seqs":
+ self.subscriber_list_seqs()
+ elif cmd == "add-seq":
+ self.subscriber_add_seq(self.args[3:])
+ elif cmd == "remove-seq":
+ self.subscriber_remove_seq(self.args[3:])
+ else:
+ self.log.error('bad subcommand: ' + cmd)
+ sys.exit(1)
+
+ def collect_fkeys(self, table_list):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+
+ oid_list = []
+ for tbl in table_list:
+ try:
+ oid = skytools.get_table_oid(dst_curs, tbl)
+ if oid:
+ oid_list.append(str(oid))
+ except:
+ pass
+ if len(oid_list) == 0:
+ print "No tables"
+ return
+ oid_str = ",".join(oid_list)
+
+ q = "SELECT n.nspname || '.' || t.relname as tbl, c.conname as con,"\
+ " pg_get_constraintdef(c.oid) as def"\
+ " FROM pg_constraint c, pg_class t, pg_namespace n"\
+ " WHERE c.contype = 'f' and c.conrelid in (%s)"\
+ " AND t.oid = c.conrelid AND n.oid = t.relnamespace" % oid_str
+ dst_curs.execute(q)
+ res = dst_curs.dictfetchall()
+ dst_db.commit()
+
+ print "-- dropping"
+ for row in res:
+ q = "ALTER TABLE ONLY %(tbl)s DROP CONSTRAINT %(con)s;"
+ print q % row
+ print "-- creating"
+ for row in res:
+ q = "ALTER TABLE ONLY %(tbl)s ADD CONSTRAINT %(con)s %(def)s;"
+ print q % row
+
+ def check_tables(self, table_list):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+
+ failed = 0
+ for tbl in table_list:
+ self.log.info('Checking %s' % tbl)
+ if not skytools.exists_table(src_curs, tbl):
+ self.log.error('Table %s missing from provider side' % tbl)
+ failed += 1
+ elif not skytools.exists_table(dst_curs, tbl):
+ self.log.error('Table %s missing from subscriber side' % tbl)
+ failed += 1
+ else:
+ failed += self.check_table_columns(src_curs, dst_curs, tbl)
+ failed += self.check_table_triggers(dst_curs, tbl)
+
+ src_db.commit()
+ dst_db.commit()
+
+ return failed
+
+ def check_table_triggers(self, dst_curs, tbl):
+ oid = skytools.get_table_oid(dst_curs, tbl)
+ if not oid:
+ self.log.error('Table %s not found' % tbl)
+ return 1
+ q = "select count(1) from pg_trigger where tgrelid = %s"
+ dst_curs.execute(q, [oid])
+ got = dst_curs.fetchone()[0]
+ if got:
+ self.log.error('found trigger on table %s (%s)' % (tbl, str(oid)))
+ return 1
+ else:
+ return 0
+
+ def check_table_columns(self, src_curs, dst_curs, tbl):
+ src_colrows = find_column_types(src_curs, tbl)
+ dst_colrows = find_column_types(dst_curs, tbl)
+
+ src_cols = make_type_string(src_colrows)
+ dst_cols = make_type_string(dst_colrows)
+ if src_cols.find('k') < 0:
+ self.log.error('provider table %s has no primary key (%s)' % (
+ tbl, src_cols))
+ return 1
+ if dst_cols.find('k') < 0:
+ self.log.error('subscriber table %s has no primary key (%s)' % (
+ tbl, dst_cols))
+ return 1
+
+ if src_cols != dst_cols:
+ self.log.warning('table %s structure is not same (%s/%s)'\
+ ', trying to continue' % (tbl, src_cols, dst_cols))
+
+ err = 0
+ for row in src_colrows:
+ found = 0
+ for row2 in dst_colrows:
+ if row2['name'] == row['name']:
+ found = 1
+ break
+ if not found:
+ err = 1
+ self.log.error('%s: column %s on provider not on subscriber'
+ % (tbl, row['name']))
+ elif row['type'] != row2['type']:
+ err = 1
+ self.log.error('%s: pk different on column %s'
+ % (tbl, row['name']))
+
+ return err
+
+ def subscriber_install(self):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+
+ install_subscriber(dst_curs, self.log)
+
+ if self.fake:
+ self.log.debug('rollback')
+ dst_db.rollback()
+ else:
+ self.log.debug('commit')
+ dst_db.commit()
+
+ def subscriber_register(self):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ src_curs.execute("select pgq.register_consumer(%s, %s)",
+ [self.pgq_queue_name, self.consumer_id])
+ src_db.commit()
+
+ def subscriber_unregister(self):
+ q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)"
+
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ tbl_rows = self.fetch_subscriber_tables(dst_curs)
+ for row in tbl_rows:
+ dst_curs.execute(q, [self.pgq_queue_name, row['table_name']])
+ dst_db.commit()
+
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ src_curs.execute("select pgq.unregister_consumer(%s, %s)",
+ [self.pgq_queue_name, self.consumer_id])
+ src_db.commit()
+
+ def subscriber_show_tables(self):
+ list = self.get_subscriber_table_list()
+ for tbl in list:
+ print tbl
+
+ def subscriber_missing_tables(self):
+ provider_tables = self.get_provider_table_list()
+ subscriber_tables = self.get_subscriber_table_list()
+ for tbl in provider_tables:
+ if tbl not in subscriber_tables:
+ print tbl
+
+ def subscriber_add_tables(self, table_list):
+ provider_tables = self.get_provider_table_list()
+ subscriber_tables = self.get_subscriber_table_list()
+
+ err = 0
+ for tbl in table_list:
+ tbl = skytools.fq_name(tbl)
+ if tbl not in provider_tables:
+ err = 1
+ self.log.error("Table %s not attached to queue" % tbl)
+ if err:
+ if self.options.force:
+ self.log.warning('--force used, ignoring errors')
+ else:
+ sys.exit(1)
+
+ err = self.check_tables(table_list)
+ if err:
+ if self.options.force:
+ self.log.warning('--force used, ignoring errors')
+ else:
+ sys.exit(1)
+
+ for tbl in table_list:
+ tbl = skytools.fq_name(tbl)
+ if tbl in subscriber_tables:
+ self.log.info("Table %s already added" % tbl)
+ else:
+ self.log.info("Adding %s" % tbl)
+ self.subscriber_add_one_table(tbl)
+
+ def subscriber_remove_tables(self, table_list):
+ subscriber_tables = self.get_subscriber_table_list()
+ for tbl in table_list:
+ tbl = skytools.fq_name(tbl)
+ if tbl in subscriber_tables:
+ self.subscriber_remove_one_table(tbl)
+ else:
+ self.log.info("Table %s already removed" % tbl)
+
+ def subscriber_resync_tables(self, table_list):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ list = self.fetch_subscriber_tables(dst_curs)
+ for tbl in table_list:
+ tbl = skytools.fq_name(tbl)
+ tbl_row = None
+ for row in list:
+ if row['table_name'] == tbl:
+ tbl_row = row
+ break
+ if not tbl_row:
+ self.log.warning("Table %s not found" % tbl)
+ elif tbl_row['merge_state'] != 'ok':
+ self.log.warning("Table %s is not in stable state" % tbl)
+ else:
+ self.log.info("Resyncing %s" % tbl)
+ q = "select londiste.subscriber_set_table_state(%s, %s, NULL, NULL)"
+ dst_curs.execute(q, [self.pgq_queue_name, tbl])
+ dst_db.commit()
+
+ def subscriber_add_one_table(self, tbl):
+ q = "select londiste.subscriber_add_table(%s, %s)"
+
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ dst_curs.execute(q, [self.pgq_queue_name, tbl])
+ if self.options.expect_sync:
+ q = "select londiste.subscriber_set_table_state(%s, %s, null, 'ok')"
+ dst_curs.execute(q, [self.pgq_queue_name, tbl])
+ dst_db.commit()
+
+ def subscriber_remove_one_table(self, tbl):
+ q = "select londiste.subscriber_remove_table(%s, %s)"
+
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ dst_curs.execute(q, [self.pgq_queue_name, tbl])
+ dst_db.commit()
+
+ def get_subscriber_seq_list(self):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ q = "SELECT * from londiste.subscriber_get_seq_list(%s)"
+ dst_curs.execute(q, [self.pgq_queue_name])
+ list = dst_curs.fetchall()
+ dst_db.commit()
+ res = []
+ for row in list:
+ res.append(row[0])
+ return res
+
+ def subscriber_list_seqs(self):
+ list = self.get_subscriber_seq_list()
+ for seq in list:
+ print seq
+
+ def subscriber_add_seq(self, seq_list):
+ src_db = self.get_database('provider_db')
+ src_curs = src_db.cursor()
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+
+ prov_list = self.get_provider_seqs(src_curs)
+ src_db.commit()
+
+ full_list = self.get_all_seqs(dst_curs)
+ cur_list = self.get_subscriber_seq_list()
+
+ for seq in seq_list:
+ seq = skytools.fq_name(seq)
+ if seq not in prov_list:
+ self.log.error('Seq %s does not exist on provider side' % seq)
+ continue
+ if seq not in full_list:
+ self.log.error('Seq %s does not exist on subscriber side' % seq)
+ continue
+ if seq in cur_list:
+ self.log.info('Seq %s already subscribed' % seq)
+ continue
+
+ self.log.info('Adding sequence: %s' % seq)
+ q = "select londiste.subscriber_add_seq(%s, %s)"
+ dst_curs.execute(q, [self.pgq_queue_name, seq])
+
+ dst_db.commit()
+
+ def subscriber_remove_seq(self, seq_list):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ cur_list = self.get_subscriber_seq_list()
+
+ for seq in seq_list:
+ seq = skytools.fq_name(seq)
+ if seq not in cur_list:
+ self.log.warning('Seq %s not subscribed')
+ else:
+ self.log.info('Removing sequence: %s' % seq)
+ q = "select londiste.subscriber_remove_seq(%s, %s)"
+ dst_curs.execute(q, [self.pgq_queue_name, seq])
+ dst_db.commit()
+
--- /dev/null
+
+"""Catch moment when tables are in sync on master and slave.
+"""
+
+import sys, time, skytools
+
+class Syncer(skytools.DBScript):
+ """Walks tables in primary key order and checks if data matches."""
+
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'londiste', args)
+ self.set_single_loop(1)
+
+ self.pgq_queue_name = self.cf.get("pgq_queue_name")
+ self.pgq_consumer_id = self.cf.get('pgq_consumer_id', self.job_name)
+
+ if self.pidfile:
+ self.pidfile += ".repair"
+
+ def init_optparse(self, p=None):
+ p = skytools.DBScript.init_optparse(self, p)
+ p.add_option("--force", action="store_true", help="ignore lag")
+ return p
+
+ def check_consumer(self, src_db):
+ src_curs = src_db.cursor()
+
+ # before locking anything check if consumer is working ok
+ q = "select extract(epoch from ticker_lag) from pgq.get_queue_list()"\
+ " where queue_name = %s"
+ src_curs.execute(q, [self.pgq_queue_name])
+ ticker_lag = src_curs.fetchone()[0]
+ q = "select extract(epoch from lag)"\
+ " from pgq.get_consumer_list()"\
+ " where queue_name = %s"\
+ " and consumer_name = %s"
+ src_curs.execute(q, [self.pgq_queue_name, self.pgq_consumer_id])
+ res = src_curs.fetchall()
+ src_db.commit()
+
+ if len(res) == 0:
+ self.log.error('No such consumer')
+ sys.exit(1)
+ consumer_lag = res[0][0]
+
+ if consumer_lag > ticker_lag + 10 and not self.options.force:
+ self.log.error('Consumer lagging too much, cannot proceed')
+ sys.exit(1)
+
+ def get_subscriber_table_state(self):
+ dst_db = self.get_database('subscriber_db')
+ dst_curs = dst_db.cursor()
+ q = "select * from londiste.subscriber_get_table_list(%s)"
+ dst_curs.execute(q, [self.pgq_queue_name])
+ res = dst_curs.dictfetchall()
+ dst_db.commit()
+ return res
+
+ def work(self):
+ src_loc = self.cf.get('provider_db')
+ lock_db = self.get_database('provider_db', cache='lock_db')
+ src_db = self.get_database('provider_db')
+ dst_db = self.get_database('subscriber_db')
+
+ self.check_consumer(src_db)
+
+ state_list = self.get_subscriber_table_state()
+ state_map = {}
+ full_list = []
+ for ts in state_list:
+ name = ts['table_name']
+ full_list.append(name)
+ state_map[name] = ts
+
+ if len(self.args) > 2:
+ tlist = self.args[2:]
+ else:
+ tlist = full_list
+
+ for tbl in tlist:
+ if not tbl in state_map:
+ self.log.warning('Table not subscribed: %s' % tbl)
+ continue
+ st = state_map[tbl]
+ if st['merge_state'] != 'ok':
+ self.log.info('Table %s not synced yet, no point' % tbl)
+ continue
+ self.check_table(tbl, lock_db, src_db, dst_db)
+ lock_db.commit()
+ src_db.commit()
+ dst_db.commit()
+
+ def check_table(self, tbl, lock_db, src_db, dst_db):
+ """Get transaction to same state, then process."""
+
+
+ lock_curs = lock_db.cursor()
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+
+ if not skytools.exists_table(src_curs, tbl):
+ self.log.warning("Table %s does not exist on provider side" % tbl)
+ return
+ if not skytools.exists_table(dst_curs, tbl):
+ self.log.warning("Table %s does not exist on subscriber side" % tbl)
+ return
+
+ # lock table in separate connection
+ self.log.info('Locking %s' % tbl)
+ lock_db.commit()
+ lock_curs.execute("LOCK TABLE %s IN SHARE MODE" % tbl)
+ lock_time = time.time()
+
+ # now wait until consumer has updated target table until locking
+ self.log.info('Syncing %s' % tbl)
+
+ # consumer must get futher than this tick
+ src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name])
+ tick_id = src_curs.fetchone()[0]
+ src_db.commit()
+ # avoid depending on ticker by inserting second tick also
+ time.sleep(0.1)
+ src_curs.execute("select pgq.ticker(%s)", [self.pgq_queue_name])
+ src_db.commit()
+ src_curs.execute("select to_char(now(), 'YYYY-MM-DD HH24:MI:SS.MS')")
+ tpos = src_curs.fetchone()[0]
+ src_db.commit()
+ # now wait
+ while 1:
+ time.sleep(0.2)
+
+ q = """select now() - lag > %s, now(), lag
+ from pgq.get_consumer_list()
+ where consumer_name = %s
+ and queue_name = %s"""
+ src_curs.execute(q, [tpos, self.pgq_consumer_id, self.pgq_queue_name])
+ res = src_curs.fetchall()
+ src_db.commit()
+
+ if len(res) == 0:
+ raise Exception('No such consumer')
+
+ row = res[0]
+ self.log.debug("tpos=%s now=%s lag=%s ok=%s" % (tpos, row[1], row[2], row[0]))
+ if row[0]:
+ break
+
+ # loop max 10 secs
+ if time.time() > lock_time + 10 and not self.options.force:
+ self.log.error('Consumer lagging too much, exiting')
+ lock_db.rollback()
+ sys.exit(1)
+
+ # take snapshot on provider side
+ src_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
+ src_curs.execute("SELECT 1")
+
+ # take snapshot on subscriber side
+ dst_db.commit()
+ dst_curs.execute("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE")
+ dst_curs.execute("SELECT 1")
+
+ # release lock
+ lock_db.commit()
+
+ # do work
+ self.process_sync(tbl, src_db, dst_db)
+
+ # done
+ src_db.commit()
+ dst_db.commit()
+
+ def process_sync(self, tbl, src_db, dst_db):
+ """It gets 2 connections in state where tbl should be in same state.
+ """
+ raise Exception('process_sync not implemented')
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Do a full table copy.
+
+For internal usage.
+"""
+
+import sys, os, skytools
+
+from skytools.dbstruct import *
+from playback import *
+
+__all__ = ['CopyTable']
+
+class CopyTable(Replicator):
+ def __init__(self, args, copy_thread = 1):
+ Replicator.__init__(self, args)
+
+ if copy_thread:
+ self.pidfile += ".copy"
+ self.consumer_id += "_copy"
+ self.copy_thread = 1
+
+ def init_optparse(self, parser=None):
+ p = Replicator.init_optparse(self, parser)
+ p.add_option("--skip-truncate", action="store_true", dest="skip_truncate",
+ help = "avoid truncate", default=False)
+ return p
+
+ def do_copy(self, tbl_stat):
+ src_db = self.get_database('provider_db')
+ dst_db = self.get_database('subscriber_db')
+
+ # it should not matter to pgq
+ src_db.commit()
+ dst_db.commit()
+
+ # change to SERIALIZABLE isolation level
+ src_db.set_isolation_level(2)
+ src_db.commit()
+
+ # initial sync copy
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+
+ self.log.info("Starting full copy of %s" % tbl_stat.name)
+
+ # find dst struct
+ src_struct = TableStruct(src_curs, tbl_stat.name)
+ dst_struct = TableStruct(dst_curs, tbl_stat.name)
+
+ # check if columns match
+ dlist = dst_struct.get_column_list()
+ for c in src_struct.get_column_list():
+ if c not in dlist:
+ raise Exception('Column %s does not exist on dest side' % c)
+
+ # drop unnecessary stuff
+ objs = T_CONSTRAINT | T_INDEX | T_TRIGGER | T_RULE
+ dst_struct.drop(dst_curs, objs, log = self.log)
+
+ # do truncate & copy
+ self.real_copy(src_curs, dst_curs, tbl_stat.name)
+
+ # get snapshot
+ src_curs.execute("select get_current_snapshot()")
+ snapshot = src_curs.fetchone()[0]
+ src_db.commit()
+
+ # restore READ COMMITTED behaviour
+ src_db.set_isolation_level(1)
+ src_db.commit()
+
+ # create previously dropped objects
+ dst_struct.create(dst_curs, objs, log = self.log)
+
+ # set state
+ tbl_stat.change_snapshot(snapshot)
+ if self.copy_thread:
+ tbl_stat.change_state(TABLE_CATCHING_UP)
+ else:
+ tbl_stat.change_state(TABLE_OK)
+ self.save_table_state(dst_curs)
+ dst_db.commit()
+
+ def real_copy(self, srccurs, dstcurs, tablename):
+ "Main copy logic."
+
+ # drop data
+ if self.options.skip_truncate:
+ self.log.info("%s: skipping truncate" % tablename)
+ else:
+ self.log.info("%s: truncating" % tablename)
+ dstcurs.execute("truncate " + tablename)
+
+ # do copy
+ self.log.info("%s: start copy" % tablename)
+ col_list = skytools.get_table_columns(srccurs, tablename)
+ stats = skytools.full_copy(tablename, srccurs, dstcurs, col_list)
+ if stats:
+ self.log.info("%s: copy finished: %d bytes, %d rows" % (
+ tablename, stats[0], stats[1]))
+
+if __name__ == '__main__':
+ script = CopyTable(sys.argv[1:])
+ script.start()
+
--- /dev/null
+"""PgQ framework for Python."""
+
+from pgq.event import *
+from pgq.consumer import *
+from pgq.producer import *
+
--- /dev/null
+
+"""PgQ consumer framework for Python.
+
+API problems(?):
+ - process_event() and process_batch() should have db as argument.
+ - should ev.tag*() update db immidiately?
+
+"""
+
+import sys, time, skytools
+
+from pgq.event import *
+
+__all__ = ['Consumer', 'RemoteConsumer', 'SerialConsumer']
+
+class Consumer(skytools.DBScript):
+ """Consumer base class.
+ """
+
+ def __init__(self, service_name, db_name, args):
+ """Initialize new consumer.
+
+ @param service_name: service_name for DBScript
+ @param db_name: name of database for get_database()
+ @param args: cmdline args for DBScript
+ """
+
+ skytools.DBScript.__init__(self, service_name, args)
+
+ self.db_name = db_name
+ self.reg_list = []
+ self.consumer_id = self.cf.get("pgq_consumer_id", self.job_name)
+ self.pgq_queue_name = self.cf.get("pgq_queue_name")
+
+ def attach(self):
+ """Attach consumer to interesting queues."""
+ res = self.register_consumer(self.pgq_queue_name)
+ return res
+
+ def detach(self):
+ """Detach consumer from all queues."""
+ tmp = self.reg_list[:]
+ for q in tmp:
+ self.unregister_consumer(q)
+
+ def process_event(self, db, event):
+ """Process one event.
+
+ Should be overrided by user code.
+
+ Event should be tagged as done, retry or failed.
+ If not, it will be tagged as for retry.
+ """
+ raise Exception("needs to be implemented")
+
+ def process_batch(self, db, batch_id, event_list):
+ """Process all events in batch.
+
+ By default calls process_event for each.
+ Can be overrided by user code.
+
+ Events should be tagged as done, retry or failed.
+ If not, they will be tagged as for retry.
+ """
+ for ev in event_list:
+ self.process_event(db, ev)
+
+ def work(self):
+ """Do the work loop, once (internal)."""
+
+ if len(self.reg_list) == 0:
+ self.log.debug("Attaching")
+ self.attach()
+
+ db = self.get_database(self.db_name)
+ curs = db.cursor()
+
+ data_avail = 0
+ for queue in self.reg_list:
+ self.stat_start()
+
+ # acquire batch
+ batch_id = self._load_next_batch(curs, queue)
+ db.commit()
+ if batch_id == None:
+ continue
+ data_avail = 1
+
+ # load events
+ list = self._load_batch_events(curs, batch_id, queue)
+ db.commit()
+
+ # process events
+ self._launch_process_batch(db, batch_id, list)
+
+ # done
+ self._finish_batch(curs, batch_id, list)
+ db.commit()
+ self.stat_end(len(list))
+
+ # if false, script sleeps
+ return data_avail
+
+ def register_consumer(self, queue_name):
+ db = self.get_database(self.db_name)
+ cx = db.cursor()
+ cx.execute("select pgq.register_consumer(%s, %s)",
+ [queue_name, self.consumer_id])
+ res = cx.fetchone()[0]
+ db.commit()
+
+ self.reg_list.append(queue_name)
+
+ return res
+
+ def unregister_consumer(self, queue_name):
+ db = self.get_database(self.db_name)
+ cx = db.cursor()
+ cx.execute("select pgq.unregister_consumer(%s, %s)",
+ [queue_name, self.consumer_id])
+ db.commit()
+
+ self.reg_list.remove(queue_name)
+
+ def _launch_process_batch(self, db, batch_id, list):
+ self.process_batch(db, batch_id, list)
+
+ def _load_batch_events(self, curs, batch_id, queue_name):
+ """Fetch all events for this batch."""
+
+ # load events
+ sql = "select * from pgq.get_batch_events(%d)" % batch_id
+ curs.execute(sql)
+ rows = curs.dictfetchall()
+
+ # map them to python objects
+ list = []
+ for r in rows:
+ ev = Event(queue_name, r)
+ list.append(ev)
+
+ return list
+
+ def _load_next_batch(self, curs, queue_name):
+ """Allocate next batch. (internal)"""
+
+ q = "select pgq.next_batch(%s, %s)"
+ curs.execute(q, [queue_name, self.consumer_id])
+ return curs.fetchone()[0]
+
+ def _finish_batch(self, curs, batch_id, list):
+ """Tag events and notify that the batch is done."""
+
+ retry = failed = 0
+ for ev in list:
+ if ev.status == EV_FAILED:
+ self._tag_failed(curs, batch_id, ev)
+ failed += 1
+ elif ev.status == EV_RETRY:
+ self._tag_retry(curs, batch_id, ev)
+ retry += 1
+ curs.execute("select pgq.finish_batch(%s)", [batch_id])
+
+ def _tag_failed(self, curs, batch_id, ev):
+ """Tag event as failed. (internal)"""
+ curs.execute("select pgq.event_failed(%s, %s, %s)",
+ [batch_id, ev.id, ev.fail_reason])
+
+ def _tag_retry(self, cx, batch_id, ev):
+ """Tag event for retry. (internal)"""
+ cx.execute("select pgq.event_retry(%s, %s, %s)",
+ [batch_id, ev.id, ev.retry_time])
+
+ def get_batch_info(self, batch_id):
+ """Get info about batch.
+
+ @return: Return value is a dict of:
+
+ - queue_name: queue name
+ - consumer_name: consumers name
+ - batch_start: batch start time
+ - batch_end: batch end time
+ - tick_id: end tick id
+ - prev_tick_id: start tick id
+ - lag: how far is batch_end from current moment.
+ """
+ db = self.get_database(self.db_name)
+ cx = db.cursor()
+ q = "select queue_name, consumer_name, batch_start, batch_end,"\
+ " prev_tick_id, tick_id, lag"\
+ " from pgq.get_batch_info(%s)"
+ cx.execute(q, [batch_id])
+ row = cx.dictfetchone()
+ db.commit()
+ return row
+
+ def stat_start(self):
+ self.stat_batch_start = time.time()
+
+ def stat_end(self, count):
+ t = time.time()
+ self.stat_add('count', count)
+ self.stat_add('duration', t - self.stat_batch_start)
+
+
+class RemoteConsumer(Consumer):
+ """Helper for doing event processing in another database.
+
+ Requires that whole batch is processed in one TX.
+ """
+
+ def __init__(self, service_name, db_name, remote_db, args):
+ Consumer.__init__(self, service_name, db_name, args)
+ self.remote_db = remote_db
+
+ def process_batch(self, db, batch_id, event_list):
+ """Process all events in batch.
+
+ By default calls process_event for each.
+ """
+ dst_db = self.get_database(self.remote_db)
+ curs = dst_db.cursor()
+
+ if self.is_last_batch(curs, batch_id):
+ for ev in event_list:
+ ev.tag_done()
+ return
+
+ self.process_remote_batch(db, batch_id, event_list, dst_db)
+
+ self.set_last_batch(curs, batch_id)
+ dst_db.commit()
+
+ def is_last_batch(self, dst_curs, batch_id):
+ """Helper function to keep track of last successful batch
+ in external database.
+ """
+ q = "select pgq_ext.is_batch_done(%s, %s)"
+ dst_curs.execute(q, [ self.consumer_id, batch_id ])
+ return dst_curs.fetchone()[0]
+
+ def set_last_batch(self, dst_curs, batch_id):
+ """Helper function to set last successful batch
+ in external database.
+ """
+ q = "select pgq_ext.set_batch_done(%s, %s)"
+ dst_curs.execute(q, [ self.consumer_id, batch_id ])
+
+ def process_remote_batch(self, db, batch_id, event_list, dst_db):
+ raise Exception('process_remote_batch not implemented')
+
+class SerialConsumer(Consumer):
+ """Consumer that applies batches sequentially in second database.
+
+ Requirements:
+ - Whole batch in one TX.
+ - Must not use retry queue.
+
+ Features:
+ - Can detect if several batches are already applied to dest db.
+ - If some ticks are lost. allows to seek back on queue.
+ Whether it succeeds, depends on pgq configuration.
+ """
+
+ def __init__(self, service_name, db_name, remote_db, args):
+ Consumer.__init__(self, service_name, db_name, args)
+ self.remote_db = remote_db
+ self.dst_completed_table = "pgq_ext.completed_tick"
+ self.cur_batch_info = None
+
+ def startup(self):
+ if self.options.rewind:
+ self.rewind()
+ sys.exit(0)
+ if self.options.reset:
+ self.dst_reset()
+ sys.exit(0)
+ return Consumer.startup(self)
+
+ def init_optparse(self, parser = None):
+ p = Consumer.init_optparse(self, parser)
+ p.add_option("--rewind", action = "store_true",
+ help = "change queue position according to destination")
+ p.add_option("--reset", action = "store_true",
+ help = "reset queue pos on destination side")
+ return p
+
+ def process_batch(self, db, batch_id, event_list):
+ """Process all events in batch.
+ """
+
+ dst_db = self.get_database(self.remote_db)
+ curs = dst_db.cursor()
+
+ self.cur_batch_info = self.get_batch_info(batch_id)
+
+ # check if done
+ if self.is_batch_done(curs):
+ for ev in event_list:
+ ev.tag_done()
+ return
+
+ # actual work
+ self.process_remote_batch(db, batch_id, event_list, dst_db)
+
+ # make sure no retry events
+ for ev in event_list:
+ if ev.status == EV_RETRY:
+ raise Exception("SerialConsumer must not use retry queue")
+
+ # finish work
+ self.set_batch_done(curs)
+ dst_db.commit()
+
+ def is_batch_done(self, dst_curs):
+ """Helper function to keep track of last successful batch
+ in external database.
+ """
+
+ prev_tick = self.cur_batch_info['prev_tick_id']
+
+ q = "select last_tick_id from %s where consumer_id = %%s" % (
+ self.dst_completed_table ,)
+ dst_curs.execute(q, [self.consumer_id])
+ res = dst_curs.fetchone()
+
+ if not res or not res[0]:
+ # seems this consumer has not run yet against dst_db
+ return False
+ dst_tick = res[0]
+
+ if prev_tick == dst_tick:
+ # on track
+ return False
+
+ if prev_tick < dst_tick:
+ self.log.warning('Got tick %d, dst has %d - skipping' % (prev_tick, dst_tick))
+ return True
+ else:
+ self.log.error('Got tick %d, dst has %d - ticks lost' % (prev_tick, dst_tick))
+ raise Exception('Lost ticks')
+
+ def set_batch_done(self, dst_curs):
+ """Helper function to set last successful batch
+ in external database.
+ """
+ tick_id = self.cur_batch_info['tick_id']
+ q = "delete from %s where consumer_id = %%s; "\
+ "insert into %s (consumer_id, last_tick_id) values (%%s, %%s)" % (
+ self.dst_completed_table,
+ self.dst_completed_table)
+ dst_curs.execute(q, [ self.consumer_id,
+ self.consumer_id, tick_id ])
+
+ def attach(self):
+ new = Consumer.attach(self)
+ if new:
+ self.clean_completed_tick()
+
+ def detach(self):
+ """If detaching, also clean completed tick table on dest."""
+
+ Consumer.detach(self)
+ self.clean_completed_tick()
+
+ def clean_completed_tick(self):
+ self.log.info("removing completed tick from dst")
+ dst_db = self.get_database(self.remote_db)
+ dst_curs = dst_db.cursor()
+
+ q = "delete from %s where consumer_id = %%s" % (
+ self.dst_completed_table,)
+ dst_curs.execute(q, [self.consumer_id])
+ dst_db.commit()
+
+ def process_remote_batch(self, db, batch_id, event_list, dst_db):
+ raise Exception('process_remote_batch not implemented')
+
+ def rewind(self):
+ self.log.info("Rewinding queue")
+ src_db = self.get_database(self.db_name)
+ dst_db = self.get_database(self.remote_db)
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+
+ q = "select last_tick_id from %s where consumer_id = %%s" % (
+ self.dst_completed_table,)
+ dst_curs.execute(q, [self.consumer_id])
+ row = dst_curs.fetchone()
+ if row:
+ dst_tick = row[0]
+ q = "select pgq.register_consumer(%s, %s, %s)"
+ src_curs.execute(q, [self.pgq_queue_name, self.consumer_id, dst_tick])
+ else:
+ self.log.warning('No tick found on dst side')
+
+ dst_db.commit()
+ src_db.commit()
+
+ def dst_reset(self):
+ self.log.info("Resetting queue tracking on dst side")
+ dst_db = self.get_database(self.remote_db)
+ dst_curs = dst_db.cursor()
+
+ q = "delete from %s where consumer_id = %%s" % (
+ self.dst_completed_table,)
+ dst_curs.execute(q, [self.consumer_id])
+ dst_db.commit()
+
+
--- /dev/null
+
+"""PgQ event container.
+"""
+
+__all__ = ('EV_RETRY', 'EV_DONE', 'EV_FAILED', 'Event')
+
+# Event status codes
+EV_RETRY = 0
+EV_DONE = 1
+EV_FAILED = 2
+
+_fldmap = {
+ 'ev_id': 'ev_id',
+ 'ev_txid': 'ev_txid',
+ 'ev_time': 'ev_time',
+ 'ev_type': 'ev_type',
+ 'ev_data': 'ev_data',
+ 'ev_extra1': 'ev_extra1',
+ 'ev_extra2': 'ev_extra2',
+ 'ev_extra3': 'ev_extra3',
+ 'ev_extra4': 'ev_extra4',
+
+ 'id': 'ev_id',
+ 'txid': 'ev_txid',
+ 'time': 'ev_time',
+ 'type': 'ev_type',
+ 'data': 'ev_data',
+ 'extra1': 'ev_extra1',
+ 'extra2': 'ev_extra2',
+ 'extra3': 'ev_extra3',
+ 'extra4': 'ev_extra4',
+}
+
+class Event(object):
+ """Event data for consumers.
+
+ Consumer is supposed to tag them after processing.
+ If not, events will stay in retry queue.
+ """
+ def __init__(self, queue_name, row):
+ self._event_row = row
+ self.status = EV_RETRY
+ self.retry_time = 60
+ self.fail_reason = "Buggy consumer"
+ self.queue_name = queue_name
+
+ def __getattr__(self, key):
+ return self._event_row[_fldmap[key]]
+
+ def tag_done(self):
+ self.status = EV_DONE
+
+ def tag_retry(self, retry_time = 60):
+ self.status = EV_RETRY
+ self.retry_time = retry_time
+
+ def tag_failed(self, reason):
+ self.status = EV_FAILED
+ self.fail_reason = reason
+
--- /dev/null
+"""PgQ maintenance functions."""
+
+import skytools, time
+
+def get_pgq_api_version(curs):
+ q = "select count(1) from pg_proc p, pg_namespace n"\
+ " where n.oid = p.pronamespace and n.nspname='pgq'"\
+ " and p.proname='version';"
+ curs.execute(q)
+ if not curs.fetchone()[0]:
+ return '1.0.0'
+
+ curs.execute("select pgq.version()")
+ return curs.fetchone()[0]
+
+def version_ge(curs, want_ver):
+ """Check is db version of pgq is greater than want_ver."""
+ db_ver = get_pgq_api_version(curs)
+ want_tuple = map(int, want_ver.split('.'))
+ db_tuple = map(int, db_ver.split('.'))
+ if db_tuple[0] != want_tuple[0]:
+ raise Exception('Wrong major version')
+ if db_tuple[1] >= want_tuple[1]:
+ return 1
+ return 0
+
+class MaintenanceJob(skytools.DBScript):
+ """Periodic maintenance."""
+ def __init__(self, ticker, args):
+ skytools.DBScript.__init__(self, 'pgqadm', args)
+ self.ticker = ticker
+ self.last_time = 0 # start immidiately
+ self.last_ticks = 0
+ self.clean_ticks = 1
+ self.maint_delay = 5*60
+
+ def startup(self):
+ # disable regular DBScript startup()
+ pass
+
+ def reload(self):
+ skytools.DBScript.reload(self)
+
+ # force loop_delay
+ self.loop_delay = 5
+
+ self.maint_delay = 60 * self.cf.getfloat('maint_delay_min', 5)
+ self.maint_delay = self.cf.getfloat('maint_delay', self.maint_delay)
+
+ def work(self):
+ t = time.time()
+ if self.last_time + self.maint_delay > t:
+ return
+
+ self.do_maintenance()
+
+ self.last_time = t
+ duration = time.time() - t
+ self.stat_add('maint_duration', duration)
+
+ def do_maintenance(self):
+ """Helper function for running maintenance."""
+
+ db = self.get_database('db', autocommit=1)
+ cx = db.cursor()
+
+ if skytools.exists_function(cx, "pgq.maint_rotate_tables_step1", 1):
+ # rotate each queue in own TX
+ q = "select queue_name from pgq.get_queue_info()"
+ cx.execute(q)
+ for row in cx.fetchall():
+ cx.execute("select pgq.maint_rotate_tables_step1(%s)", [row[0]])
+ res = cx.fetchone()[0]
+ if res:
+ self.log.info('Rotating %s' % row[0])
+ else:
+ cx.execute("select pgq.maint_rotate_tables_step1();")
+
+ # finish rotation
+ cx.execute("select pgq.maint_rotate_tables_step2();")
+
+ # move retry events to main queue in small blocks
+ rcount = 0
+ while 1:
+ cx.execute('select pgq.maint_retry_events();')
+ res = cx.fetchone()[0]
+ rcount += res
+ if res == 0:
+ break
+ if rcount:
+ self.log.info('Got %d events for retry' % rcount)
+
+ # vacuum tables that are needed
+ cx.execute('set maintenance_work_mem = 32768')
+ cx.execute('select * from pgq.maint_tables_to_vacuum()')
+ for row in cx.fetchall():
+ cx.execute('vacuum %s;' % row[0])
+
+
--- /dev/null
+
+"""PgQ producer helpers for Python.
+"""
+
+import skytools
+
+_fldmap = {
+ 'id': 'ev_id',
+ 'time': 'ev_time',
+ 'type': 'ev_type',
+ 'data': 'ev_data',
+ 'extra1': 'ev_extra1',
+ 'extra2': 'ev_extra2',
+ 'extra3': 'ev_extra3',
+ 'extra4': 'ev_extra4',
+
+ 'ev_id': 'ev_id',
+ 'ev_time': 'ev_time',
+ 'ev_type': 'ev_type',
+ 'ev_data': 'ev_data',
+ 'ev_extra1': 'ev_extra1',
+ 'ev_extra2': 'ev_extra2',
+ 'ev_extra3': 'ev_extra3',
+ 'ev_extra4': 'ev_extra4',
+}
+
+def bulk_insert_events(curs, rows, fields, queue_name):
+ q = "select pgq.current_event_table(%s)"
+ curs.execute(q, [queue_name])
+ tbl = curs.fetchone()[0]
+ db_fields = map(_fldmap.get, fields)
+ skytools.magic_insert(curs, tbl, rows, db_fields)
+
+def insert_event(curs, queue, ev_type, ev_data,
+ extra1=None, extra2=None,
+ extra3=None, extra4=None):
+ q = "select pgq.insert_event(%s, %s, %s, %s, %s, %s, %s)"
+ curs.execute(q, [queue, ev_type, ev_data,
+ extra1, extra2, extra3, extra4])
+ return curs.fetchone()[0]
+
--- /dev/null
+
+"""Status display.
+"""
+
+import sys, os, skytools
+
+def ival(data, as = None):
+ "Format interval for output"
+ if not as:
+ as = data.split('.')[-1]
+ numfmt = 'FM9999999'
+ expr = "coalesce(to_char(extract(epoch from %s), '%s') || 's', 'NULL') as %s"
+ return expr % (data, numfmt, as)
+
+class PGQStatus(skytools.DBScript):
+ def __init__(self, args, check = 0):
+ skytools.DBScript.__init__(self, 'pgqadm', args)
+
+ self.show_status()
+
+ sys.exit(0)
+
+ def show_status(self):
+ db = self.get_database("db", autocommit=1)
+ cx = db.cursor()
+
+ cx.execute("show server_version")
+ pgver = cx.fetchone()[0]
+ cx.execute("select pgq.version()")
+ qver = cx.fetchone()[0]
+ print "Postgres version: %s PgQ version: %s" % (pgver, qver)
+
+ q = """select f.queue_name, f.num_tables, %s, %s, %s,
+ q.queue_ticker_max_lag, q.queue_ticker_max_amount,
+ q.queue_ticker_idle_interval
+ from pgq.get_queue_info() f, pgq.queue q
+ where q.queue_name = f.queue_name""" % (
+ ival('f.rotation_delay'),
+ ival('f.ticker_lag'),
+ )
+ cx.execute(q)
+ event_rows = cx.dictfetchall()
+
+ q = """select queue_name, consumer_name, %s, %s, %s
+ from pgq.get_consumer_info()""" % (
+ ival('lag'),
+ ival('last_seen'),
+ )
+ cx.execute(q)
+ consumer_rows = cx.dictfetchall()
+
+ print "\n%-32s %s %9s %13s %6s" % ('Event queue',
+ 'Rotation', 'Ticker', 'TLag')
+ print '-' * 78
+ for ev_row in event_rows:
+ tck = "%s/%ss/%ss" % (ev_row['queue_ticker_max_amount'],
+ ev_row['queue_ticker_max_lag'],
+ ev_row['queue_ticker_idle_interval'])
+ rot = "%s/%s" % (ev_row['queue_ntables'], ev_row['queue_rotation_period'])
+ print "%-39s%7s %9s %13s %6s" % (
+ ev_row['queue_name'],
+ rot,
+ tck,
+ ev_row['ticker_lag'],
+ )
+ print '-' * 78
+ print "\n%-42s %9s %9s" % (
+ 'Consumer', 'Lag', 'LastSeen')
+ print '-' * 78
+ for ev_row in event_rows:
+ cons = self.pick_consumers(ev_row, consumer_rows)
+ self.show_queue(ev_row, cons)
+ print '-' * 78
+ db.commit()
+
+ def show_consumer(self, cons):
+ print " %-48s %9s %9s" % (
+ cons['consumer_name'],
+ cons['lag'], cons['last_seen'])
+ def show_queue(self, ev_row, consumer_rows):
+ print "%(queue_name)s:" % ev_row
+ for cons in consumer_rows:
+ self.show_consumer(cons)
+
+
+ def pick_consumers(self, ev_row, consumer_rows):
+ res = []
+ for con in consumer_rows:
+ if con['queue_name'] != ev_row['queue_name']:
+ continue
+ res.append(con)
+ return res
+
--- /dev/null
+"""PgQ ticker.
+
+It will also launch maintenance job.
+"""
+
+import sys, os, time, threading
+import skytools
+
+from maint import MaintenanceJob
+
+__all__ = ['SmartTicker']
+
+def is_txid_sane(curs):
+ curs.execute("select get_current_txid()")
+ txid = curs.fetchone()[0]
+
+ # on 8.2 theres no such table
+ if not skytools.exists_table(curs, 'txid.epoch'):
+ return 1
+
+ curs.execute("select epoch, last_value from txid.epoch")
+ epoch, last_val = curs.fetchone()
+ stored_val = (epoch << 32) | last_val
+
+ if stored_val <= txid:
+ return 1
+ else:
+ return 0
+
+class QueueStatus(object):
+ def __init__(self, name):
+ self.queue_name = name
+ self.seq_name = None
+ self.idle_period = 60
+ self.max_lag = 3
+ self.max_count = 200
+ self.last_tick_time = 0
+ self.last_count = 0
+ self.quiet_count = 0
+
+ def set_data(self, row):
+ self.seq_name = row['queue_event_seq']
+ self.idle_period = row['queue_ticker_idle_period']
+ self.max_lag = row['queue_ticker_max_lag']
+ self.max_count = row['queue_ticker_max_count']
+
+ def need_tick(self, cur_count, cur_time):
+ # check if tick is needed
+ need_tick = 0
+ lag = cur_time - self.last_tick_time
+
+ if cur_count == self.last_count:
+ # totally idle database
+
+ # don't go immidiately to big delays, as seq grows before commit
+ if self.quiet_count < 5:
+ if lag >= self.max_lag:
+ need_tick = 1
+ self.quiet_count += 1
+ else:
+ if lag >= self.idle_period:
+ need_tick = 1
+ else:
+ self.quiet_count = 0
+ # somewhat loaded machine
+ if cur_count - self.last_count >= self.max_count:
+ need_tick = 1
+ elif lag >= self.max_lag:
+ need_tick = 1
+ if need_tick:
+ self.last_tick_time = cur_time
+ self.last_count = cur_count
+ return need_tick
+
+class SmartTicker(skytools.DBScript):
+ last_tick_event = 0
+ last_tick_time = 0
+ quiet_count = 0
+ tick_count = 0
+ maint_thread = None
+
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'pgqadm', args)
+
+ self.ticker_log_time = 0
+ self.ticker_log_delay = 5*60
+ self.queue_map = {}
+ self.refresh_time = 0
+
+ def reload(self):
+ skytools.DBScript.reload(self)
+ self.ticker_log_delay = self.cf.getfloat("ticker_log_delay", 5*60)
+
+ def startup(self):
+ if self.maint_thread:
+ return
+
+ db = self.get_database("db", autocommit = 1)
+ cx = db.cursor()
+ ok = is_txid_sane(cx)
+ if not ok:
+ self.log.error('txid in bad state')
+ sys.exit(1)
+
+ self.maint_thread = MaintenanceJob(self, [self.cf.filename])
+ t = threading.Thread(name = 'maint_thread',
+ target = self.maint_thread.run)
+ t.setDaemon(1)
+ t.start()
+
+ def refresh_queues(self, cx):
+ q = "select queue_name, queue_event_seq, queue_ticker_idle_period,"\
+ " queue_ticker_max_lag, queue_ticker_max_count"\
+ " from pgq.queue"\
+ " where not queue_external_ticker"
+ cx.execute(q)
+ new_map = {}
+ data_list = []
+ from_list = []
+ for row in cx.dictfetchall():
+ queue_name = row['queue_name']
+ try:
+ que = self.queue_map[queue_name]
+ except KeyError, x:
+ que = QueueStatus(queue_name)
+ que.set_data(row)
+ new_map[queue_name] = que
+
+ p1 = "'%s', %s.last_value" % (queue_name, que.seq_name)
+ data_list.append(p1)
+ from_list.append(que.seq_name)
+
+ self.queue_map = new_map
+ self.seq_query = "select %s from %s" % (
+ ", ".join(data_list),
+ ", ".join(from_list))
+
+ if len(from_list) == 0:
+ self.seq_query = None
+
+ self.refresh_time = time.time()
+
+ def work(self):
+ db = self.get_database("db", autocommit = 1)
+ cx = db.cursor()
+
+ cur_time = time.time()
+
+ if cur_time >= self.refresh_time + 30:
+ self.refresh_queues(cx)
+
+ if not self.seq_query:
+ return
+
+ # now check seqs
+ cx.execute(self.seq_query)
+ res = cx.fetchone()
+ pos = 0
+ while pos < len(res):
+ id = res[pos]
+ val = res[pos + 1]
+ pos += 2
+ que = self.queue_map[id]
+ if que.need_tick(val, cur_time):
+ cx.execute("select pgq.ticker(%s)", [que.queue_name])
+ self.tick_count += 1
+
+ if cur_time > self.ticker_log_time + self.ticker_log_delay:
+ self.ticker_log_time = cur_time
+ self.stat_add('ticks', self.tick_count)
+ self.tick_count = 0
+
--- /dev/null
+#! /usr/bin/env python
+
+"""PgQ ticker and maintenance.
+"""
+
+import sys
+import skytools
+
+from pgq.ticker import SmartTicker
+from pgq.status import PGQStatus
+#from pgq.admin import PGQAdmin
+
+"""TODO:
+pgqadm ini check
+"""
+
+command_usage = """
+%prog [options] INI CMD [subcmd args]
+
+commands:
+ ticker start ticking & maintenance process
+
+ status show overview of queue health
+ check show problematic consumers
+
+ install install code into db
+ create QNAME create queue
+ drop QNAME drop queue
+ register QNAME CONS install code into db
+ unregister QNAME CONS install code into db
+ config QNAME [VAR=VAL] show or change queue config
+"""
+
+config_allowed_list = [
+ 'queue_ticker_max_lag', 'queue_ticker_max_amount',
+ 'queue_ticker_idle_interval', 'queue_rotation_period']
+
+class PGQAdmin(skytools.DBScript):
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'pgqadm', args)
+ self.set_single_loop(1)
+
+ if len(self.args) < 2:
+ print "need command"
+ sys.exit(1)
+
+ int_cmds = {
+ 'create': self.create_queue,
+ 'drop': self.drop_queue,
+ 'register': self.register,
+ 'unregister': self.unregister,
+ 'install': self.installer,
+ 'config': self.change_config,
+ }
+
+ cmd = self.args[1]
+ if cmd == "ticker":
+ script = SmartTicker(args)
+ elif cmd == "status":
+ script = PGQStatus(args)
+ elif cmd == "check":
+ script = PGQStatus(args, check = 1)
+ elif cmd in int_cmds:
+ script = None
+ self.work = int_cmds[cmd]
+ else:
+ print "unknown command"
+ sys.exit(1)
+
+ if self.pidfile:
+ self.pidfile += ".admin"
+ self.run_script = script
+
+ def start(self):
+ if self.run_script:
+ self.run_script.start()
+ else:
+ skytools.DBScript.start(self)
+
+ def init_optparse(self, parser=None):
+ p = skytools.DBScript.init_optparse(self, parser)
+ p.set_usage(command_usage.strip())
+ return p
+
+ def installer(self):
+ objs = [
+ skytools.DBLanguage("plpgsql"),
+ skytools.DBLanguage("plpythonu"),
+ skytools.DBFunction("get_current_txid", 0, sql_file="txid.sql"),
+ skytools.DBSchema("pgq", sql_file="pgq.sql"),
+ ]
+
+ db = self.get_database('db')
+ curs = db.cursor()
+ skytools.db_install(curs, objs, self.log)
+ db.commit()
+
+ def create_queue(self):
+ qname = self.args[2]
+ self.log.info('Creating queue: %s' % qname)
+ self.exec_sql("select pgq.create_queue(%s)", [qname])
+
+ def drop_queue(self):
+ qname = self.args[2]
+ self.log.info('Dropping queue: %s' % qname)
+ self.exec_sql("select pgq.drop_queue(%s)", [qname])
+
+ def register(self):
+ qname = self.args[2]
+ cons = self.args[3]
+ self.log.info('Registering consumer %s on queue %s' % (cons, qname))
+ self.exec_sql("select pgq.register_consumer(%s, %s)", [qname, cons])
+
+ def unregister(self):
+ qname = self.args[2]
+ cons = self.args[3]
+ self.log.info('Unregistering consumer %s from queue %s' % (cons, qname))
+ self.exec_sql("select pgq.unregister_consumer(%s, %s)", [qname, cons])
+
+ def change_config(self):
+ qname = self.args[2]
+ if len(self.args) == 3:
+ self.show_config(qname)
+ return
+ alist = []
+ for el in self.args[3:]:
+ k, v = el.split('=')
+ if k not in config_allowed_list:
+ raise Exception('unknown config var: '+k)
+ expr = "%s=%s" % (k, skytools.quote_literal(v))
+ alist.append(expr)
+ self.log.info('Change queue %s config to: %s' % (qname, ", ".join(alist)))
+ sql = "update pgq.queue set %s where queue_name = %s" % (
+ ", ".join(alist), skytools.quote_literal(qname))
+ self.exec_sql(sql, [])
+
+ def exec_sql(self, q, args):
+ self.log.debug(q)
+ db = self.get_database('db')
+ curs = db.cursor()
+ curs.execute(q, args)
+ db.commit()
+
+ def show_config(self, qname):
+ klist = ",".join(config_allowed_list)
+ q = "select * from pgq.queue where queue_name = %s"
+ db = self.get_database('db')
+ curs = db.cursor()
+ curs.execute(q, [qname])
+ res = curs.dictfetchone()
+ db.commit()
+
+ print qname
+ for k in config_allowed_list:
+ print " %s=%s" % (k, res[k])
+
+if __name__ == '__main__':
+ script = PGQAdmin(sys.argv[1:])
+ script.start()
+
+
+
--- /dev/null
+
+"""Tools for Python database scripts."""
+
+from config import *
+from dbstruct import *
+from gzlog import *
+from quoting import *
+from scripting import *
+from sqltools import *
+
--- /dev/null
+
+"""Nicer config class."""
+
+import sys, os, ConfigParser, socket
+
+__all__ = ['Config']
+
+class Config(object):
+ """Bit improved ConfigParser.
+
+ Additional features:
+ - Remembers section.
+ - Acceps defaults in get() functions.
+ - List value support.
+ """
+ def __init__(self, main_section, filename, sane_config = 1):
+ """Initialize Config and read from file.
+
+ @param sane_config: chooses between ConfigParser/SafeConfigParser.
+ """
+ defs = {
+ 'job_name': main_section,
+ 'service_name': main_section,
+ 'host_name': socket.gethostname(),
+ }
+ if not os.path.isfile(filename):
+ raise Exception('Config file not found: '+filename)
+
+ self.filename = filename
+ self.sane_config = sane_config
+ if sane_config:
+ self.cf = ConfigParser.SafeConfigParser(defs)
+ else:
+ self.cf = ConfigParser.ConfigParser(defs)
+ self.cf.read(filename)
+ self.main_section = main_section
+ if not self.cf.has_section(main_section):
+ raise Exception("Wrong config file, no section '%s'"%main_section)
+
+ def reload(self):
+ """Re-reads config file."""
+ self.cf.read(self.filename)
+
+ def get(self, key, default=None):
+ """Reads string value, if not set then default."""
+ try:
+ return self.cf.get(self.main_section, key)
+ except ConfigParser.NoOptionError, det:
+ if default == None:
+ raise Exception("Config value not set: " + key)
+ return default
+
+ def getint(self, key, default=None):
+ """Reads int value, if not set then default."""
+ try:
+ return self.cf.getint(self.main_section, key)
+ except ConfigParser.NoOptionError, det:
+ if default == None:
+ raise Exception("Config value not set: " + key)
+ return default
+
+ def getboolean(self, key, default=None):
+ """Reads boolean value, if not set then default."""
+ try:
+ return self.cf.getboolean(self.main_section, key)
+ except ConfigParser.NoOptionError, det:
+ if default == None:
+ raise Exception("Config value not set: " + key)
+ return default
+
+ def getfloat(self, key, default=None):
+ """Reads float value, if not set then default."""
+ try:
+ return self.cf.getfloat(self.main_section, key)
+ except ConfigParser.NoOptionError, det:
+ if default == None:
+ raise Exception("Config value not set: " + key)
+ return default
+
+ def getlist(self, key, default=None):
+ """Reads comma-separated list from key."""
+ try:
+ s = self.cf.get(self.main_section, key).strip()
+ res = []
+ if not s:
+ return res
+ for v in s.split(","):
+ res.append(v.strip())
+ return res
+ except ConfigParser.NoOptionError, det:
+ if default == None:
+ raise Exception("Config value not set: " + key)
+ return default
+
+ def getfile(self, key, default=None):
+ """Reads filename from config.
+
+ In addition to reading string value, expands ~ to user directory.
+ """
+ fn = self.get(key, default)
+ if fn == "" or fn == "-":
+ return fn
+ # simulate that the cwd is script location
+ #path = os.path.dirname(sys.argv[0])
+ # seems bad idea, cwd should be cwd
+
+ fn = os.path.expanduser(fn)
+
+ return fn
+
+ def get_wildcard(self, key, values=[], default=None):
+ """Reads a wildcard property from conf and returns its string value, if not set then default."""
+
+ orig_key = key
+ keys = [key]
+
+ for wild in values:
+ key = key.replace('*', wild, 1)
+ keys.append(key)
+ keys.reverse()
+
+ for key in keys:
+ try:
+ return self.cf.get(self.main_section, key)
+ except ConfigParser.NoOptionError, det:
+ pass
+
+ if default == None:
+ raise Exception("Config value not set: " + orig_key)
+ return default
+
+ def sections(self):
+ """Returns list of sections in config file, excluding DEFAULT."""
+ return self.cf.sections()
+
+ def clone(self, main_section):
+ """Return new Config() instance with new main section on same config file."""
+ return Config(main_section, self.filename, self.sane_config)
+
--- /dev/null
+"""Find table structure and allow CREATE/DROP elements from it.
+"""
+
+import sys, re
+
+from sqltools import fq_name_parts, get_table_oid
+
+__all__ = ['TableStruct',
+ 'T_TABLE', 'T_CONSTRAINT', 'T_INDEX', 'T_TRIGGER',
+ 'T_RULE', 'T_GRANT', 'T_OWNER', 'T_PKEY', 'T_ALL']
+
+T_TABLE = 1 << 0
+T_CONSTRAINT = 1 << 1
+T_INDEX = 1 << 2
+T_TRIGGER = 1 << 3
+T_RULE = 1 << 4
+T_GRANT = 1 << 5
+T_OWNER = 1 << 6
+T_PKEY = 1 << 20 # special, one of constraints
+T_ALL = ( T_TABLE | T_CONSTRAINT | T_INDEX
+ | T_TRIGGER | T_RULE | T_GRANT | T_OWNER )
+
+#
+# Utility functions
+#
+
+def find_new_name(curs, name):
+ """Create new object name for case the old exists.
+
+ Needed when creating a new table besides old one.
+ """
+ # cut off previous numbers
+ m = re.search('_[0-9]+$', name)
+ if m:
+ name = name[:m.start()]
+
+ # now loop
+ for i in range(1, 1000):
+ tname = "%s_%d" % (name, i)
+ q = "select count(1) from pg_class where relname = %s"
+ curs.execute(q, [tname])
+ if curs.fetchone()[0] == 0:
+ return tname
+
+ # failed
+ raise Exception('find_new_name failed')
+
+def rx_replace(rx, sql, new_part):
+ """Find a regex match and replace that part with new_part."""
+ m = re.search(rx, sql, re.I)
+ if not m:
+ raise Exception('rx_replace failed')
+ p1 = sql[:m.start()]
+ p2 = sql[m.end():]
+ return p1 + new_part + p2
+
+#
+# Schema objects
+#
+
+class TElem(object):
+ """Keeps info about one metadata object."""
+ SQL = ""
+ type = 0
+ def get_create_sql(self, curs):
+ """Return SQL statement for creating or None if not supported."""
+ return None
+ def get_drop_sql(self, curs):
+ """Return SQL statement for dropping or None of not supported."""
+ return None
+
+class TConstraint(TElem):
+ """Info about constraint."""
+ type = T_CONSTRAINT
+ SQL = """
+ SELECT conname as name, pg_get_constraintdef(oid) as def, contype
+ FROM pg_constraint WHERE conrelid = %(oid)s
+ """
+ def __init__(self, table_name, row):
+ self.table_name = table_name
+ self.name = row['name']
+ self.defn = row['def']
+ self.contype = row['contype']
+
+ # tag pkeys
+ if self.contype == 'p':
+ self.type += T_PKEY
+
+ def get_create_sql(self, curs, new_table_name=None):
+ fmt = "ALTER TABLE ONLY %s ADD CONSTRAINT %s %s;"
+ if new_table_name:
+ name = self.name
+ if self.contype in ('p', 'u'):
+ name = find_new_name(curs, self.name)
+ sql = fmt % (new_table_name, name, self.defn)
+ else:
+ sql = fmt % (self.table_name, self.name, self.defn)
+ return sql
+
+ def get_drop_sql(self, curs):
+ fmt = "ALTER TABLE ONLY %s DROP CONSTRAINT %s;"
+ sql = fmt % (self.table_name, self.name)
+ return sql
+
+class TIndex(TElem):
+ """Info about index."""
+ type = T_INDEX
+ SQL = """
+ SELECT n.nspname || '.' || c.relname as name,
+ pg_get_indexdef(i.indexrelid) as defn
+ FROM pg_index i, pg_class c, pg_namespace n
+ WHERE c.oid = i.indexrelid AND i.indrelid = %(oid)s
+ AND n.oid = c.relnamespace
+ AND NOT EXISTS
+ (select objid from pg_depend
+ where classid = %(pg_class_oid)s
+ and objid = c.oid
+ and deptype = 'i')
+ """
+ def __init__(self, table_name, row):
+ self.name = row['name']
+ self.defn = row['defn'] + ';'
+
+ def get_create_sql(self, curs, new_table_name = None):
+ if not new_table_name:
+ return self.defn
+ name = find_new_name(curs, self.name)
+ pnew = "INDEX %s ON %s " % (name, new_table_name)
+ rx = r"\bINDEX[ ][a-z0-9._]+[ ]ON[ ][a-z0-9._]+[ ]"
+ sql = rx_replace(rx, self.defn, pnew)
+ return sql
+ def get_drop_sql(self, curs):
+ return 'DROP INDEX %s;' % self.name
+
+class TRule(TElem):
+ """Info about rule."""
+ type = T_RULE
+ SQL = """
+ SELECT rulename as name, pg_get_ruledef(oid) as def
+ FROM pg_rewrite
+ WHERE ev_class = %(oid)s AND rulename <> '_RETURN'::name
+ """
+ def __init__(self, table_name, row, new_name = None):
+ self.table_name = table_name
+ self.name = row['name']
+ self.defn = row['def']
+
+ def get_create_sql(self, curs, new_table_name = None):
+ if not new_table_name:
+ return self.defn
+ rx = r"\bTO[ ][a-z0-9._]+[ ]DO[ ]"
+ pnew = "TO %s DO " % new_table_name
+ return rx_replace(rx, self.defn, pnew)
+
+ def get_drop_sql(self, curs):
+ return 'DROP RULE %s ON %s' % (self.name, self.table_name)
+
+class TTrigger(TElem):
+ """Info about trigger."""
+ type = T_TRIGGER
+ SQL = """
+ SELECT tgname as name, pg_get_triggerdef(oid) as def
+ FROM pg_trigger
+ WHERE tgrelid = %(oid)s AND NOT tgisconstraint
+ """
+ def __init__(self, table_name, row):
+ self.table_name = table_name
+ self.name = row['name']
+ self.defn = row['def'] + ';'
+
+ def get_create_sql(self, curs, new_table_name = None):
+ if not new_table_name:
+ return self.defn
+
+ rx = r"\bON[ ][a-z0-9._]+[ ]"
+ pnew = "ON %s " % new_table_name
+ return rx_replace(rx, self.defn, pnew)
+
+ def get_drop_sql(self, curs):
+ return 'DROP TRIGGER %s ON %s' % (self.name, self.table_name)
+
+class TOwner(TElem):
+ """Info about table owner."""
+ type = T_OWNER
+ SQL = """
+ SELECT pg_get_userbyid(relowner) as owner FROM pg_class
+ WHERE oid = %(oid)s
+ """
+ def __init__(self, table_name, row, new_name = None):
+ self.table_name = table_name
+ self.name = 'Owner'
+ self.owner = row['owner']
+
+ def get_create_sql(self, curs, new_name = None):
+ if not new_name:
+ new_name = self.table_name
+ return 'ALTER TABLE %s OWNER TO %s;' % (new_name, self.owner)
+
+class TGrant(TElem):
+ """Info about permissions."""
+ type = T_GRANT
+ SQL = "SELECT relacl FROM pg_class where oid = %(oid)s"
+ acl_map = {
+ 'r': 'SELECT', 'w': 'UPDATE', 'a': 'INSERT', 'd': 'DELETE',
+ 'R': 'RULE', 'x': 'REFERENCES', 't': 'TRIGGER', 'X': 'EXECUTE',
+ 'U': 'USAGE', 'C': 'CREATE', 'T': 'TEMPORARY'
+ }
+ def acl_to_grants(self, acl):
+ if acl == "arwdRxt": # ALL for tables
+ return "ALL"
+ return ", ".join([ self.acl_map[c] for c in acl ])
+
+ def parse_relacl(self, relacl):
+ if relacl is None:
+ return []
+ if len(relacl) > 0 and relacl[0] == '{' and relacl[-1] == '}':
+ relacl = relacl[1:-1]
+ list = []
+ for f in relacl.split(','):
+ user, tmp = f.strip('"').split('=')
+ acl, who = tmp.split('/')
+ list.append((user, acl, who))
+ return list
+
+ def __init__(self, table_name, row, new_name = None):
+ self.name = table_name
+ self.acl_list = self.parse_relacl(row['relacl'])
+
+ def get_create_sql(self, curs, new_name = None):
+ if not new_name:
+ new_name = self.name
+
+ list = []
+ for user, acl, who in self.acl_list:
+ astr = self.acl_to_grants(acl)
+ sql = "GRANT %s ON %s TO %s;" % (astr, new_name, user)
+ list.append(sql)
+ return "\n".join(list)
+
+ def get_drop_sql(self, curs):
+ list = []
+ for user, acl, who in self.acl_list:
+ sql = "REVOKE ALL FROM %s ON %s;" % (user, self.name)
+ list.append(sql)
+ return "\n".join(list)
+
+class TColumn(TElem):
+ """Info about table column."""
+ SQL = """
+ select a.attname as name,
+ a.attname || ' '
+ || format_type(a.atttypid, a.atttypmod)
+ || case when a.attnotnull then ' not null' else '' end
+ || case when a.atthasdef then ' ' || d.adsrc else '' end
+ as def
+ from pg_attribute a left join pg_attrdef d
+ on (d.adrelid = a.attrelid and d.adnum = a.attnum)
+ where a.attrelid = %(oid)s
+ and not a.attisdropped
+ and a.attnum > 0
+ order by a.attnum;
+ """
+ def __init__(self, table_name, row):
+ self.name = row['name']
+ self.column_def = row['def']
+
+class TTable(TElem):
+ """Info about table only (columns)."""
+ type = T_TABLE
+ def __init__(self, table_name, col_list):
+ self.name = table_name
+ self.col_list = col_list
+
+ def get_create_sql(self, curs, new_name = None):
+ if not new_name:
+ new_name = self.name
+ sql = "create table %s (" % new_name
+ sep = "\n\t"
+ for c in self.col_list:
+ sql += sep + c.column_def
+ sep = ",\n\t"
+ sql += "\n);"
+ return sql
+
+ def get_drop_sql(self, curs):
+ return "DROP TABLE %s;" % self.name
+
+#
+# Main table object, loads all the others
+#
+
+class TableStruct(object):
+ """Collects and manages all info about table.
+
+ Allow to issue CREATE/DROP statements about any
+ group of elements.
+ """
+ def __init__(self, curs, table_name):
+ """Initializes class by loading info about table_name from database."""
+
+ self.table_name = table_name
+
+ # fill args
+ schema, name = fq_name_parts(table_name)
+ args = {
+ 'schema': schema,
+ 'table': name,
+ 'oid': get_table_oid(curs, table_name),
+ 'pg_class_oid': get_table_oid(curs, 'pg_catalog.pg_class'),
+ }
+
+ # load table struct
+ self.col_list = self._load_elem(curs, args, TColumn)
+ self.object_list = [ TTable(table_name, self.col_list) ]
+
+ # load additional objects
+ to_load = [TConstraint, TIndex, TTrigger, TRule, TGrant, TOwner]
+ for eclass in to_load:
+ self.object_list += self._load_elem(curs, args, eclass)
+
+ def _load_elem(self, curs, args, eclass):
+ list = []
+ curs.execute(eclass.SQL % args)
+ for row in curs.dictfetchall():
+ list.append(eclass(self.table_name, row))
+ return list
+
+ def create(self, curs, objs, new_table_name = None, log = None):
+ """Issues CREATE statements for requested set of objects.
+
+ If new_table_name is giver, creates table under that name
+ and also tries to rename all indexes/constraints that conflict
+ with existing table.
+ """
+
+ for o in self.object_list:
+ if o.type & objs:
+ sql = o.get_create_sql(curs, new_table_name)
+ if not sql:
+ continue
+ if log:
+ log.info('Creating %s' % o.name)
+ log.debug(sql)
+ curs.execute(sql)
+
+ def drop(self, curs, objs, log = None):
+ """Issues DROP statements for requested set of objects."""
+ for o in self.object_list:
+ if o.type & objs:
+ sql = o.get_drop_sql(curs)
+ if not sql:
+ continue
+ if log:
+ log.info('Dropping %s' % o.name)
+ log.debug(sql)
+ curs.execute(sql)
+
+ def get_column_list(self):
+ """Returns list of column names the table has."""
+
+ res = []
+ for c in self.col_list:
+ res.append(c.name)
+ return res
+
+def test():
+ import psycopg
+ db = psycopg.connect("dbname=fooz")
+ curs = db.cursor()
+
+ s = TableStruct(curs, "public.data1")
+
+ s.drop(curs, T_ALL)
+ s.create(curs, T_ALL)
+ s.create(curs, T_ALL, "data1_new")
+ s.create(curs, T_PKEY)
+
+if __name__ == '__main__':
+ test()
+
--- /dev/null
+
+"""Atomic append of gzipped data.
+
+The point is - if several gzip streams are concated, they
+are read back as one whose stream.
+"""
+
+import gzip
+from cStringIO import StringIO
+
+__all__ = ['gzip_append']
+
+#
+# gzip storage
+#
+def gzip_append(filename, data, level = 6):
+ """Append a block of data to file with safety checks."""
+
+ # compress data
+ buf = StringIO()
+ g = gzip.GzipFile(fileobj = buf, compresslevel = level, mode = "w")
+ g.write(data)
+ g.close()
+ zdata = buf.getvalue()
+
+ # append, safely
+ f = open(filename, "a+", 0)
+ f.seek(0, 2)
+ pos = f.tell()
+ try:
+ f.write(zdata)
+ f.close()
+ except Exception, ex:
+ # rollback on error
+ f.seek(pos, 0)
+ f.truncate()
+ f.close()
+ raise ex
+
--- /dev/null
+# quoting.py
+
+"""Various helpers for string quoting/unquoting."""
+
+import psycopg, urllib, re
+
+#
+# SQL quoting
+#
+
+def quote_literal(s):
+ """Quote a literal value for SQL.
+
+ Surronds it with single-quotes.
+ """
+
+ if s == None:
+ return "null"
+ s = psycopg.QuotedString(str(s))
+ return str(s)
+
+def quote_copy(s):
+ """Quoting for copy command."""
+
+ if s == None:
+ return "\\N"
+ s = str(s)
+ s = s.replace("\\", "\\\\")
+ s = s.replace("\t", "\\t")
+ s = s.replace("\n", "\\n")
+ s = s.replace("\r", "\\r")
+ return s
+
+def quote_bytea_raw(s):
+ """Quoting for bytea parser."""
+
+ if s == None:
+ return None
+ return s.replace("\\", "\\\\").replace("\0", "\\000")
+
+def quote_bytea_literal(s):
+ """Quote bytea for regular SQL."""
+
+ return quote_literal(quote_bytea_raw(s))
+
+def quote_bytea_copy(s):
+ """Quote bytea for COPY."""
+
+ return quote_copy(quote_bytea_raw(s))
+
+def quote_statement(sql, dict):
+ """Quote whose statement.
+
+ Data values are taken from dict.
+ """
+ xdict = {}
+ for k, v in dict.items():
+ xdict[k] = quote_literal(v)
+ return sql % xdict
+
+#
+# quoting for JSON strings
+#
+
+_jsre = re.compile(r'[\x00-\x1F\\/"]')
+_jsmap = { "\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r",
+ "\t": "\\t", "\\": "\\\\", '"': '\\"',
+ "/": "\\/", # to avoid html attacks
+}
+
+def _json_quote_char(m):
+ c = m.group(0)
+ try:
+ return _jsmap[c]
+ except KeyError:
+ return r"\u%04x" % ord(c)
+
+def quote_json(s):
+ """JSON style quoting."""
+ if s is None:
+ return "null"
+ return '"%s"' % _jsre.sub(_json_quote_char, s)
+
+#
+# Database specific urlencode and urldecode.
+#
+
+def db_urlencode(dict):
+ """Database specific urlencode.
+
+ Encode None as key without '='. That means that in "foo&bar=",
+ foo is NULL and bar is empty string.
+ """
+
+ elem_list = []
+ for k, v in dict.items():
+ if v is None:
+ elem = urllib.quote_plus(str(k))
+ else:
+ elem = urllib.quote_plus(str(k)) + '=' + urllib.quote_plus(str(v))
+ elem_list.append(elem)
+ return '&'.join(elem_list)
+
+def db_urldecode(qs):
+ """Database specific urldecode.
+
+ Decode key without '=' as None.
+ This also does not support one key several times.
+ """
+
+ res = {}
+ for elem in qs.split('&'):
+ if not elem:
+ continue
+ pair = elem.split('=', 1)
+ name = urllib.unquote_plus(pair[0])
+ if len(pair) == 1:
+ res[name] = None
+ else:
+ res[name] = urllib.unquote_plus(pair[1])
+ return res
+
+#
+# Remove C-like backslash escapes
+#
+
+_esc_re = r"\\([0-7][0-7][0-7]|.)"
+_esc_rc = re.compile(_esc_re)
+_esc_map = {
+ 't': '\t',
+ 'n': '\n',
+ 'r': '\r',
+ 'a': '\a',
+ 'b': '\b',
+ "'": "'",
+ '"': '"',
+ '\\': '\\',
+}
+
+def _sub_unescape(m):
+ v = m.group(1)
+ if len(v) == 1:
+ return _esc_map[v]
+ else:
+ return chr(int(v, 8))
+
+def unescape(val):
+ """Removes C-style escapes from string."""
+ return _esc_rc.sub(_sub_unescape, val)
+
+def unescape_copy(val):
+ """Removes C-style escapes, also converts "\N" to None."""
+ if val == r"\N":
+ return None
+ return unescape(val)
+
--- /dev/null
+
+"""Useful functions and classes for database scripts."""
+
+import sys, os, signal, psycopg, optparse, traceback, time
+import logging, logging.handlers, logging.config
+
+from skytools.config import *
+import skytools.skylog
+
+__all__ = ['daemonize', 'run_single_process', 'DBScript',
+ 'I_AUTOCOMMIT', 'I_READ_COMMITTED', 'I_SERIALIZABLE']
+
+#
+# daemon mode
+#
+
+def daemonize():
+ """Turn the process into daemon.
+
+ Goes background and disables all i/o.
+ """
+
+ # launch new process, kill parent
+ pid = os.fork()
+ if pid != 0:
+ os._exit(0)
+
+ # start new session
+ os.setsid()
+
+ # stop i/o
+ fd = os.open("/dev/null", os.O_RDWR)
+ os.dup2(fd, 0)
+ os.dup2(fd, 1)
+ os.dup2(fd, 2)
+ if fd > 2:
+ os.close(fd)
+
+#
+# Pidfile locking+cleanup & daemonization combined
+#
+
+def _write_pidfile(pidfile):
+ pid = os.getpid()
+ f = open(pidfile, 'w')
+ f.write(str(pid))
+ f.close()
+
+def run_single_process(runnable, daemon, pidfile):
+ """Run runnable class, possibly daemonized, locked on pidfile."""
+
+ # check if another process is running
+ if pidfile and os.path.isfile(pidfile):
+ print "Pidfile exists, another process running?"
+ sys.exit(1)
+
+ # daemonize if needed and write pidfile
+ if daemon:
+ daemonize()
+ if pidfile:
+ _write_pidfile(pidfile)
+
+ # Catch SIGTERM to cleanup pidfile
+ def sigterm_hook(signum, frame):
+ try:
+ os.remove(pidfile)
+ except: pass
+ sys.exit(0)
+ # attach it to signal
+ if pidfile:
+ signal.signal(signal.SIGTERM, sigterm_hook)
+
+ # run
+ try:
+ runnable.run()
+ finally:
+ # another try of cleaning up
+ if pidfile:
+ try:
+ os.remove(pidfile)
+ except: pass
+
+#
+# logging setup
+#
+
+_log_config_done = 0
+_log_init_done = {}
+
+def _init_log(job_name, cf, log_level):
+ """Logging setup happens here."""
+ global _log_init_done, _log_config_done
+
+ got_skylog = 0
+ use_skylog = cf.getint("use_skylog", 0)
+
+ # load logging config if needed
+ if use_skylog and not _log_config_done:
+ # python logging.config braindamage:
+ # cannot specify external classess without such hack
+ logging.skylog = skytools.skylog
+
+ # load general config
+ list = ['skylog.ini', '~/.skylog.ini', '/etc/skylog.ini']
+ for fn in list:
+ fn = os.path.expanduser(fn)
+ if os.path.isfile(fn):
+ defs = {'job_name': job_name}
+ logging.config.fileConfig(fn, defs)
+ got_skylog = 1
+ break
+ _log_config_done = 1
+ if not got_skylog:
+ sys.stderr.write("skylog.ini not found!\n")
+ sys.exit(1)
+
+ # avoid duplicate logging init for job_name
+ log = logging.getLogger(job_name)
+ if job_name in _log_init_done:
+ return log
+ _log_init_done[job_name] = 1
+
+ # compatibility: specify ini file in script config
+ logfile = cf.getfile("logfile", "")
+ if logfile:
+ fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s')
+ size = cf.getint('log_size', 10*1024*1024)
+ num = cf.getint('log_count', 3)
+ hdlr = logging.handlers.RotatingFileHandler(
+ logfile, 'a', size, num)
+ hdlr.setFormatter(fmt)
+ log.addHandler(hdlr)
+
+ # if skylog.ini is disabled or not available, log at least to stderr
+ if not got_skylog:
+ hdlr = logging.StreamHandler()
+ fmt = logging.Formatter('%(asctime)s %(process)s %(levelname)s %(message)s')
+ hdlr.setFormatter(fmt)
+ log.addHandler(hdlr)
+
+ log.setLevel(log_level)
+
+ return log
+
+#: how old connections need to be closed
+DEF_CONN_AGE = 20*60 # 20 min
+
+#: isolation level not set
+I_DEFAULT = -1
+
+#: isolation level constant for AUTOCOMMIT
+I_AUTOCOMMIT = 0
+#: isolation level constant for READ COMMITTED
+I_READ_COMMITTED = 1
+#: isolation level constant for SERIALIZABLE
+I_SERIALIZABLE = 2
+
+class DBCachedConn(object):
+ """Cache a db connection."""
+ def __init__(self, name, loc, max_age = DEF_CONN_AGE):
+ self.name = name
+ self.loc = loc
+ self.conn = None
+ self.conn_time = 0
+ self.max_age = max_age
+ self.autocommit = -1
+ self.isolation_level = -1
+
+ def get_connection(self, autocommit = 0, isolation_level = -1):
+ # autocommit overrider isolation_level
+ if autocommit:
+ isolation_level = I_AUTOCOMMIT
+
+ # default isolation_level is READ COMMITTED
+ if isolation_level < 0:
+ isolation_level = I_READ_COMMITTED
+
+ # new conn?
+ if not self.conn:
+ self.isolation_level = isolation_level
+ self.conn = psycopg.connect(self.loc)
+
+ self.conn.set_isolation_level(isolation_level)
+ self.conn_time = time.time()
+ else:
+ if self.isolation_level != isolation_level:
+ raise Exception("Conflict in isolation_level")
+
+ # done
+ return self.conn
+
+ def refresh(self):
+ if not self.conn:
+ return
+ #for row in self.conn.notifies():
+ # if row[0].lower() == "reload":
+ # self.reset()
+ # return
+ if not self.max_age:
+ return
+ if time.time() - self.conn_time >= self.max_age:
+ self.reset()
+
+ def reset(self):
+ if not self.conn:
+ return
+
+ # drop reference
+ conn = self.conn
+ self.conn = None
+
+ if self.isolation_level == I_AUTOCOMMIT:
+ return
+
+ # rollback & close
+ try:
+ conn.rollback()
+ except: pass
+ try:
+ conn.close()
+ except: pass
+
+class DBScript(object):
+ """Base class for database scripts.
+
+ Handles logging, daemonizing, config, errors.
+ """
+ service_name = None
+ job_name = None
+ cf = None
+ log = None
+
+ def __init__(self, service_name, args):
+ """Script setup.
+
+ User class should override work() and optionally __init__(), startup(),
+ reload(), reset() and init_optparse().
+
+ NB: in case of daemon, the __init__() and startup()/work() will be
+ run in different processes. So nothing fancy should be done in __init__().
+
+ @param service_name: unique name for script.
+ It will be also default job_name, if not specified in config.
+ @param args: cmdline args (sys.argv[1:]), but can be overrided
+ """
+ self.service_name = service_name
+ self.db_cache = {}
+ self.go_daemon = 0
+ self.do_single_loop = 0
+ self.looping = 1
+ self.need_reload = 1
+ self.stat_dict = {}
+ self.log_level = logging.INFO
+ self.work_state = 1
+
+ # parse command line
+ parser = self.init_optparse()
+ self.options, self.args = parser.parse_args(args)
+
+ # check args
+ if self.options.daemon:
+ self.go_daemon = 1
+ if self.options.quiet:
+ self.log_level = logging.WARNING
+ if self.options.verbose:
+ self.log_level = logging.DEBUG
+ if len(self.args) < 1:
+ print "need config file"
+ sys.exit(1)
+ conf_file = self.args[0]
+
+ # load config
+ self.cf = Config(self.service_name, conf_file)
+ self.job_name = self.cf.get("job_name", self.service_name)
+ self.pidfile = self.cf.getfile("pidfile", '')
+
+ self.reload()
+
+ # init logging
+ self.log = _init_log(self.job_name, self.cf, self.log_level)
+
+ # send signal, if needed
+ if self.options.cmd == "kill":
+ self.send_signal(signal.SIGTERM)
+ elif self.options.cmd == "stop":
+ self.send_signal(signal.SIGINT)
+ elif self.options.cmd == "reload":
+ self.send_signal(signal.SIGHUP)
+
+ def init_optparse(self, parser = None):
+ """Initialize a OptionParser() instance that will be used to
+ parse command line arguments.
+
+ Note that it can be overrided both directions - either DBScript
+ will initialize a instance and passes to user code or user can
+ initialize and then pass to DBScript.init_optparse().
+
+ @param parser: optional OptionParser() instance,
+ where DBScript should attachs its own arguments.
+ @return: initialized OptionParser() instance.
+ """
+ if parser:
+ p = parser
+ else:
+ p = optparse.OptionParser()
+ p.set_usage("%prog [options] INI")
+ # generic options
+ p.add_option("-q", "--quiet", action="store_true",
+ help = "make program silent")
+ p.add_option("-v", "--verbose", action="store_true",
+ help = "make program verbose")
+ p.add_option("-d", "--daemon", action="store_true",
+ help = "go background")
+
+ # control options
+ g = optparse.OptionGroup(p, 'control running process')
+ g.add_option("-r", "--reload",
+ action="store_const", const="reload", dest="cmd",
+ help = "reload config (send SIGHUP)")
+ g.add_option("-s", "--stop",
+ action="store_const", const="stop", dest="cmd",
+ help = "stop program safely (send SIGINT)")
+ g.add_option("-k", "--kill",
+ action="store_const", const="kill", dest="cmd",
+ help = "kill program immidiately (send SIGTERM)")
+ p.add_option_group(g)
+
+ return p
+
+ def send_signal(self, sig):
+ if not self.pidfile:
+ self.log.warning("No pidfile in config, nothing todo")
+ sys.exit(0)
+ if not os.path.isfile(self.pidfile):
+ self.log.warning("No pidfile, process not running")
+ sys.exit(0)
+ pid = int(open(self.pidfile, "r").read())
+ os.kill(pid, sig)
+ sys.exit(0)
+
+ def set_single_loop(self, do_single_loop):
+ """Changes whether the script will loop or not."""
+ self.do_single_loop = do_single_loop
+
+ def start(self):
+ """This will launch main processing thread."""
+ if self.go_daemon:
+ if not self.pidfile:
+ self.log.error("Daemon needs pidfile")
+ sys.exit(1)
+ run_single_process(self, self.go_daemon, self.pidfile)
+
+ def stop(self):
+ """Safely stops processing loop."""
+ self.looping = 0
+
+ def reload(self):
+ "Reload config."
+ self.cf.reload()
+ self.loop_delay = self.cf.getfloat("loop_delay", 1.0)
+
+ def hook_sighup(self, sig, frame):
+ "Internal SIGHUP handler. Minimal code here."
+ self.need_reload = 1
+
+ def hook_sigint(self, sig, frame):
+ "Internal SIGINT handler. Minimal code here."
+ self.stop()
+
+ def stat_add(self, key, value):
+ self.stat_put(key, value)
+
+ def stat_put(self, key, value):
+ """Sets a stat value."""
+ self.stat_dict[key] = value
+
+ def stat_increase(self, key, increase = 1):
+ """Increases a stat value."""
+ if key in self.stat_dict:
+ self.stat_dict[key] += increase
+ else:
+ self.stat_dict[key] = increase
+
+ def send_stats(self):
+ "Send statistics to log."
+
+ res = []
+ for k, v in self.stat_dict.items():
+ res.append("%s: %s" % (k, str(v)))
+
+ if len(res) == 0:
+ return
+
+ logmsg = "{%s}" % ", ".join(res)
+ self.log.info(logmsg)
+ self.stat_dict = {}
+
+ def get_database(self, dbname, autocommit = 0, isolation_level = -1,
+ cache = None, max_age = DEF_CONN_AGE):
+ """Load cached database connection.
+
+ User must not store it permanently somewhere,
+ as all connections will be invalidated on reset.
+ """
+
+ if not cache:
+ cache = dbname
+ if cache in self.db_cache:
+ dbc = self.db_cache[cache]
+ else:
+ loc = self.cf.get(dbname)
+ dbc = DBCachedConn(cache, loc, max_age)
+ self.db_cache[cache] = dbc
+
+ return dbc.get_connection(autocommit, isolation_level)
+
+ def close_database(self, dbname):
+ """Explicitly close a cached connection.
+
+ Next call to get_database() will reconnect.
+ """
+ if dbname in self.db_cache:
+ dbc = self.db_cache[dbname]
+ dbc.reset()
+
+ def reset(self):
+ "Something bad happened, reset all connections."
+ for dbc in self.db_cache.values():
+ dbc.reset()
+ self.db_cache = {}
+
+ def run(self):
+ "Thread main loop."
+
+ # run startup, safely
+ try:
+ self.startup()
+ except KeyboardInterrupt, det:
+ raise
+ except SystemExit, det:
+ raise
+ except Exception, det:
+ exc, msg, tb = sys.exc_info()
+ self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % (
+ self.job_name, str(exc), str(msg).rstrip(),
+ str(tb), repr(traceback.format_tb(tb))))
+ del tb
+ self.reset()
+ sys.exit(1)
+
+ while self.looping:
+ # reload config, if needed
+ if self.need_reload:
+ self.reload()
+ self.need_reload = 0
+
+ # do some work
+ work = self.run_once()
+
+ # send stats that was added
+ self.send_stats()
+
+ # reconnect if needed
+ for dbc in self.db_cache.values():
+ dbc.refresh()
+
+ # exit if needed
+ if self.do_single_loop:
+ self.log.debug("Only single loop requested, exiting")
+ break
+
+ # remember work state
+ self.work_state = work
+ # should sleep?
+ if not work:
+ try:
+ time.sleep(self.loop_delay)
+ except Exception, d:
+ self.log.debug("sleep failed: "+str(d))
+ sys.exit(0)
+
+ def run_once(self):
+ "Run users work function, safely."
+ try:
+ return self.work()
+ except SystemExit, d:
+ self.send_stats()
+ self.log.info("got SystemExit(%s), exiting" % str(d))
+ self.reset()
+ raise d
+ except KeyboardInterrupt, d:
+ self.send_stats()
+ self.log.info("got KeyboardInterrupt, exiting")
+ self.reset()
+ sys.exit(1)
+ except Exception, d:
+ self.send_stats()
+ exc, msg, tb = sys.exc_info()
+ self.log.fatal("Job %s crashed: %s: '%s' (%s: %s)" % (
+ self.job_name, str(exc), str(msg).rstrip(),
+ str(tb), repr(traceback.format_tb(tb))))
+ del tb
+ self.reset()
+ if self.looping:
+ time.sleep(20)
+ return 1
+
+ def work(self):
+ "Here should user's processing happen."
+ raise Exception("Nothing implemented?")
+
+ def startup(self):
+ """Will be called just before entering main loop.
+
+ In case of daemon, if will be called in same process as work(),
+ unlike __init__().
+ """
+
+ # set signals
+ signal.signal(signal.SIGHUP, self.hook_sighup)
+ signal.signal(signal.SIGINT, self.hook_sigint)
+
+
--- /dev/null
+"""Our log handlers for Python's logging package.
+"""
+
+import sys, os, time, socket, psycopg
+import logging, logging.handlers
+
+from quoting import quote_json
+
+# configurable file logger
+class EasyRotatingFileHandler(logging.handlers.RotatingFileHandler):
+ """Easier setup for RotatingFileHandler."""
+ def __init__(self, filename, maxBytes = 10*1024*1024, backupCount = 3):
+ """Args same as for RotatingFileHandler, but in filename '~' is expanded."""
+ fn = os.path.expanduser(filename)
+ logging.handlers.RotatingFileHandler.__init__(self, fn, maxBytes=maxBytes, backupCount=backupCount)
+
+# send JSON message over UDP
+class UdpLogServerHandler(logging.handlers.DatagramHandler):
+ """Sends log records over UDP to logserver in JSON format."""
+
+ # map logging levels to logserver levels
+ _level_map = {
+ logging.DEBUG : 'DEBUG',
+ logging.INFO : 'INFO',
+ logging.WARNING : 'WARN',
+ logging.ERROR : 'ERROR',
+ logging.CRITICAL: 'FATAL',
+ }
+
+ # JSON message template
+ _log_template = '{\n\t'\
+ '"logger": "skytools.UdpLogServer",\n\t'\
+ '"timestamp": %.0f,\n\t'\
+ '"level": "%s",\n\t'\
+ '"thread": null,\n\t'\
+ '"message": %s,\n\t'\
+ '"properties": {"application":"%s", "hostname":"%s"}\n'\
+ '}'
+
+ # cut longer msgs
+ MAXMSG = 1024
+
+ def makePickle(self, record):
+ """Create message in JSON format."""
+ # get & cut msg
+ msg = self.format(record)
+ if len(msg) > self.MAXMSG:
+ msg = msg[:self.MAXMSG]
+ txt_level = self._level_map.get(record.levelno, "ERROR")
+ pkt = self._log_template % (time.time()*1000, txt_level,
+ quote_json(msg), record.name, socket.gethostname())
+ return pkt
+
+class LogDBHandler(logging.handlers.SocketHandler):
+ """Sends log records into PostgreSQL server.
+
+ Additionally, does some statistics aggregating,
+ to avoid overloading log server.
+
+ It subclasses SocketHandler to get throtthling for
+ failed connections.
+ """
+
+ # map codes to string
+ _level_map = {
+ logging.DEBUG : 'DEBUG',
+ logging.INFO : 'INFO',
+ logging.WARNING : 'WARNING',
+ logging.ERROR : 'ERROR',
+ logging.CRITICAL: 'FATAL',
+ }
+
+ def __init__(self, connect_string):
+ """
+ Initializes the handler with a specific connection string.
+ """
+
+ logging.handlers.SocketHandler.__init__(self, None, None)
+ self.closeOnError = 1
+
+ self.connect_string = connect_string
+
+ self.stat_cache = {}
+ self.stat_flush_period = 60
+ # send first stat line immidiately
+ self.last_stat_flush = 0
+
+ def createSocket(self):
+ try:
+ logging.handlers.SocketHandler.createSocket(self)
+ except:
+ self.sock = self.makeSocket()
+
+ def makeSocket(self):
+ """Create server connection.
+ In this case its not socket but psycopg conection."""
+
+ db = psycopg.connect(self.connect_string)
+ db.autocommit(1)
+ return db
+
+ def emit(self, record):
+ """Process log record."""
+
+ # we do not want log debug messages
+ if record.levelno < logging.INFO:
+ return
+
+ try:
+ self.process_rec(record)
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except:
+ self.handleError(record)
+
+ def process_rec(self, record):
+ """Aggregate stats if needed, and send to logdb."""
+ # render msg
+ msg = self.format(record)
+
+ # dont want to send stats too ofter
+ if record.levelno == logging.INFO and msg and msg[0] == "{":
+ self.aggregate_stats(msg)
+ if time.time() - self.last_stat_flush >= self.stat_flush_period:
+ self.flush_stats(record.name)
+ return
+
+ if record.levelno < logging.INFO:
+ self.flush_stats(record.name)
+
+ # dont send more than one line
+ ln = msg.find('\n')
+ if ln > 0:
+ msg = msg[:ln]
+
+ txt_level = self._level_map.get(record.levelno, "ERROR")
+ self.send_to_logdb(record.name, txt_level, msg)
+
+ def aggregate_stats(self, msg):
+ """Sum stats together, to lessen load on logdb."""
+
+ msg = msg[1:-1]
+ for rec in msg.split(", "):
+ k, v = rec.split(": ")
+ agg = self.stat_cache.get(k, 0)
+ if v.find('.') >= 0:
+ agg += float(v)
+ else:
+ agg += int(v)
+ self.stat_cache[k] = agg
+
+ def flush_stats(self, service):
+ """Send awuired stats to logdb."""
+ res = []
+ for k, v in self.stat_cache.items():
+ res.append("%s: %s" % (k, str(v)))
+ if len(res) > 0:
+ logmsg = "{%s}" % ", ".join(res)
+ self.send_to_logdb(service, "INFO", logmsg)
+ self.stat_cache = {}
+ self.last_stat_flush = time.time()
+
+ def send_to_logdb(self, service, type, msg):
+ """Actual sending is done here."""
+
+ if self.sock is None:
+ self.createSocket()
+
+ if self.sock:
+ logcur = self.sock.cursor()
+ query = "select * from log.add(%s, %s, %s)"
+ logcur.execute(query, [type, service, msg])
+
--- /dev/null
+
+"""Database tools."""
+
+import os
+from cStringIO import StringIO
+from quoting import quote_copy, quote_literal
+
+#
+# Fully qualified table name
+#
+
+def fq_name_parts(tbl):
+ "Return fully qualified name parts."
+
+ tmp = tbl.split('.')
+ if len(tmp) == 1:
+ return ('public', tbl)
+ elif len(tmp) == 2:
+ return tmp
+ else:
+ raise Exception('Syntax error in table name:'+tbl)
+
+def fq_name(tbl):
+ "Return fully qualified name."
+ return '.'.join(fq_name_parts(tbl))
+
+#
+# info about table
+#
+def get_table_oid(curs, table_name):
+ schema, name = fq_name_parts(table_name)
+ q = """select c.oid from pg_namespace n, pg_class c
+ where c.relnamespace = n.oid
+ and n.nspname = %s and c.relname = %s"""
+ curs.execute(q, [schema, name])
+ res = curs.fetchall()
+ if len(res) == 0:
+ raise Exception('Table not found: '+table_name)
+ return res[0][0]
+
+def get_table_pkeys(curs, tbl):
+ oid = get_table_oid(curs, tbl)
+ q = "SELECT k.attname FROM pg_index i, pg_attribute k"\
+ " WHERE i.indrelid = %s AND k.attrelid = i.indexrelid"\
+ " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\
+ " ORDER BY k.attnum"
+ curs.execute(q, [oid])
+ return map(lambda x: x[0], curs.fetchall())
+
+def get_table_columns(curs, tbl):
+ oid = get_table_oid(curs, tbl)
+ q = "SELECT k.attname FROM pg_attribute k"\
+ " WHERE k.attrelid = %s"\
+ " AND k.attnum > 0 AND NOT k.attisdropped"\
+ " ORDER BY k.attnum"
+ curs.execute(q, [oid])
+ return map(lambda x: x[0], curs.fetchall())
+
+#
+# exist checks
+#
+def exists_schema(curs, schema):
+ q = "select count(1) from pg_namespace where nspname = %s"
+ curs.execute(q, [schema])
+ res = curs.fetchone()
+ return res[0]
+
+def exists_table(curs, table_name):
+ schema, name = fq_name_parts(table_name)
+ q = """select count(1) from pg_namespace n, pg_class c
+ where c.relnamespace = n.oid and c.relkind = 'r'
+ and n.nspname = %s and c.relname = %s"""
+ curs.execute(q, [schema, name])
+ res = curs.fetchone()
+ return res[0]
+
+def exists_type(curs, type_name):
+ schema, name = fq_name_parts(type_name)
+ q = """select count(1) from pg_namespace n, pg_type t
+ where t.typnamespace = n.oid
+ and n.nspname = %s and t.typname = %s"""
+ curs.execute(q, [schema, name])
+ res = curs.fetchone()
+ return res[0]
+
+def exists_function(curs, function_name, nargs):
+ # this does not check arg types, so may match several functions
+ schema, name = fq_name_parts(function_name)
+ q = """select count(1) from pg_namespace n, pg_proc p
+ where p.pronamespace = n.oid and p.pronargs = %s
+ and n.nspname = %s and p.proname = %s"""
+ curs.execute(q, [nargs, schema, name])
+ res = curs.fetchone()
+ return res[0]
+
+def exists_language(curs, lang_name):
+ q = """select count(1) from pg_language
+ where lanname = %s"""
+ curs.execute(q, [lang_name])
+ res = curs.fetchone()
+ return res[0]
+
+#
+# Support for PostgreSQL snapshot
+#
+
+class Snapshot(object):
+ "Represents a PostgreSQL snapshot."
+
+ def __init__(self, str):
+ "Create snapshot from string."
+
+ self.sn_str = str
+ tmp = str.split(':')
+ if len(tmp) != 3:
+ raise Exception('Unknown format for snapshot')
+ self.xmin = int(tmp[0])
+ self.xmax = int(tmp[1])
+ self.txid_list = []
+ if tmp[2] != "":
+ for s in tmp[2].split(','):
+ self.txid_list.append(int(s))
+
+ def contains(self, txid):
+ "Is txid visible in snapshot."
+
+ txid = int(txid)
+
+ if txid < self.xmin:
+ return True
+ if txid >= self.xmax:
+ return False
+ if txid in self.txid_list:
+ return False
+ return True
+
+#
+# Copy helpers
+#
+
+def _gen_dict_copy(tbl, row, fields):
+ tmp = []
+ for f in fields:
+ v = row[f]
+ tmp.append(quote_copy(v))
+ return "\t".join(tmp)
+
+def _gen_dict_insert(tbl, row, fields):
+ tmp = []
+ for f in fields:
+ v = row[f]
+ tmp.append(quote_literal(v))
+ fmt = "insert into %s (%s) values (%s);"
+ return fmt % (tbl, ",".join(fields), ",".join(tmp))
+
+def _gen_list_copy(tbl, row, fields):
+ tmp = []
+ for i in range(len(fields)):
+ v = row[i]
+ tmp.append(quote_copy(v))
+ return "\t".join(tmp)
+
+def _gen_list_insert(tbl, row, fields):
+ tmp = []
+ for i in range(len(fields)):
+ v = row[i]
+ tmp.append(quote_literal(v))
+ fmt = "insert into %s (%s) values (%s);"
+ return fmt % (tbl, ",".join(fields), ",".join(tmp))
+
+def magic_insert(curs, tablename, data, fields = None, use_insert = 0):
+ """Copy/insert a list of dict/list data to database.
+
+ If curs == None, then the copy or insert statements are returned
+ as string. For list of dict the field list is optional, as its
+ possible to guess them from dict keys.
+ """
+ if len(data) == 0:
+ return
+
+ # decide how to process
+ if type(data[0]) == type({}):
+ if fields == None:
+ fields = data[0].keys()
+ if use_insert:
+ row_func = _gen_dict_insert
+ else:
+ row_func = _gen_dict_copy
+ else:
+ if fields == None:
+ raise Exception("Non-dict data needs field list")
+ if use_insert:
+ row_func = _gen_list_insert
+ else:
+ row_func = _gen_list_copy
+
+ # init processing
+ buf = StringIO()
+ if curs == None and use_insert == 0:
+ fmt = "COPY %s (%s) FROM STDIN;\n"
+ buf.write(fmt % (tablename, ",".join(fields)))
+
+ # process data
+ for row in data:
+ buf.write(row_func(tablename, row, fields))
+ buf.write("\n")
+
+ # if user needs only string, return it
+ if curs == None:
+ if use_insert == 0:
+ buf.write("\\.\n")
+ return buf.getvalue()
+
+ # do the actual copy/inserts
+ if use_insert:
+ curs.execute(buf.getvalue())
+ else:
+ buf.seek(0)
+ hdr = "%s (%s)" % (tablename, ",".join(fields))
+ curs.copy_from(buf, hdr)
+
+def db_copy_from_dict(curs, tablename, dict_list, fields = None):
+ """Do a COPY FROM STDIN using list of dicts as source."""
+
+ if len(dict_list) == 0:
+ return
+
+ if fields == None:
+ fields = dict_list[0].keys()
+
+ buf = StringIO()
+ for dat in dict_list:
+ row = []
+ for k in fields:
+ row.append(quote_copy(dat[k]))
+ buf.write("\t".join(row))
+ buf.write("\n")
+
+ buf.seek(0)
+ hdr = "%s (%s)" % (tablename, ",".join(fields))
+
+ curs.copy_from(buf, hdr)
+
+def db_copy_from_list(curs, tablename, row_list, fields):
+ """Do a COPY FROM STDIN using list of lists as source."""
+
+ if len(row_list) == 0:
+ return
+
+ if fields == None or len(fields) == 0:
+ raise Exception('Need field list')
+
+ buf = StringIO()
+ for dat in row_list:
+ row = []
+ for i in range(len(fields)):
+ row.append(quote_copy(dat[i]))
+ buf.write("\t".join(row))
+ buf.write("\n")
+
+ buf.seek(0)
+ hdr = "%s (%s)" % (tablename, ",".join(fields))
+
+ curs.copy_from(buf, hdr)
+
+#
+# Full COPY of table from one db to another
+#
+
+class CopyPipe(object):
+ "Splits one big COPY to chunks."
+
+ def __init__(self, dstcurs, tablename, limit = 512*1024, cancel_func=None):
+ self.tablename = tablename
+ self.dstcurs = dstcurs
+ self.buf = StringIO()
+ self.limit = limit
+ self.cancel_func = None
+ self.total_rows = 0
+ self.total_bytes = 0
+
+ def write(self, data):
+ "New data from psycopg"
+
+ self.total_bytes += len(data)
+ self.total_rows += data.count("\n")
+
+ if self.buf.tell() >= self.limit:
+ pos = data.find('\n')
+ if pos >= 0:
+ # split at newline
+ p1 = data[:pos + 1]
+ p2 = data[pos + 1:]
+ self.buf.write(p1)
+ self.flush()
+
+ data = p2
+
+ self.buf.write(data)
+
+ def flush(self):
+ "Send data out."
+
+ if self.cancel_func:
+ self.cancel_func()
+
+ if self.buf.tell() > 0:
+ self.buf.seek(0)
+ self.dstcurs.copy_from(self.buf, self.tablename)
+ self.buf.seek(0)
+ self.buf.truncate()
+
+def full_copy(tablename, src_curs, dst_curs, column_list = []):
+ """COPY table from one db to another."""
+
+ if column_list:
+ hdr = "%s (%s)" % (tablename, ",".join(column_list))
+ else:
+ hdr = tablename
+ buf = CopyPipe(dst_curs, hdr)
+ src_curs.copy_to(buf, hdr)
+ buf.flush()
+
+ return (buf.total_bytes, buf.total_rows)
+
+
+#
+# SQL installer
+#
+
+class DBObject(object):
+ """Base class for installable DB objects."""
+ name = None
+ sql = None
+ sql_file = None
+ def __init__(self, name, sql = None, sql_file = None):
+ self.name = name
+ self.sql = sql
+ self.sql_file = sql_file
+ def get_sql(self):
+ if self.sql:
+ return self.sql
+ if self.sql_file:
+ if self.sql_file[0] == "/":
+ fn = self.sql_file
+ else:
+ contrib_list = [
+ "/opt/pgsql/share/contrib",
+ "/usr/share/postgresql/8.0/contrib",
+ "/usr/share/postgresql/8.0/contrib",
+ "/usr/share/postgresql/8.1/contrib",
+ "/usr/share/postgresql/8.2/contrib",
+ ]
+ for dir in contrib_list:
+ fn = os.path.join(dir, self.sql_file)
+ if os.path.isfile(fn):
+ return open(fn, "r").read()
+ raise Exception('File not found: '+self.sql_file)
+ raise Exception('object not defined')
+ def create(self, curs):
+ curs.execute(self.get_sql())
+
+class DBSchema(DBObject):
+ """Handles db schema."""
+ def exists(self, curs):
+ return exists_schema(curs, self.name)
+
+class DBTable(DBObject):
+ """Handles db table."""
+ def exists(self, curs):
+ return exists_table(curs, self.name)
+
+class DBFunction(DBObject):
+ """Handles db function."""
+ def __init__(self, name, nargs, sql = None, sql_file = None):
+ DBObject.__init__(self, name, sql, sql_file)
+ self.nargs = nargs
+ def exists(self, curs):
+ return exists_function(curs, self.name, self.nargs)
+
+class DBLanguage(DBObject):
+ """Handles db language."""
+ def __init__(self, name):
+ DBObject.__init__(self, name, sql = "create language %s" % name)
+ def exists(self, curs):
+ return exists_language(curs, self.name)
+
+def db_install(curs, list, log = None):
+ """Installs list of objects into db."""
+ for obj in list:
+ if not obj.exists(curs):
+ if log:
+ log.info('Installing %s' % obj.name)
+ obj.create(curs)
+ else:
+ if log:
+ log.info('%s is installed' % obj.name)
+
--- /dev/null
+#! /usr/bin/env python
+
+"""WALShipping manager.
+
+walmgr [-n] COMMAND
+
+Master commands:
+ setup Configure PostgreSQL for WAL archiving
+ backup Copies all master data to slave
+ sync Copies in-progress WALs to slave
+ syncdaemon Daemon mode for regular syncing
+ stop Stop archiving - de-configure PostgreSQL
+
+Slave commands:
+ restore Stop postmaster, move new data dir to right
+ location and start postmaster in playback mode.
+ boot Stop playback, accept queries.
+ pause Just wait, don't play WAL-s
+ continue Start playing WAL-s again
+
+Internal commands:
+ xarchive archive one WAL file (master)
+ xrestore restore one WAL file (slave)
+
+Switches:
+ -n no action, just print commands
+"""
+
+import os, sys, skytools, getopt, re, signal, time, traceback
+
+MASTER = 1
+SLAVE = 0
+
+def usage(err):
+ if err > 0:
+ print >>sys.stderr, __doc__
+ else:
+ print __doc__
+ sys.exit(err)
+
+class WalMgr(skytools.DBScript):
+ def __init__(self, wtype, cf_file, not_really, internal = 0, go_daemon = 0):
+ self.not_really = not_really
+ self.pg_backup = 0
+
+ if wtype == MASTER:
+ service_name = "wal-master"
+ else:
+ service_name = "wal-slave"
+
+ if not os.path.isfile(cf_file):
+ print "Config not found:", cf_file
+ sys.exit(1)
+
+ if go_daemon:
+ s_args = ["-d", cf_file]
+ else:
+ s_args = [cf_file]
+
+ skytools.DBScript.__init__(self, service_name, s_args,
+ force_logfile = internal)
+
+ def pg_start_backup(self, code):
+ q = "select pg_start_backup('FullBackup')"
+ self.log.info("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db")))
+ if self.not_really:
+ self.pg_backup = 1
+ return
+ db = self.get_database("master_db")
+ db.cursor().execute(q)
+ db.commit()
+ self.close_database("master_db")
+ self.pg_backup = 1
+
+ def pg_stop_backup(self):
+ if not self.pg_backup:
+ return
+
+ q = "select pg_stop_backup()"
+ self.log.debug("Execute SQL: %s; [%s]" % (q, self.cf.get("master_db")))
+ if self.not_really:
+ return
+ db = self.get_database("master_db")
+ db.cursor().execute(q)
+ db.commit()
+ self.close_database("master_db")
+
+ def signal_postmaster(self, data_dir, sgn):
+ pidfile = os.path.join(data_dir, "postmaster.pid")
+ if not os.path.isfile(pidfile):
+ self.log.info("postmaster is not running")
+ return
+ buf = open(pidfile, "r").readline()
+ pid = int(buf.strip())
+ self.log.debug("Signal %d to process %d" % (sgn, pid))
+ if not self.not_really:
+ os.kill(pid, sgn)
+
+ def exec_big_rsync(self, cmdline):
+ cmd = "' '".join(cmdline)
+ self.log.debug("Execute big rsync cmd: '%s'" % (cmd))
+ if self.not_really:
+ return
+ res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline)
+ if res == 24:
+ self.log.info("Some files vanished, but thats OK")
+ elif res != 0:
+ self.log.fatal("exec failed, res=%d" % res)
+ self.pg_stop_backup()
+ sys.exit(1)
+
+ def exec_cmd(self, cmdline):
+ cmd = "' '".join(cmdline)
+ self.log.debug("Execute cmd: '%s'" % (cmd))
+ if self.not_really:
+ return
+ res = os.spawnvp(os.P_WAIT, cmdline[0], cmdline)
+ if res != 0:
+ self.log.fatal("exec failed, res=%d" % res)
+ sys.exit(1)
+
+ def chdir(self, loc):
+ self.log.debug("chdir: '%s'" % (loc))
+ if self.not_really:
+ return
+ try:
+ os.chdir(loc)
+ except os.error:
+ self.log.fatal("CHDir failed")
+ self.pg_stop_backup()
+ sys.exit(1)
+
+ def get_last_complete(self):
+ """Get the name of last xarchived segment."""
+
+ data_dir = self.cf.get("master_data")
+ fn = os.path.join(data_dir, ".walshipping.last")
+ try:
+ last = open(fn, "r").read().strip()
+ return last
+ except:
+ self.log.info("Failed to read %s" % fn)
+ return None
+
+ def set_last_complete(self, last):
+ """Set the name of last xarchived segment."""
+
+ data_dir = self.cf.get("master_data")
+ fn = os.path.join(data_dir, ".walshipping.last")
+ fn_tmp = fn + ".new"
+ try:
+ f = open(fn_tmp, "w")
+ f.write(last)
+ f.close()
+ os.rename(fn_tmp, fn)
+ except:
+ self.log.fatal("Cannot write to %s" % fn)
+
+ def master_setup(self):
+ self.log.info("Configuring WAL archiving")
+
+ script = os.path.abspath(sys.argv[0])
+ cf_file = os.path.abspath(self.cf.filename)
+ cf_val = "%s %s %s" % (script, cf_file, "xarchive %p %f")
+
+ self.master_configure_archiving(cf_val)
+
+ def master_stop(self):
+ self.log.info("Disabling WAL archiving")
+
+ self.master_configure_archiving('')
+
+ def master_configure_archiving(self, cf_val):
+ cf_file = self.cf.get("master_config")
+ data_dir = self.cf.get("master_data")
+ r_active = re.compile("^[ ]*archive_command[ ]*=[ ]*'(.*)'.*$", re.M)
+ r_disabled = re.compile("^.*archive_command.*$", re.M)
+
+ cf_full = "archive_command = '%s'" % cf_val
+
+ if not os.path.isfile(cf_file):
+ self.log.fatal("Config file not found: %s" % cf_file)
+ self.log.info("Using config file: %s", cf_file)
+
+ buf = open(cf_file, "r").read()
+ m = r_active.search(buf)
+ if m:
+ old_val = m.group(1)
+ if old_val == cf_val:
+ self.log.debug("postmaster already configured")
+ else:
+ self.log.debug("found active but different conf")
+ newbuf = "%s%s%s" % (buf[:m.start()], cf_full, buf[m.end():])
+ self.change_config(cf_file, newbuf)
+ else:
+ m = r_disabled.search(buf)
+ if m:
+ self.log.debug("found disabled value")
+ newbuf = "%s\n%s%s" % (buf[:m.end()], cf_full, buf[m.end():])
+ self.change_config(cf_file, newbuf)
+ else:
+ self.log.debug("found no value")
+ newbuf = "%s\n%s\n\n" % (buf, cf_full)
+ self.change_config(cf_file, newbuf)
+
+ self.log.info("Sending SIGHUP to postmaster")
+ self.signal_postmaster(data_dir, signal.SIGHUP)
+ self.log.info("Done")
+
+ def change_config(self, cf_file, buf):
+ cf_old = cf_file + ".old"
+ cf_new = cf_file + ".new"
+
+ if self.not_really:
+ cf_new = "/tmp/postgresql.conf.new"
+ open(cf_new, "w").write(buf)
+ self.log.info("Showing diff")
+ os.system("diff -u %s %s" % (cf_file, cf_new))
+ self.log.info("Done diff")
+ os.remove(cf_new)
+ return
+
+ # polite method does not work, as usually not enough perms for it
+ if 0:
+ open(cf_new, "w").write(buf)
+ bak = open(cf_file, "r").read()
+ open(cf_old, "w").write(bak)
+ os.rename(cf_new, cf_file)
+ else:
+ open(cf_file, "w").write(buf)
+
+ def remote_mkdir(self, remdir):
+ tmp = remdir.split(":", 1)
+ if len(tmp) != 2:
+ raise Exception("cannot find hostname")
+ host, path = tmp
+ cmdline = ["ssh", host, "mkdir", "-p", path]
+ self.exec_cmd(cmdline)
+
+ def master_backup(self):
+ """Copy master data directory to slave."""
+
+ data_dir = self.cf.get("master_data")
+ dst_loc = self.cf.get("full_backup")
+ if dst_loc[-1] != "/":
+ dst_loc += "/"
+
+ self.pg_start_backup("FullBackup")
+
+ master_spc_dir = os.path.join(data_dir, "pg_tblspc")
+ slave_spc_dir = dst_loc + "tmpspc"
+
+ # copy data
+ self.chdir(data_dir)
+ cmdline = ["rsync", "-a", "--delete",
+ "--exclude", ".*",
+ "--exclude", "*.pid",
+ "--exclude", "*.opts",
+ "--exclude", "*.conf",
+ "--exclude", "*.conf.*",
+ "--exclude", "pg_xlog",
+ ".", dst_loc]
+ self.exec_big_rsync(cmdline)
+
+ # copy tblspc first, to test
+ if os.path.isdir(master_spc_dir):
+ self.log.info("Checking tablespaces")
+ list = os.listdir(master_spc_dir)
+ if len(list) > 0:
+ self.remote_mkdir(slave_spc_dir)
+ for tblspc in list:
+ if tblspc[0] == ".":
+ continue
+ tfn = os.path.join(master_spc_dir, tblspc)
+ if not os.path.islink(tfn):
+ self.log.info("Suspicious pg_tblspc entry: "+tblspc)
+ continue
+ spc_path = os.path.realpath(tfn)
+ self.log.info("Got tablespace %s: %s" % (tblspc, spc_path))
+ dstfn = slave_spc_dir + "/" + tblspc
+
+ try:
+ os.chdir(spc_path)
+ except Exception, det:
+ self.log.warning("Broken link:" + str(det))
+ continue
+ cmdline = ["rsync", "-a", "--delete",
+ "--exclude", ".*",
+ ".", dstfn]
+ self.exec_big_rsync(cmdline)
+
+ # copy pg_xlog
+ self.chdir(data_dir)
+ cmdline = ["rsync", "-a",
+ "--exclude", "*.done",
+ "--exclude", "*.backup",
+ "--delete", "pg_xlog", dst_loc]
+ self.exec_big_rsync(cmdline)
+
+ self.pg_stop_backup()
+
+ self.log.info("Full backup successful")
+
+ def master_xarchive(self, srcpath, srcname):
+ """Copy a complete WAL segment to slave."""
+
+ start_time = time.time()
+ self.log.debug("%s: start copy", srcname)
+
+ self.set_last_complete(srcname)
+
+ dst_loc = self.cf.get("completed_wals")
+ if dst_loc[-1] != "/":
+ dst_loc += "/"
+
+ # copy data
+ cmdline = ["rsync", "-t", srcpath, dst_loc]
+ self.exec_cmd(cmdline)
+
+ self.log.debug("%s: done", srcname)
+ end_time = time.time()
+ self.stat_add('count', 1)
+ self.stat_add('duration', end_time - start_time)
+
+ def master_sync(self):
+ """Copy partial WAL segments."""
+
+ data_dir = self.cf.get("master_data")
+ xlog_dir = os.path.join(data_dir, "pg_xlog")
+ dst_loc = self.cf.get("partial_wals")
+ if dst_loc[-1] != "/":
+ dst_loc += "/"
+
+ files = os.listdir(xlog_dir)
+ files.sort()
+
+ last = self.get_last_complete()
+ if last:
+ self.log.info("%s: last complete" % last)
+ else:
+ self.log.info("last complete not found, copying all")
+
+ for fn in files:
+ # check if interesting file
+ if len(fn) < 10:
+ continue
+ if fn[0] < "0" or fn[0] > '9':
+ continue
+ if fn.find(".") > 0:
+ continue
+ # check if to old
+ if last:
+ dot = last.find(".")
+ if dot > 0:
+ xlast = last[:dot]
+ if fn < xlast:
+ continue
+ else:
+ if fn <= last:
+ continue
+
+ # got interesting WAL
+ xlog = os.path.join(xlog_dir, fn)
+ # copy data
+ cmdline = ["rsync", "-t", xlog, dst_loc]
+ self.exec_cmd(cmdline)
+
+ self.log.info("Partial copy done")
+
+ def slave_xrestore(self, srcname, dstpath):
+ loop = 1
+ while loop:
+ try:
+ self.slave_xrestore_unsafe(srcname, dstpath)
+ loop = 0
+ except SystemExit, d:
+ sys.exit(1)
+ except Exception, d:
+ exc, msg, tb = sys.exc_info()
+ self.log.fatal("xrestore %s crashed: %s: '%s' (%s: %s)" % (
+ srcname, str(exc), str(msg).rstrip(),
+ str(tb), repr(traceback.format_tb(tb))))
+ time.sleep(10)
+ self.log.info("Re-exec: %s", repr(sys.argv))
+ os.execv(sys.argv[0], sys.argv)
+
+ def slave_xrestore_unsafe(self, srcname, dstpath):
+ srcdir = self.cf.get("completed_wals")
+ partdir = self.cf.get("partial_wals")
+ keep_old_logs = self.cf.getint("keep_old_logs", 0)
+ pausefile = os.path.join(srcdir, "PAUSE")
+ stopfile = os.path.join(srcdir, "STOP")
+ srcfile = os.path.join(srcdir, srcname)
+ partfile = os.path.join(partdir, srcname)
+
+ # loop until srcfile or stopfile appears
+ while 1:
+ if os.path.isfile(pausefile):
+ self.log.info("pause requested, sleeping")
+ time.sleep(20)
+ continue
+
+ if os.path.isfile(srcfile):
+ self.log.info("%s: Found" % srcname)
+ break
+
+ # ignore .history files
+ unused, ext = os.path.splitext(srcname)
+ if ext == ".history":
+ self.log.info("%s: not found, ignoring" % srcname)
+ sys.exit(1)
+
+ # if stopping, include also partial wals
+ if os.path.isfile(stopfile):
+ if os.path.isfile(partfile):
+ self.log.info("%s: found partial" % srcname)
+ srcfile = partfile
+ break
+ else:
+ self.log.info("%s: not found, stopping" % srcname)
+ sys.exit(1)
+
+ # nothing to do, sleep
+ self.log.debug("%s: not found, sleeping" % srcname)
+ time.sleep(20)
+
+ # got one, copy it
+ cmdline = ["cp", srcfile, dstpath]
+ self.exec_cmd(cmdline)
+
+ self.log.debug("%s: copy done, cleanup" % srcname)
+ self.slave_cleanup(srcname)
+
+ # it would be nice to have apply time too
+ self.stat_add('count', 1)
+
+ def slave_startup(self):
+ data_dir = self.cf.get("slave_data")
+ full_dir = self.cf.get("full_backup")
+ stop_cmd = self.cf.get("slave_stop_cmd", "")
+ start_cmd = self.cf.get("slave_start_cmd")
+ pidfile = os.path.join(data_dir, "postmaster.pid")
+
+ # stop postmaster if ordered
+ if stop_cmd and os.path.isfile(pidfile):
+ self.log.info("Stopping postmaster: " + stop_cmd)
+ if not self.not_really:
+ os.system(stop_cmd)
+ time.sleep(3)
+
+ # is it dead?
+ if os.path.isfile(pidfile):
+ self.log.fatal("Postmaster still running. Cannot continue.")
+ sys.exit(1)
+
+ # find name for data backup
+ i = 0
+ while 1:
+ bak = "%s.%d" % (data_dir, i)
+ if not os.path.isdir(bak):
+ break
+ i += 1
+
+ # move old data away
+ if os.path.isdir(data_dir):
+ self.log.info("Move %s to %s" % (data_dir, bak))
+ if not self.not_really:
+ os.rename(data_dir, bak)
+
+ # move new data
+ self.log.info("Move %s to %s" % (full_dir, data_dir))
+ if not self.not_really:
+ os.rename(full_dir, data_dir)
+ else:
+ data_dir = full_dir
+
+ # re-link tablespaces
+ spc_dir = os.path.join(data_dir, "pg_tblspc")
+ tmp_dir = os.path.join(data_dir, "tmpspc")
+ if os.path.isdir(spc_dir) and os.path.isdir(tmp_dir):
+ self.log.info("Linking tablespaces to temporary location")
+
+ # don't look into spc_dir, thus allowing
+ # user to move them before. re-link only those
+ # that are still in tmp_dir
+ list = os.listdir(tmp_dir)
+ list.sort()
+
+ for d in list:
+ if d[0] == ".":
+ continue
+ link_loc = os.path.join(spc_dir, d)
+ link_dst = os.path.join(tmp_dir, d)
+ self.log.info("Linking tablespace %s to %s" % (d, link_dst))
+ if not self.not_really:
+ if os.path.islink(link_loc):
+ os.remove(link_loc)
+ os.symlink(link_dst, link_loc)
+
+ # write recovery.conf
+ rconf = os.path.join(data_dir, "recovery.conf")
+ script = os.path.abspath(sys.argv[0])
+ cf_file = os.path.abspath(self.cf.filename)
+ conf = "\nrestore_command = '%s %s %s'\n" % (
+ script, cf_file, 'xrestore %f "%p"')
+ self.log.info("Write %s" % rconf)
+ if self.not_really:
+ print conf
+ else:
+ f = open(rconf, "w")
+ f.write(conf)
+ f.close()
+
+ # remove stopfile
+ srcdir = self.cf.get("completed_wals")
+ stopfile = os.path.join(srcdir, "STOP")
+ if os.path.isfile(stopfile):
+ self.log.info("Removing stopfile: "+stopfile)
+ if not self.not_really:
+ os.remove(stopfile)
+
+ # run database in recovery mode
+ self.log.info("Starting postmaster: " + start_cmd)
+ if not self.not_really:
+ os.system(start_cmd)
+
+ def slave_boot(self):
+ srcdir = self.cf.get("completed_wals")
+ stopfile = os.path.join(srcdir, "STOP")
+ open(stopfile, "w").write("1")
+ self.log.info("Stopping recovery mode")
+
+ def slave_pause(self):
+ srcdir = self.cf.get("completed_wals")
+ pausefile = os.path.join(srcdir, "PAUSE")
+ open(pausefile, "w").write("1")
+ self.log.info("Pausing recovery mode")
+
+ def slave_continue(self):
+ srcdir = self.cf.get("completed_wals")
+ pausefile = os.path.join(srcdir, "PAUSE")
+ if os.path.isfile(pausefile):
+ os.remove(pausefile)
+ self.log.info("Continuing with recovery")
+ else:
+ self.log.info("Recovery not paused?")
+
+ def slave_cleanup(self, last_applied):
+ completed_wals = self.cf.get("completed_wals")
+ partial_wals = self.cf.get("partial_wals")
+
+ self.log.debug("cleaning completed wals since %s" % last_applied)
+ last = self.del_wals(completed_wals, last_applied)
+ if last:
+ if os.path.isdir(partial_wals):
+ self.log.debug("cleaning partial wals since %s" % last)
+ self.del_wals(partial_wals, last)
+ else:
+ self.log.warning("partial_wals dir does not exist: %s"
+ % partial_wals)
+ self.log.debug("cleaning done")
+
+ def del_wals(self, path, last):
+ dot = last.find(".")
+ if dot > 0:
+ last = last[:dot]
+ list = os.listdir(path)
+ list.sort()
+ cur_last = None
+ n = len(list)
+ for i in range(n):
+ fname = list[i]
+ full = os.path.join(path, fname)
+ if fname[0] < "0" or fname[0] > "9":
+ continue
+
+ ok_del = 0
+ if fname < last:
+ self.log.debug("deleting %s" % full)
+ os.remove(full)
+ cur_last = fname
+ return cur_last
+
+ def work(self):
+ self.master_sync()
+
+def main():
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "nh")
+ except getopt.error, det:
+ print det
+ usage(1)
+ not_really = 0
+ for o, v in opts:
+ if o == "-n":
+ not_really = 1
+ elif o == "-h":
+ usage(0)
+ if len(args) < 2:
+ usage(1)
+ ini = args[0]
+ cmd = args[1]
+
+ if cmd == "setup":
+ script = WalMgr(MASTER, ini, not_really)
+ script.master_setup()
+ elif cmd == "stop":
+ script = WalMgr(MASTER, ini, not_really)
+ script.master_stop()
+ elif cmd == "backup":
+ script = WalMgr(MASTER, ini, not_really)
+ script.master_backup()
+ elif cmd == "xarchive":
+ if len(args) != 4:
+ print >> sys.stderr, "usage: walmgr INI xarchive %p %f"
+ sys.exit(1)
+ script = WalMgr(MASTER, ini, not_really, 1)
+ script.master_xarchive(args[2], args[3])
+ elif cmd == "sync":
+ script = WalMgr(MASTER, ini, not_really)
+ script.master_sync()
+ elif cmd == "syncdaemon":
+ script = WalMgr(MASTER, ini, not_really, go_daemon=1)
+ script.start()
+ elif cmd == "xrestore":
+ if len(args) != 4:
+ print >> sys.stderr, "usage: walmgr INI xrestore %p %f"
+ sys.exit(1)
+ script = WalMgr(SLAVE, ini, not_really, 1)
+ script.slave_xrestore(args[2], args[3])
+ elif cmd == "restore":
+ script = WalMgr(SLAVE, ini, not_really)
+ script.slave_startup()
+ elif cmd == "boot":
+ script = WalMgr(SLAVE, ini, not_really)
+ script.slave_boot()
+ elif cmd == "pause":
+ script = WalMgr(SLAVE, ini, not_really)
+ script.slave_pause()
+ elif cmd == "continue":
+ script = WalMgr(SLAVE, ini, not_really)
+ script.slave_continue()
+ else:
+ usage(1)
+
+if __name__ == '__main__':
+ main()
+
--- /dev/null
+[bulk_loader]
+job_name = bizgres_loader
+
+src_db = dbname=bulksrc
+dst_db = dbname=bulkdst
+
+pgq_queue_name = xx
+
+use_skylog = 1
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Bulkloader for slow databases (Bizgres).
+
+Idea is following:
+ - Script reads from queue a batch of urlencoded row changes.
+ Inserts/updates/deletes, maybe many per one row.
+ - It changes them to minimal amount of DELETE commands
+ followed by big COPY of new data.
+ - One side-effect is that total order of how rows appear
+ changes, but per-row changes will be kept in order.
+
+The speedup from the COPY will happen only if the batches are
+large enough. So the ticks should happen only after couple
+of minutes.
+
+"""
+
+import sys, os, pgq, skytools
+
+def mk_delete_sql(tbl, key_list, data):
+ """ generate delete command """
+ whe_list = []
+ for k in key_list:
+ whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k])))
+ whe_str = " and ".join(whe_list)
+ return "delete from %s where %s;" % (tbl, whe_str)
+
+class TableCache(object):
+ """Per-table data hander."""
+
+ def __init__(self, tbl):
+ """Init per-batch table data cache."""
+ self.name = tbl
+ self.ev_list = []
+ self.pkey_map = {}
+ self.pkey_list = []
+ self.pkey_str = None
+ self.col_list = None
+
+ def add_event(self, ev):
+ """Store new event."""
+
+ # op & data
+ ev.op = ev.type[0]
+ ev.row = skytools.db_urldecode(ev.data)
+
+ # get pkey column names
+ if self.pkey_str is None:
+ self.pkey_str = ev.type.split(':')[1]
+ if self.pkey_str:
+ self.pkey_list = self.pkey_str.split(',')
+
+ # get pkey value
+ if self.pkey_str:
+ pk_data = []
+ for k in self.pkey_list:
+ pk_data.append(ev.row[k])
+ ev.pk_data = tuple(pk_data)
+ elif ev.op == 'I':
+ # fake pkey, just to get them spread out
+ ev.pk_data = ev.id
+ else:
+ raise Exception('non-pk tables not supported: %s' % ev.extra1)
+
+ # get full column list, detect added columns
+ if not self.col_list:
+ self.col_list = ev.row.keys()
+ elif self.col_list != ev.row.keys():
+ # ^ supposedly python guarantees same order in keys()
+
+ # find new columns
+ for c in ev.row.keys():
+ if c not in self.col_list:
+ for oldev in self.ev_list:
+ oldev.row[c] = None
+ self.col_list = ev.row.keys()
+
+ # add to list
+ self.ev_list.append(ev)
+
+ # keep all versions of row data
+ if ev.pk_data in self.pkey_map:
+ self.pkey_map[ev.pk_data].append(ev)
+ else:
+ self.pkey_map[ev.pk_data] = [ev]
+
+ def finish(self):
+ """Got all data, prepare for insertion."""
+
+ del_list = []
+ copy_list = []
+ for ev_list in self.pkey_map.values():
+ # rewrite list of I/U/D events to
+ # optional DELETE and optional INSERT/COPY command
+ exists_before = -1
+ exists_after = 1
+ for ev in ev_list:
+ if ev.op == "I":
+ if exists_before < 0:
+ exists_before = 0
+ exists_after = 1
+ elif ev.op == "U":
+ if exists_before < 0:
+ exists_before = 1
+ #exists_after = 1 # this shouldnt be needed
+ elif ev.op == "D":
+ if exists_before < 0:
+ exists_before = 1
+ exists_after = 0
+ else:
+ raise Exception('unknown event type: %s' % ev.op)
+
+ # skip short-lived rows
+ if exists_before == 0 and exists_after == 0:
+ continue
+
+ # take last event
+ ev = ev_list[-1]
+
+ # generate needed commands
+ if exists_before:
+ del_list.append(mk_delete_sql(self.name, self.pkey_list, ev.row))
+ if exists_after:
+ copy_list.append(ev.row)
+
+ # reorder cols
+ new_list = self.pkey_list[:]
+ for k in self.col_list:
+ if k not in self.pkey_list:
+ new_list.append(k)
+
+ return del_list, new_list, copy_list
+
+class BulkLoader(pgq.SerialConsumer):
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, "bulk_loader", "src_db", "dst_db", args)
+
+ def process_remote_batch(self, batch_id, ev_list, dst_db):
+ """Content dispatcher."""
+
+ # add events to per-table caches
+ tables = {}
+ for ev in ev_list:
+ tbl = ev.extra1
+
+ if not tbl in tables:
+ tables[tbl] = TableCache(tbl)
+ cache = tables[tbl]
+ cache.add_event(ev)
+ ev.tag_done()
+
+ # then process them
+ for tbl, cache in tables.items():
+ self.process_one_table(dst_db, tbl, cache)
+
+ def process_one_table(self, dst_db, tbl, cache):
+ self.log.debug("process_one_table: %s" % tbl)
+ del_list, col_list, copy_list = cache.finish()
+ curs = dst_db.cursor()
+
+ if not skytools.exists_table(curs, tbl):
+ self.log.warning("Ignoring events for table: %s" % tbl)
+ return
+
+ if len(del_list) > 0:
+ self.log.info("Deleting %d rows from %s" % (len(del_list), tbl))
+
+ q = " ".join(del_list)
+ self.log.debug(q)
+ curs.execute(q)
+
+ if len(copy_list) > 0:
+ self.log.info("Copying %d rows into %s" % (len(copy_list), tbl))
+ self.log.debug("COPY %s (%s)" % (tbl, ','.join(col_list)))
+ skytools.magic_insert(curs, tbl, copy_list, col_list)
+
+if __name__ == '__main__':
+ script = BulkLoader(sys.argv[1:])
+ script.start()
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Prints out SQL files with psql command execution.
+
+Supported psql commands: \i, \cd, \q
+Others are skipped.
+
+Aditionally does some pre-processing for NDoc.
+NDoc is looks nice but needs some hand-holding.
+
+Bug:
+
+- function def end detection searches for 'as'/'is' but does not check
+ word boundaries - finds them even in function name. That means in
+ main conf, as/is must be disabled and $ ' added. This script can
+ remove the unnecessary AS from output.
+
+Niceties:
+
+- Ndoc includes function def in output only if def is after comment.
+ But for SQL functions its better to have it after def.
+ This script can swap comment and def.
+
+- Optionally remove CREATE FUNCTION (OR REPLACE) from def to
+ keep it shorter in doc.
+
+Note:
+
+- NDoc compares real function name and name in comment. if differ,
+ it decides detection failed.
+
+"""
+
+import sys, os, re, getopt
+
+def usage(x):
+ print "usage: catsql [--ndoc] FILE [FILE ...]"
+ sys.exit(x)
+
+# NDoc specific changes
+cf_ndoc = 0
+
+# compile regexes
+func_re = r"create\s+(or\s+replace\s+)?function\s+"
+func_rc = re.compile(func_re, re.I)
+comm_rc = re.compile(r"^\s*([#]\s*)?(?P<com>--.*)", re.I)
+end_rc = re.compile(r"\b([;]|begin|declare|end)\b", re.I)
+as_rc = re.compile(r"\s+as\s+", re.I)
+cmd_rc = re.compile(r"^\\([a-z]*)(\s+.*)?", re.I)
+
+# conversion func
+def fix_func(ln):
+ # if ndoc, replace AS with ' '
+ if cf_ndoc:
+ return as_rc.sub(' ', ln)
+ else:
+ return ln
+
+# got function def
+def proc_func(f, ln):
+ # remove CREATE OR REPLACE
+ if cf_ndoc:
+ ln = func_rc.sub('', ln)
+
+ ln = fix_func(ln)
+ pre_list = [ln]
+ comm_list = []
+ n_comm = 0
+ while 1:
+ ln = f.readline()
+ if not ln:
+ break
+
+ com = None
+ if cf_ndoc:
+ com = comm_rc.search(ln)
+ if cf_ndoc and com:
+ pos = com.start('com')
+ comm_list.append(ln[pos:])
+ elif end_rc.search(ln):
+ break
+ elif len(comm_list) > 0:
+ break
+ else:
+ pre_list.append(fix_func(ln))
+
+ if len(comm_list) > 2:
+ map(sys.stdout.write, comm_list)
+ map(sys.stdout.write, pre_list)
+ else:
+ map(sys.stdout.write, pre_list)
+ map(sys.stdout.write, comm_list)
+ if ln:
+ sys.stdout.write(fix_func(ln))
+
+def cat_file(fn):
+ sys.stdout.write("\n")
+ f = open(fn)
+ while 1:
+ ln = f.readline()
+ if not ln:
+ break
+ m = cmd_rc.search(ln)
+ if m:
+ cmd = m.group(1)
+ if cmd == "i": # include a file
+ fn2 = m.group(2).strip()
+ cat_file(fn2)
+ elif cmd == "q": # quit
+ sys.exit(0)
+ elif cmd == "cd": # chdir
+ dir = m.group(2).strip()
+ os.chdir(dir)
+ else: # skip all others
+ pass
+ else:
+ if func_rc.search(ln): # function header
+ proc_func(f, ln)
+ else: # normal sql
+ sys.stdout.write(ln)
+ sys.stdout.write("\n")
+
+def main():
+ global cf_ndoc
+
+ try:
+ opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['ndoc'])
+ except getopt.error, d:
+ print d
+ usage(1)
+ for o, v in opts:
+ if o == "-h":
+ usage(0)
+ elif o == "--ndoc":
+ cf_ndoc = 1
+ for fn in args:
+ cat_file(fn)
+
+if __name__ == '__main__':
+ main()
+
--- /dev/null
+[cube_dispatcher]
+job_name = some_queue_to_cube
+
+src_db = dbname=sourcedb_test
+dst_db = dbname=dataminedb_test
+
+pgq_queue_name = udata.some_queue
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+# how many rows are kept: keep_latest, keep_all
+mode = keep_latest
+
+# to_char() fmt for table suffix
+#dateformat = YYYY_MM_DD
+# following disables table suffixes:
+#dateformat =
+
+part_template =
+ create table _DEST_TABLE (like _PARENT);
+ alter table only _DEST_TABLE add primary key (_PKEY);
+
--- /dev/null
+#! /usr/bin/env python
+
+# it accepts urlencoded rows for multiple tables from queue
+# and insert them into actual tables, with partitioning on tick time
+
+import sys, os, pgq, skytools
+
+DEF_CREATE = """
+create table _DEST_TABLE (like _PARENT);
+alter table only _DEST_TABLE add primary key (_PKEY);
+"""
+
+class CubeDispatcher(pgq.SerialConsumer):
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, "cube_dispatcher", "src_db", "dst_db", args)
+
+ self.dateformat = self.cf.get('dateformat', 'YYYY_MM_DD')
+
+ self.part_template = self.cf.get('part_template', DEF_CREATE)
+
+ mode = self.cf.get('mode', 'keep_latest')
+ if mode == 'keep_latest':
+ self.keep_latest = 1
+ elif mode == 'keep_all':
+ self.keep_latest = 0
+ else:
+ self.log.fatal('wrong mode setting')
+ sys.exit(1)
+
+ def get_part_date(self, batch_id):
+ if not self.dateformat:
+ return None
+
+ # fetch and format batch date
+ src_db = self.get_database('src_db')
+ curs = src_db.cursor()
+ q = 'select to_char(batch_end, %s) from pgq.get_batch_info(%s)'
+ curs.execute(q, [self.dateformat, batch_id])
+ src_db.commit()
+ return curs.fetchone()[0]
+
+ def process_remote_batch(self, batch_id, ev_list, dst_db):
+
+ # actual processing
+ date_str = self.get_part_date(batch_id)
+ self.dispatch(dst_db, ev_list, self.get_part_date(batch_id))
+
+ # tag as done
+ for ev in ev_list:
+ ev.tag_done()
+
+ def dispatch(self, dst_db, ev_list, date_str):
+ """Actual event processing."""
+
+ # get tables and sql
+ tables = {}
+ sql_list = []
+ for ev in ev_list:
+ if date_str:
+ tbl = "%s_%s" % (ev.extra1, date_str)
+ else:
+ tbl = ev.extra1
+
+ if not tbl in tables:
+ tables[tbl] = self.get_table_info(ev, tbl)
+
+ sql = self.make_sql(tbl, ev)
+ sql_list.append(sql)
+
+ # create tables if needed
+ self.check_tables(dst_db, tables)
+
+ # insert into data tables
+ curs = dst_db.cursor()
+ block = []
+ for sql in sql_list:
+ self.log.debug(sql)
+ block.append(sql)
+ if len(block) > 100:
+ curs.execute("\n".join(block))
+ block = []
+ if len(block) > 0:
+ curs.execute("\n".join(block))
+
+ def get_table_info(self, ev, tbl):
+ inf = {
+ 'parent': ev.extra1,
+ 'table': tbl,
+ 'key_list': ev.type.split(':')[1]
+ }
+ return inf
+
+ def make_sql(self, tbl, ev):
+ """Return SQL statement(s) for that event."""
+
+ # parse data
+ data = skytools.db_urldecode(ev.data)
+
+ # parse tbl info
+ op, keys = ev.type.split(':')
+ key_list = keys.split(',')
+ if self.keep_latest and len(key_list) == 0:
+ raise Exception('No pkey on table %s' % tbl)
+
+ # generate sql
+ if op in ('I', 'U'):
+ if self.keep_latest:
+ sql = "%s %s" % (self.mk_delete_sql(tbl, key_list, data),
+ self.mk_insert_sql(tbl, key_list, data))
+ else:
+ sql = self.mk_insert_sql(tbl, key_list, data)
+ elif op == "D":
+ if not self.keep_latest:
+ raise Exception('Delete op not supported if mode=keep_all')
+
+ sql = self.mk_delete_sql(tbl, key_list, data)
+ else:
+ raise Exception('Unknown row op: %s' % op)
+ return sql
+
+ def mk_delete_sql(self, tbl, key_list, data):
+ # generate delete command
+ whe_list = []
+ for k in key_list:
+ whe_list.append("%s = %s" % (k, skytools.quote_literal(data[k])))
+ whe_str = " and ".join(whe_list)
+ return "delete from %s where %s;" % (tbl, whe_str)
+
+ def mk_insert_sql(self, tbl, key_list, data):
+ # generate insert command
+ col_list = []
+ val_list = []
+ for c, v in data.items():
+ col_list.append(c)
+ val_list.append(skytools.quote_literal(v))
+ col_str = ",".join(col_list)
+ val_str = ",".join(val_list)
+ return "insert into %s (%s) values (%s);" % (
+ tbl, col_str, val_str)
+
+ def check_tables(self, dcon, tables):
+ """Checks that tables needed for copy are there. If not
+ then creates them.
+
+ Used by other procedures to ensure that table is there
+ before they start inserting.
+
+ The commits should not be dangerous, as we haven't done anything
+ with cdr's yet, so they should still be in one TX.
+
+ Although it would be nicer to have a lock for table creation.
+ """
+
+ dcur = dcon.cursor()
+ exist_map = {}
+ for tbl, inf in tables.items():
+ if skytools.exists_table(dcur, tbl):
+ continue
+
+ sql = self.part_template
+ sql = sql.replace('_DEST_TABLE', inf['table'])
+ sql = sql.replace('_PARENT', inf['parent'])
+ sql = sql.replace('_PKEY', inf['key_list'])
+ # be similar to table_dispatcher
+ schema_table = inf['table'].replace(".", "__")
+ sql = sql.replace('_SCHEMA_TABLE', schema_table)
+
+ dcur.execute(sql)
+ dcon.commit()
+ self.log.info('%s: Created table %s' % (self.job_name, tbl))
+
+if __name__ == '__main__':
+ script = CubeDispatcher(sys.argv[1:])
+ script.start()
+
--- /dev/null
+[queue_mover]
+job_name = queue_mover_test
+
+src_db = dbname=sourcedb_test
+dst_db = dbname=dataminedb_test
+
+pgq_queue_name = source_queue
+dst_queue_name = dest_queue
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+use_skylog = 0
+
--- /dev/null
+#! /usr/bin/env python
+
+# this script simply mover events from one queue to another
+
+import sys, os, pgq, skytools
+
+class QueueMover(pgq.SerialConsumer):
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, "queue_mover", "src_db", "dst_db", args)
+
+ self.dst_queue_name = self.cf.get("dst_queue_name")
+
+ def process_remote_batch(self, db, batch_id, ev_list, dst_db):
+
+ # load data
+ rows = []
+ for ev in ev_list:
+ data = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time]
+ rows.append(data)
+ ev.tag_done()
+ fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time']
+
+ # insert data
+ curs = dst_db.cursor()
+ pgq.bulk_insert_events(curs, rows, fields, self.dst_queue_name)
+
+if __name__ == '__main__':
+ script = QueueMover(sys.argv[1:])
+ script.start()
+
--- /dev/null
+[queue_splitter]
+job_name = queue_splitter_test
+
+src_db = dbname=sourcedb_test
+dst_db = dbname=destdb_test
+
+pgq_queue_name = source_queue
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+use_skylog = 0
+
--- /dev/null
+#! /usr/bin/env python
+
+# puts events into queue specified by field from 'queue_field' config parameter
+
+import sys, os, pgq, skytools
+
+class QueueSplitter(pgq.SerialConsumer):
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, "queue_splitter", "src_db", "dst_db", args)
+
+ def process_remote_batch(self, db, batch_id, ev_list, dst_db):
+ cache = {}
+ queue_field = self.cf.get('queue_field', 'extra1')
+ for ev in ev_list:
+ row = [ev.type, ev.data, ev.extra1, ev.extra2, ev.extra3, ev.extra4, ev.time]
+ queue = ev.__getattr__(queue_field)
+ if queue not in cache:
+ cache[queue] = []
+ cache[queue].append(row)
+ ev.tag_done()
+
+ # should match the composed row
+ fields = ['type', 'data', 'extra1', 'extra2', 'extra3', 'extra4', 'time']
+
+ # now send them to right queues
+ curs = dst_db.cursor()
+ for queue, rows in cache.items():
+ pgq.bulk_insert_events(curs, rows, fields, queue)
+
+if __name__ == '__main__':
+ script = QueueSplitter(sys.argv[1:])
+ script.start()
+
--- /dev/null
+
+[scriptmgr]
+job_name = scriptmgr_cphdb5
+config_list = ~/dbscripts/conf/*.ini, ~/random/conf/*.ini
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+#use_skylog = 1
+
+#
+# defaults for services
+#
+[DEFAULT]
+cwd = ~/dbscripts
+args = -v
+
+#
+# service descriptions
+#
+
+[cube_dispatcher]
+script = cube_dispatcher.py
+
+[table_dispatcher]
+script = table_dispatcher.py
+
+[bulk_loader]
+script = bulk_loader.py
+
+[londiste]
+script = londiste.py
+args = replay
+
+[pgqadm]
+script = pgqadm.py
+args = ticker
+
+#
+# services to be ignored
+#
+
+[log_checker]
+disabled = 1
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Bulk start/stop of scripts.
+
+Reads a bunch of config files and maps them to scripts, then handles those.
+"""
+
+import sys, os, skytools, signal, glob, ConfigParser, time
+
+command_usage = """
+%prog [options] INI CMD [subcmd args]
+
+commands:
+ start [-a | jobname ..] start a job
+ stop [-a | jobname ..] stop a job
+ restart [-a | jobname ..] restart job(s)
+ reload [-a | jobname ..] send reload signal
+ status
+"""
+
+def job_sort_cmp(j1, j2):
+ d1 = j1['service'] + j1['job_name']
+ d2 = j2['service'] + j2['job_name']
+ if d1 < d2: return -1
+ elif d1 > d2: return 1
+ else: return 0
+
+class ScriptMgr(skytools.DBScript):
+ def init_optparse(self, p = None):
+ p = skytools.DBScript.init_optparse(self, p)
+ p.add_option("-a", "--all", action="store_true", help="apply command to all jobs")
+ p.set_usage(command_usage.strip())
+ return p
+
+ def load_jobs(self):
+ self.svc_list = []
+ self.svc_map = {}
+ self.config_list = []
+
+ # load services
+ svc_list = self.cf.sections()
+ svc_list.remove(self.service_name)
+ for svc_name in svc_list:
+ cf = self.cf.clone(svc_name)
+ disabled = cf.getboolean('disabled', 0)
+ defscript = None
+ if disabled:
+ defscript = '/disabled'
+ svc = {
+ 'service': svc_name,
+ 'script': cf.getfile('script', defscript),
+ 'cwd': cf.getfile('cwd'),
+ 'disabled': cf.getboolean('disabled', 0),
+ 'args': cf.get('args', ''),
+ }
+ self.svc_list.append(svc)
+ self.svc_map[svc_name] = svc
+
+ # generate config list
+ for tmp in self.cf.getlist('config_list'):
+ tmp = os.path.expanduser(tmp)
+ tmp = os.path.expandvars(tmp)
+ for fn in glob.glob(tmp):
+ self.config_list.append(fn)
+
+ # read jobs
+ for fn in self.config_list:
+ raw = ConfigParser.SafeConfigParser({'job_name':'?', 'service_name':'?'})
+ raw.read(fn)
+
+ # skip its own config
+ if raw.has_section(self.service_name):
+ continue
+
+ got = 0
+ for sect in raw.sections():
+ if sect in self.svc_map:
+ got = 1
+ self.add_job(fn, sect)
+ if not got:
+ self.log.warning('Cannot find service for %s' % fn)
+
+ def add_job(self, cf_file, service_name):
+ svc = self.svc_map[service_name]
+ cf = skytools.Config(service_name, cf_file)
+ disabled = svc['disabled']
+ if not disabled:
+ disabled = cf.getboolean('disabled', 0)
+ job = {
+ 'disabled': disabled,
+ 'config': cf_file,
+ 'cwd': svc['cwd'],
+ 'script': svc['script'],
+ 'args': svc['args'],
+ 'service': svc['service'],
+ 'job_name': cf.get('job_name'),
+ 'pidfile': cf.getfile('pidfile'),
+ }
+ self.job_list.append(job)
+ self.job_map[job['job_name']] = job
+
+ def cmd_status(self):
+ for job in self.job_list:
+ os.chdir(job['cwd'])
+ cf = skytools.Config(job['service'], job['config'])
+ pidfile = cf.getfile('pidfile')
+ name = job['job_name']
+ svc = job['service']
+ if job['disabled']:
+ name += " (disabled)"
+
+ if os.path.isfile(pidfile):
+ print " OK [%s] %s" % (svc, name)
+ else:
+ print " STOPPED [%s] %s" % (svc, name)
+
+ def cmd_info(self):
+ for job in self.job_list:
+ print job
+
+ def cmd_start(self, job_name):
+ job = self.job_map[job_name]
+ if job['disabled']:
+ self.log.info("Skipping %s" % job_name)
+ return 0
+ self.log.info('Starting %s' % job_name)
+ os.chdir(job['cwd'])
+ pidfile = job['pidfile']
+ if os.path.isfile(pidfile):
+ self.log.warning("Script %s seems running")
+ return 0
+ cmd = "%(script)s %(config)s %(args)s -d" % job
+ res = os.system(cmd)
+ self.log.debug(res)
+ if res != 0:
+ self.log.error('startup failed: %s' % job_name)
+ return 1
+ else:
+ return 0
+
+ def cmd_stop(self, job_name):
+ job = self.job_map[job_name]
+ if job['disabled']:
+ self.log.info("Skipping %s" % job_name)
+ return
+ self.log.info('Stopping %s' % job_name)
+ self.signal_job(job, signal.SIGINT)
+
+ def cmd_reload(self, job_name):
+ job = self.job_map[job_name]
+ if job['disabled']:
+ self.log.info("Skipping %s" % job_name)
+ return
+ self.log.info('Reloading %s' % job_name)
+ self.signal_job(job, signal.SIGHUP)
+
+ def signal_job(self, job, sig):
+ os.chdir(job['cwd'])
+ pidfile = job['pidfile']
+ if os.path.isfile(pidfile):
+ pid = int(open(pidfile).read())
+ os.kill(pid, sig)
+ else:
+ self.log.warning("Job %s not running" % job['job_name'])
+
+ def work(self):
+ self.set_single_loop(1)
+ self.job_list = []
+ self.job_map = {}
+ self.load_jobs()
+
+ if len(self.args) < 2:
+ print "need command"
+ sys.exit(1)
+
+ jobs = self.args[2:]
+ if len(jobs) == 0 and self.options.all:
+ for job in self.job_list:
+ jobs.append(job['job_name'])
+
+ self.job_list.sort(job_sort_cmp)
+
+ cmd = self.args[1]
+ if cmd == "status":
+ self.cmd_status()
+ return
+ elif cmd == "info":
+ self.cmd_info()
+ return
+
+ if len(jobs) == 0:
+ print "no jobs given?"
+ sys.exit(1)
+
+ if cmd == "start":
+ err = 0
+ for n in jobs:
+ err += self.cmd_start(n)
+ if err > 0:
+ self.log.error('some scripts failed')
+ sys.exit(1)
+ elif cmd == "stop":
+ for n in jobs:
+ self.cmd_stop(n)
+ elif cmd == "restart":
+ for n in jobs:
+ self.cmd_stop(n)
+ time.sleep(2)
+ self.cmd_start(n)
+ elif cmd == "reload":
+ for n in self.jobs:
+ self.cmd_reload(n)
+ else:
+ print "unknown command:", cmd
+ sys.exit(1)
+
+if __name__ == '__main__':
+ script = ScriptMgr('scriptmgr', sys.argv[1:])
+ script.start()
+
--- /dev/null
+[udata_dispatcher]
+job_name = test_move
+
+src_db = dbname=sourcedb_test
+dst_db = dbname=dataminedb_test
+
+pgq_queue_name = OrderLog
+
+logfile = ~/log/%(job_name)s.log
+pidfile = ~/pid/%(job_name)s.pid
+
+# where to put data. when partitioning, will be used as base name
+dest_table = orders
+
+# date field with will be used for partitioning
+# special value: _EVTIME - event creation time
+part_column = start_date
+
+#fields = *
+#fields = id, name
+#fields = id:newid, name, bar:baz
+
+
+# template used for creating partition tables
+# _DEST_TABLE
+part_template =
+ create table _DEST_TABLE () inherits (orders);
+ alter table only _DEST_TABLE add constraint _DEST_TABLE_pkey primary key (id);
+ grant select on _DEST_TABLE to group reporting;
+
+
--- /dev/null
+#! /usr/bin/env python
+
+# it loads urlencoded rows for one trable from queue and inserts
+# them into actual tables, with optional partitioning
+
+import sys, os, pgq, skytools
+
+DEST_TABLE = "_DEST_TABLE"
+SCHEMA_TABLE = "_SCHEMA_TABLE"
+
+class TableDispatcher(pgq.SerialConsumer):
+ def __init__(self, args):
+ pgq.SerialConsumer.__init__(self, "table_dispatcher", "src_db", "dst_db", args)
+
+ self.part_template = self.cf.get("part_template", '')
+ self.dest_table = self.cf.get("dest_table")
+ self.part_field = self.cf.get("part_field", '')
+ self.part_method = self.cf.get("part_method", 'daily')
+ if self.part_method not in ('daily', 'monthly'):
+ raise Exception('bad part_method')
+
+ if self.cf.get("fields", "*") == "*":
+ self.field_map = None
+ else:
+ self.field_map = {}
+ for fval in self.cf.getlist('fields'):
+ tmp = fval.split(':')
+ if len(tmp) == 1:
+ self.field_map[tmp[0]] = tmp[0]
+ else:
+ self.field_map[tmp[0]] = tmp[1]
+
+ def process_remote_batch(self, batch_id, ev_list, dst_db):
+ if len(ev_list) == 0:
+ return
+
+ # actual processing
+ self.dispatch(dst_db, ev_list)
+
+ # tag as done
+ for ev in ev_list:
+ ev.tag_done()
+
+ def dispatch(self, dst_db, ev_list):
+ """Generic dispatcher."""
+
+ # load data
+ tables = {}
+ for ev in ev_list:
+ row = skytools.db_urldecode(ev.data)
+
+ # guess dest table
+ if self.part_field:
+ if self.part_field == "_EVTIME":
+ partval = str(ev.creation_date)
+ else:
+ partval = str(row[self.part_field])
+ partval = partval.split(' ')[0]
+ date = partval.split('-')
+ if self.part_method == 'monthly':
+ date = date[:2]
+ suffix = '_'.join(date)
+ tbl = "%s_%s" % (self.dest_table, suffix)
+ else:
+ tbl = self.dest_table
+
+ # map fields
+ if self.field_map is None:
+ dstrow = row
+ else:
+ dstrow = {}
+ for k, v in self.field_map.items():
+ dstrow[v] = row[k]
+
+ # add row into table
+ if not tbl in tables:
+ tables[tbl] = [dstrow]
+ else:
+ tables[tbl].append(dstrow)
+
+ # create tables if needed
+ self.check_tables(dst_db, tables)
+
+ # insert into data tables
+ curs = dst_db.cursor()
+ for tbl, tbl_rows in tables.items():
+ skytools.magic_insert(curs, tbl, tbl_rows)
+
+ def check_tables(self, dcon, tables):
+ """Checks that tables needed for copy are there. If not
+ then creates them.
+
+ Used by other procedures to ensure that table is there
+ before they start inserting.
+
+ The commits should not be dangerous, as we haven't done anything
+ with cdr's yet, so they should still be in one TX.
+
+ Although it would be nicer to have a lock for table creation.
+ """
+
+ dcur = dcon.cursor()
+ exist_map = {}
+ for tbl in tables.keys():
+ if not skytools.exists_table(dcur, tbl):
+ if not self.part_template:
+ raise Exception('Dest table does not exists and no way to create it.')
+
+ sql = self.part_template
+ sql = sql.replace(DEST_TABLE, tbl)
+
+ # we do this to make sure that constraints for
+ # tables who contain a schema will still work
+ schema_table = tbl.replace(".", "__")
+ sql = sql.replace(SCHEMA_TABLE, schema_table)
+
+ dcur.execute(sql)
+ dcon.commit()
+ self.log.info('%s: Created table %s' % (self.job_name, tbl))
+
+if __name__ == '__main__':
+ script = TableDispatcher(sys.argv[1:])
+ script.start()
+
--- /dev/null
+#! /usr/bin/env python
+
+from distutils.core import setup
+
+import re
+buf = open("configure.ac","r").read(256)
+m = re.search("AC_INIT[(][^,]*,\s+([^)]*)[)]", buf)
+ac_ver = m.group(1)
+
+setup(
+ name = "skytools",
+ license = "BSD",
+ version = ac_ver,
+ maintainer = "Marko Kreen",
+ maintainer_email = "marko.kreen@skype.net",
+ url = "http://pgfoundry.org/projects/skytools/",
+ package_dir = {'': 'python'},
+ packages = ['skytools', 'londiste', 'pgq'],
+ scripts = ['python/londiste.py', 'python/pgqadm.py', 'python/walmgr.py',
+ 'scripts/cube_dispatcher.py', 'scripts/queue_mover.py',
+ 'scripts/table_dispatcher.py', 'scripts/bulk_loader.py',
+ 'scripts/scriptmgr.py', 'scripts/queue_splitter.py'],
+ data_files = [ ('share/doc/skytools/conf', [
+ 'python/conf/londiste.ini',
+ 'python/conf/pgqadm.ini',
+ 'python/conf/wal-master.ini',
+ 'python/conf/wal-slave.ini',
+ 'scripts/queue_mover.ini.templ',
+ 'scripts/queue_splitter.ini.templ',
+ 'scripts/cube_dispatcher.ini.templ',
+ 'scripts/table_dispatcher.ini.templ',
+ 'scripts/bulk_loader.ini.templ',
+ 'scripts/scriptmgr.ini.templ',
+ ])]
+)
+
--- /dev/null
+# what to include in source distribution
+
+# MANIFEST.in for Python Distutils
+
+include Makefile COPYRIGHT README NEWS config.mak.in configure configure.ac source.cfg
+
+recursive-include sql *.sql Makefile *.out *.in *.[ch] README* *.in
+recursive-include python/conf *.ini
+recursive-include scripts *.py *.templ
+recursive-include debian changelog packages.in
+recursive-include doc *
+recursive-include tests *
+
--- /dev/null
+
+include ../config.mak
+
+SUBDIRS = logtriga londiste pgq pgq_ext txid
+
+all install clean distclean installcheck:
+ for dir in $(SUBDIRS); do \
+ make -C $$dir $@ DESTDIR=$(DESTDIR) || exit $?; \
+ done
+
--- /dev/null
+
+include ../../config.mak
+
+MODULE_big = logtriga
+SRCS = logtriga.c textbuf.c
+OBJS = $(SRCS:.c=.o)
+DATA_built = logtriga.sql
+
+REGRESS = logtriga
+
+include $(PGXS)
+
--- /dev/null
+
+logtriga - generic table changes logger
+=======================================
+
+logtriga provides generic table changes logging trigger.
+It prepares partial SQL statement about a change and
+gives it to user query.
+
+Usage
+-----
+
+ CREATE TRIGGER foo_log AFTER INSERT OR UPDATE OR DELETE ON foo_tbl
+ FOR EACH ROW EXECUTE PROCEDURE logtriga(column_types, query);
+
+Where column_types is a string where each charater defines type of
+that column. Known types:
+
+ * k - one of primary key columns for table.
+ * v - data column
+ * i - uninteresting column, to be ignored.
+
+Trigger function prepares 2 string arguments for query and executes it.
+
+ * $1 - Operation type: I/U/D.
+ * $2 - Partial SQL for event playback.
+
+ * INSERT INTO FOO_TBL (field, list) values (val1, val2)
+ * UPDATE FOO_TBL SET field1 = val1, field2 = val2 where key1 = kval1
+ * DELETE FROM FOO_TBL WHERE key1 = keyval1
+
+The upper-case part is left out.
+
+Example
+-------
+
+Following query emulates Slony-I behaviour:
+
+ insert into SL_SCHEMA.sl_log_1
+ (log_origin, log_xid, log_tableid,
+ log_actionseq, log_cmdtype, log_cmddata)
+ values (CLUSTER_IDENT, SL_SCHEMA.getCurrentXid(), TABLE_OID,
+ nextval('SL_SCHEMA.sl_action_seq'), $1, $2)
+
+The upper-case strings should be replaced with actual values
+on trigger creation.
+
+
--- /dev/null
+-- init
+\set ECHO none
+create table rtest (
+ id integer primary key,
+ dat text
+);
+NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest"
+create table clog (
+ id serial,
+ op text,
+ data text
+);
+NOTICE: CREATE TABLE will create implicit sequence "clog_id_seq" for serial column "clog.id"
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('kv',
+'insert into clog (op, data) values ($1, $2)');
+-- simple test
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+select * from clog; delete from clog;
+ id | op | data
+----+----+--------------------------------
+ 1 | I | (id,dat) values ('1','value1')
+ 2 | U | dat='value2' where id='1'
+ 3 | D | id='1'
+(3 rows)
+
+-- test new fields
+alter table rtest add column dat2 text;
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+select * from clog; delete from clog;
+ id | op | data
+----+----+--------------------------------
+ 4 | I | (id,dat) values ('1','value1')
+ 5 | U | dat='value2' where id='1'
+ 6 | D | id='1'
+(3 rows)
+
+-- test field rename
+alter table rtest alter column dat type integer using 0;
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5;
+delete from rtest;
+select * from clog; delete from clog;
+ id | op | data
+----+----+-----------------------------
+ 7 | I | (id,dat) values ('1','666')
+ 8 | U | dat='5' where id='1'
+ 9 | D | id='1'
+(3 rows)
+
+-- test field ignore
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('kiv',
+'insert into clog (op, data) values ($1, $2)');
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5, dat2 = 'newdat2';
+update rtest set dat = 6;
+delete from rtest;
+select * from clog; delete from clog;
+ id | op | data
+----+----+---------------------------------
+ 10 | I | (id,dat2) values ('1','newdat')
+ 11 | U | dat2='newdat2' where id='1'
+ 12 | D | id='1'
+(3 rows)
+
+-- test wrong key
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('vik',
+'insert into clog (op, data) values ($1, $2)');
+insert into rtest values (1, 0, 'non-null');
+insert into rtest values (2, 0, NULL);
+update rtest set dat2 = 'non-null2' where id=1;
+update rtest set dat2 = NULL where id=1;
+update rtest set dat2 = 'new-nonnull' where id=2;
+ERROR: logtriga: Unexpected NULL key value
+delete from rtest where id=1;
+ERROR: logtriga: Unexpected NULL key value
+delete from rtest where id=2;
+ERROR: logtriga: Unexpected NULL key value
+select * from clog; delete from clog;
+ id | op | data
+----+----+----------------------------------------
+ 13 | I | (id,dat2) values ('1','non-null')
+ 14 | I | (id,dat2) values ('2',null)
+ 15 | U | dat2='non-null2' where dat2='non-null'
+ 16 | U | dat2=NULL where dat2='non-null2'
+(4 rows)
+
--- /dev/null
+/* ----------------------------------------------------------------------
+ * logtriga.c
+ *
+ * Generic trigger for logging table changes.
+ * Based on Slony-I log trigger.
+ * Does not depend on event storage.
+ *
+ * Copyright (c) 2003-2006, PostgreSQL Global Development Group
+ * Author: Jan Wieck, Afilias USA INC.
+ *
+ * Generalized by Marko Kreen.
+ * ----------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "executor/spi.h"
+#include "commands/trigger.h"
+#include "catalog/pg_operator.h"
+#include "utils/typcache.h"
+
+#include "textbuf.h"
+
+PG_FUNCTION_INFO_V1(logtriga);
+Datum logtriga(PG_FUNCTION_ARGS);
+
+#ifdef PG_MODULE_MAGIC
+PG_MODULE_MAGIC;
+#endif
+
+/*
+ * There may be several plans to be cached.
+ *
+ * FIXME: plans are kept in singe-linked list
+ * so not very fast access. Probably they should be
+ * handled more intelligently.
+ */
+typedef struct PlanCache PlanCache;
+
+struct PlanCache {
+ PlanCache *next;
+ char *query;
+ void *plan;
+};
+
+/*
+ * Cache result allocations.
+ */
+typedef struct ArgCache
+{
+ TBuf *op_type;
+ TBuf *op_data;
+} ArgCache;
+
+
+static PlanCache *plan_cache = NULL;
+static ArgCache *arg_cache = NULL;
+
+/*
+ * Cache helpers
+ */
+
+static void *get_plan(const char *query)
+{
+ PlanCache *c;
+ void *plan;
+ Oid plan_types[2];
+
+ for (c = plan_cache; c; c = c->next)
+ if (strcmp(query, c->query) == 0)
+ return c->plan;
+
+ /*
+ * Plan not cached, prepare new plan then.
+ */
+ plan_types[0] = TEXTOID;
+ plan_types[1] = TEXTOID;
+ plan = SPI_saveplan(SPI_prepare(query, 2, plan_types));
+ if (plan == NULL)
+ elog(ERROR, "logtriga: SPI_prepare() failed");
+
+ /* create cache object */
+ c = malloc(sizeof(*c));
+ if (!c)
+ elog(ERROR, "logtriga: no memory for plan cache");
+
+ c->plan = plan;
+ c->query = strdup(query);
+
+ /* insert at start */
+ c->next = plan_cache;
+ plan_cache = c;
+
+ return plan;
+}
+
+static ArgCache *
+get_arg_cache(void)
+{
+ if (arg_cache == NULL) {
+ ArgCache *a = malloc(sizeof(*a));
+ if (!a)
+ elog(ERROR, "logtriga: no memory");
+ memset(a, 0, sizeof(*a));
+ a->op_type = tbuf_alloc(8);
+ a->op_data = tbuf_alloc(8192);
+ arg_cache = a;
+ }
+ return arg_cache;
+}
+
+static void
+append_key_eq(TBuf *tbuf, const char *col_ident, const char *col_value)
+{
+ if (col_value == NULL)
+ elog(ERROR, "logtriga: Unexpected NULL key value");
+
+ tbuf_encode_cstring(tbuf, col_ident, "quote_ident");
+ tbuf_append_char(tbuf, '=');
+ tbuf_encode_cstring(tbuf, col_value, "quote_literal");
+}
+
+static void
+append_normal_eq(TBuf *tbuf, const char *col_ident, const char *col_value)
+{
+ tbuf_encode_cstring(tbuf, col_ident, "quote_ident");
+ tbuf_append_char(tbuf, '=');
+ if (col_value != NULL)
+ tbuf_encode_cstring(tbuf, col_value, "quote_literal");
+ else
+ tbuf_append_cstring(tbuf, "NULL");
+}
+
+static void process_insert(ArgCache *cs, TriggerData *tg, char *attkind)
+{
+ HeapTuple new_row = tg->tg_trigtuple;
+ TupleDesc tupdesc = tg->tg_relation->rd_att;
+ int i;
+ int need_comma = false;
+ int attkind_idx;
+
+ /*
+ * INSERT
+ *
+ * op_type = 'I' op_data = ("non-NULL-col" [, ...]) values ('value' [,
+ * ...])
+ */
+ tbuf_append_cstring(cs->op_type, "I");
+
+ /*
+ * Specify all the columns
+ */
+ tbuf_append_char(cs->op_data, '(');
+ attkind_idx = -1;
+ for (i = 0; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ char *col_ident;
+
+ /* Skip dropped columns */
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ /* Check if allowed by colstring */
+ attkind_idx++;
+ if (attkind[attkind_idx] == '\0')
+ break;
+ if (attkind[attkind_idx] == 'i')
+ continue;
+
+ if (need_comma)
+ tbuf_append_char(cs->op_data, ',');
+ else
+ need_comma = true;
+
+ /* quote column name */
+ col_ident = SPI_fname(tupdesc, i + 1);
+ tbuf_encode_cstring(cs->op_data, col_ident, "quote_ident");
+ }
+
+ /*
+ * Append the string ") values ("
+ */
+ tbuf_append_cstring(cs->op_data, ") values (");
+
+ /*
+ * Append the values
+ */
+ need_comma = false;
+ attkind_idx = -1;
+ for (i = 0; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ char *col_value;
+
+ /* Skip dropped columns */
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ /* Check if allowed by colstring */
+ attkind_idx++;
+ if (attkind[attkind_idx] == '\0')
+ break;
+ if (attkind[attkind_idx] == 'i')
+ continue;
+
+ if (need_comma)
+ tbuf_append_char(cs->op_data, ',');
+ else
+ need_comma = true;
+
+ /* quote column value */
+ col_value = SPI_getvalue(new_row, tupdesc, i + 1);
+ if (col_value == NULL)
+ tbuf_append_cstring(cs->op_data, "null");
+ else
+ tbuf_encode_cstring(cs->op_data, col_value, "quote_literal");
+ }
+
+ /*
+ * Terminate and done
+ */
+ tbuf_append_char(cs->op_data, ')');
+}
+
+static int process_update(ArgCache *cs, TriggerData *tg, char *attkind)
+{
+ HeapTuple old_row = tg->tg_trigtuple;
+ HeapTuple new_row = tg->tg_newtuple;
+ TupleDesc tupdesc = tg->tg_relation->rd_att;
+ Datum old_value;
+ Datum new_value;
+ bool old_isnull;
+ bool new_isnull;
+
+ char *col_ident;
+ char *col_value;
+ int i;
+ int need_comma = false;
+ int need_and = false;
+ int attkind_idx;
+ int ignore_count = 0;
+
+ /*
+ * UPDATE
+ *
+ * op_type = 'U' op_data = "col_ident"='value' [, ...] where "pk_ident" =
+ * 'value' [ and ...]
+ */
+ tbuf_append_cstring(cs->op_type, "U");
+
+ attkind_idx = -1;
+ for (i = 0; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ /*
+ * Ignore dropped columns
+ */
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ attkind_idx++;
+ if (attkind[attkind_idx] == '\0')
+ break;
+
+ old_value = SPI_getbinval(old_row, tupdesc, i + 1, &old_isnull);
+ new_value = SPI_getbinval(new_row, tupdesc, i + 1, &new_isnull);
+
+ /*
+ * If old and new value are NULL, the column is unchanged
+ */
+ if (old_isnull && new_isnull)
+ continue;
+
+ /*
+ * If both are NOT NULL, we need to compare the values and skip
+ * setting the column if equal
+ */
+ if (!old_isnull && !new_isnull)
+ {
+ Oid opr_oid;
+ FmgrInfo *opr_finfo_p;
+
+ /*
+ * Lookup the equal operators function call info using the
+ * typecache if available
+ */
+ TypeCacheEntry *type_cache;
+
+ type_cache = lookup_type_cache(SPI_gettypeid(tupdesc, i + 1),
+ TYPECACHE_EQ_OPR | TYPECACHE_EQ_OPR_FINFO);
+ opr_oid = type_cache->eq_opr;
+ if (opr_oid == ARRAY_EQ_OP)
+ opr_oid = InvalidOid;
+ else
+ opr_finfo_p = &(type_cache->eq_opr_finfo);
+
+ /*
+ * If we have an equal operator, use that to do binary
+ * comparision. Else get the string representation of both
+ * attributes and do string comparision.
+ */
+ if (OidIsValid(opr_oid))
+ {
+ if (DatumGetBool(FunctionCall2(opr_finfo_p,
+ old_value, new_value)))
+ continue;
+ }
+ else
+ {
+ char *old_strval = SPI_getvalue(old_row, tupdesc, i + 1);
+ char *new_strval = SPI_getvalue(new_row, tupdesc, i + 1);
+
+ if (strcmp(old_strval, new_strval) == 0)
+ continue;
+ }
+ }
+
+ if (attkind[attkind_idx] == 'i')
+ {
+ /* this change should be ignored */
+ ignore_count++;
+ continue;
+ }
+
+ if (need_comma)
+ tbuf_append_char(cs->op_data, ',');
+ else
+ need_comma = true;
+
+ col_ident = SPI_fname(tupdesc, i + 1);
+ col_value = SPI_getvalue(new_row, tupdesc, i + 1);
+
+ append_normal_eq(cs->op_data, col_ident, col_value);
+ }
+
+ /*
+ * It can happen that the only UPDATE an application does is to set a
+ * column to the same value again. In that case, we'd end up here with
+ * no columns in the SET clause yet. We add the first key column here
+ * with it's old value to simulate the same for the replication
+ * engine.
+ */
+ if (!need_comma)
+ {
+ /* there was change in ignored columns, skip whole event */
+ if (ignore_count > 0)
+ return 0;
+
+ for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ attkind_idx++;
+ if (attkind[attkind_idx] == 'k')
+ break;
+ }
+ col_ident = SPI_fname(tupdesc, i + 1);
+ col_value = SPI_getvalue(old_row, tupdesc, i + 1);
+
+ append_key_eq(cs->op_data, col_ident, col_value);
+ }
+
+ tbuf_append_cstring(cs->op_data, " where ");
+
+ for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ /*
+ * Ignore dropped columns
+ */
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ attkind_idx++;
+ if (attkind[attkind_idx] != 'k')
+ continue;
+ if (attkind[attkind_idx] == '\0')
+ break;
+ col_ident = SPI_fname(tupdesc, i + 1);
+ col_value = SPI_getvalue(old_row, tupdesc, i + 1);
+
+ if (need_and)
+ tbuf_append_cstring(cs->op_data, " and ");
+ else
+ need_and = true;
+
+ append_key_eq(cs->op_data, col_ident, col_value);
+ }
+ return 1;
+}
+
+static void process_delete(ArgCache *cs, TriggerData *tg, char *attkind)
+{
+ HeapTuple old_row = tg->tg_trigtuple;
+ TupleDesc tupdesc = tg->tg_relation->rd_att;
+ char *col_ident;
+ char *col_value;
+ int i;
+ int need_and = false;
+ int attkind_idx;
+
+ /*
+ * DELETE
+ *
+ * op_type = 'D' op_data = "pk_ident"='value' [and ...]
+ */
+ tbuf_append_cstring(cs->op_type, "D");
+
+ for (i = 0, attkind_idx = -1; i < tg->tg_relation->rd_att->natts; i++)
+ {
+ if (tupdesc->attrs[i]->attisdropped)
+ continue;
+
+ attkind_idx++;
+ if (attkind[attkind_idx] != 'k')
+ continue;
+ if (attkind[attkind_idx] == '\0')
+ break;
+ col_ident = SPI_fname(tupdesc, i + 1);
+ col_value = SPI_getvalue(old_row, tupdesc, i + 1);
+
+ if (need_and)
+ tbuf_append_cstring(cs->op_data, " and ");
+ else
+ need_and = true;
+
+ append_key_eq(cs->op_data, col_ident, col_value);
+ }
+}
+
+Datum logtriga(PG_FUNCTION_ARGS)
+{
+ TriggerData *tg;
+ Datum argv[2];
+ int rc;
+ ArgCache *cs;
+ char *attkind;
+ char *query;
+ int need_event = 1;
+
+ /*
+ * Get the trigger call context
+ */
+ if (!CALLED_AS_TRIGGER(fcinfo))
+ elog(ERROR, "logtriga not called as trigger");
+ tg = (TriggerData *) (fcinfo->context);
+
+ /*
+ * Check all logTrigger() calling conventions
+ */
+ if (!TRIGGER_FIRED_AFTER(tg->tg_event))
+ elog(ERROR, "logtriga must be fired AFTER");
+ if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event))
+ elog(ERROR, "logtriga must be fired FOR EACH ROW");
+ if (tg->tg_trigger->tgnargs != 2)
+ elog(ERROR, "logtriga must be defined with 2 args");
+
+ /*
+ * Connect to the SPI manager
+ */
+ if ((rc = SPI_connect()) < 0)
+ elog(ERROR, "logtriga: SPI_connect() failed");
+
+ cs = get_arg_cache();
+
+ tbuf_reset(cs->op_type);
+ tbuf_reset(cs->op_data);
+
+ /*
+ * Get all the trigger arguments
+ */
+ attkind = tg->tg_trigger->tgargs[0];
+ query = tg->tg_trigger->tgargs[1];
+
+ if (strchr(attkind, 'k') == NULL)
+ elog(ERROR, "logtriga: need at least one key column");
+
+ /*
+ * Determine cmdtype and op_data depending on the command type
+ */
+ if (TRIGGER_FIRED_BY_INSERT(tg->tg_event))
+ process_insert(cs, tg, attkind);
+ else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event))
+ need_event = process_update(cs, tg, attkind);
+ else if (TRIGGER_FIRED_BY_DELETE(tg->tg_event))
+ process_delete(cs, tg, attkind);
+ else
+ elog(ERROR, "logtriga fired for unhandled event");
+
+ /*
+ * Construct the parameter array and insert the log row.
+ */
+ if (need_event)
+ {
+ argv[0] = PointerGetDatum(tbuf_look_text(cs->op_type));
+ argv[1] = PointerGetDatum(tbuf_look_text(cs->op_data));
+ SPI_execp(get_plan(query), argv, NULL, 0);
+ }
+ SPI_finish();
+ return PointerGetDatum(NULL);
+}
+
--- /dev/null
+
+-- usage: logtriga(flds, query)
+--
+-- query should include 2 args:
+-- $1 - for op type I/U/D,
+-- $2 - for op data
+
+CREATE OR REPLACE FUNCTION logtriga() RETURNS trigger
+AS 'MODULE_PATHNAME', 'logtriga' LANGUAGE C;
+
--- /dev/null
+-- init
+\set ECHO none
+\i logtriga.sql
+\set ECHO all
+
+create table rtest (
+ id integer primary key,
+ dat text
+);
+
+create table clog (
+ id serial,
+ op text,
+ data text
+);
+
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('kv',
+'insert into clog (op, data) values ($1, $2)');
+
+-- simple test
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+select * from clog; delete from clog;
+
+-- test new fields
+alter table rtest add column dat2 text;
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+select * from clog; delete from clog;
+
+-- test field rename
+alter table rtest alter column dat type integer using 0;
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5;
+delete from rtest;
+select * from clog; delete from clog;
+
+
+-- test field ignore
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('kiv',
+'insert into clog (op, data) values ($1, $2)');
+
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5, dat2 = 'newdat2';
+update rtest set dat = 6;
+delete from rtest;
+select * from clog; delete from clog;
+
+
+-- test wrong key
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure logtriga('vik',
+'insert into clog (op, data) values ($1, $2)');
+
+insert into rtest values (1, 0, 'non-null');
+insert into rtest values (2, 0, NULL);
+update rtest set dat2 = 'non-null2' where id=1;
+update rtest set dat2 = NULL where id=1;
+update rtest set dat2 = 'new-nonnull' where id=2;
+
+delete from rtest where id=1;
+delete from rtest where id=2;
+
+select * from clog; delete from clog;
+
+
+
+
--- /dev/null
+
+#include <postgres.h>
+#include "funcapi.h"
+#include "mb/pg_wchar.h"
+#include "parser/keywords.h"
+
+#if 1
+#define talloc(len) malloc(len)
+#define trealloc(p, len) realloc(p, len)
+#define tfree(p) free(p)
+#else
+#define talloc(len) palloc(len)
+#define trealloc(p, len) repalloc(p, len)
+#define tfree(p) pfree(p)
+#endif
+
+#include "textbuf.h"
+
+struct TBuf {
+ text *data;
+ int size;
+};
+
+static void request_avail(TBuf *tbuf, int len)
+{
+ int newlen = tbuf->size;
+ int need = VARSIZE(tbuf->data) + len;
+ if (need < newlen)
+ return;
+ while (need > newlen)
+ newlen *= 2;
+ tbuf->data = trealloc(tbuf->data, newlen);
+ tbuf->size = newlen;
+}
+
+static inline char *get_endp(TBuf *tbuf)
+{
+ char *p = VARDATA(tbuf->data);
+ int len = VARSIZE(tbuf->data) - VARHDRSZ;
+ return p + len;
+}
+
+static inline void inc_used(TBuf *tbuf, int len)
+{
+ VARATT_SIZEP(tbuf->data) += len;
+}
+
+static void tbuf_init(TBuf *tbuf, int start_size)
+{
+ if (start_size < VARHDRSZ)
+ start_size = VARHDRSZ;
+ tbuf->data = talloc(start_size);
+ tbuf->size = start_size;
+ VARATT_SIZEP(tbuf->data) = VARHDRSZ;
+}
+
+TBuf *tbuf_alloc(int start_size)
+{
+ TBuf *res;
+ res = talloc(sizeof(TBuf));
+ tbuf_init(res, start_size);
+ return res;
+}
+
+void tbuf_free(TBuf *tbuf)
+{
+ if (tbuf->data)
+ tfree(tbuf->data);
+ tfree(tbuf);
+}
+
+int tbuf_get_size(TBuf *tbuf)
+{
+ return VARSIZE(tbuf->data) - VARHDRSZ;
+}
+
+void tbuf_reset(TBuf *tbuf)
+{
+ VARATT_SIZEP(tbuf->data) = VARHDRSZ;
+}
+
+const text *tbuf_look_text(TBuf *tbuf)
+{
+ return tbuf->data;
+}
+
+const char *tbuf_look_cstring(TBuf *tbuf)
+{
+ char *p;
+ request_avail(tbuf, 1);
+ p = get_endp(tbuf);
+ *p = 0;
+ return VARDATA(tbuf->data);
+}
+
+void tbuf_append_cstring(TBuf *tbuf, const char *str)
+{
+ int len = strlen(str);
+ request_avail(tbuf, len);
+ memcpy(get_endp(tbuf), str, len);
+ inc_used(tbuf, len);
+}
+
+void tbuf_append_text(TBuf *tbuf, const text *str)
+{
+ int len = VARSIZE(str) - VARHDRSZ;
+ request_avail(tbuf, len);
+ memcpy(get_endp(tbuf), VARDATA(str), len);
+ inc_used(tbuf, len);
+}
+
+void tbuf_append_char(TBuf *tbuf, char chr)
+{
+ char *p;
+ request_avail(tbuf, 1);
+ p = get_endp(tbuf);
+ *p = chr;
+ inc_used(tbuf, 1);
+}
+
+text *tbuf_steal_text(TBuf *tbuf)
+{
+ text *data = tbuf->data;
+ tbuf->data = NULL;
+ return data;
+}
+
+static const char b64tbl[] =
+"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+static int b64encode(char *dst, const uint8 *src, int srclen)
+{
+ char *p = dst;
+ const uint8 *s = src, *end = src + srclen;
+ int pos = 2;
+ uint32 buf = 0;
+
+ while (s < end) {
+ buf |= (unsigned char) *s << (pos << 3);
+ pos--;
+ s++;
+ /* write it out */
+ if (pos < 0) {
+ *p++ = b64tbl[ (buf >> 18) & 0x3f ];
+ *p++ = b64tbl[ (buf >> 12) & 0x3f ];
+ *p++ = b64tbl[ (buf >> 6) & 0x3f ];
+ *p++ = b64tbl[ buf & 0x3f ];
+ pos = 2;
+ buf = 0;
+ }
+ }
+ if (pos != 2) {
+ *p++ = b64tbl[ (buf >> 18) & 0x3f ];
+ *p++ = b64tbl[ (buf >> 12) & 0x3f ];
+ *p++ = (pos == 0) ? b64tbl[ (buf >> 6) & 0x3f ] : '=';
+ *p++ = '=';
+ }
+ return p - dst;
+}
+
+static const char hextbl[] = "0123456789abcdef";
+static int urlencode(char *dst, const uint8 *src, int srclen)
+{
+ const uint8 *end = src + srclen;
+ char *p = dst;
+ while (src < end) {
+ if (*src == '=')
+ *p++ = '+';
+ else if ((*src >= '0' && *src <= '9')
+ || (*src >= 'A' && *src <= 'Z')
+ || (*src >= 'a' && *src <= 'z'))
+ *p++ = *src;
+ else {
+ *p++ = '%';
+ *p++ = hextbl[*src >> 4];
+ *p++ = hextbl[*src & 15];
+ }
+ }
+ return p - dst;
+}
+
+static int quote_literal(char *dst, const uint8 *src, int srclen)
+{
+ const uint8 *cp1;
+ char *cp2;
+ int wl;
+
+ cp1 = src;
+ cp2 = dst;
+
+ *cp2++ = '\'';
+ while (srclen > 0)
+ {
+ if ((wl = pg_mblen(cp1)) != 1)
+ {
+ srclen -= wl;
+
+ while (wl-- > 0)
+ *cp2++ = *cp1++;
+ continue;
+ }
+
+ if (*cp1 == '\'')
+ *cp2++ = '\'';
+ if (*cp1 == '\\')
+ *cp2++ = '\\';
+ *cp2++ = *cp1++;
+ srclen--;
+ }
+
+ *cp2++ = '\'';
+
+ return cp2 - dst;
+}
+
+
+/*
+ * slon_quote_identifier - Quote an identifier only if needed
+ *
+ * When quotes are needed, we palloc the required space; slightly
+ * space-wasteful but well worth it for notational simplicity.
+ *
+ * Version: pgsql/src/backend/utils/adt/ruleutils.c,v 1.188 2005/01/13 17:19:10
+ */
+static int
+quote_ident(char *dst, const uint8 *src, int srclen)
+{
+ /*
+ * Can avoid quoting if ident starts with a lowercase letter or
+ * underscore and contains only lowercase letters, digits, and
+ * underscores, *and* is not any SQL keyword. Otherwise, supply
+ * quotes.
+ */
+ int nquotes = 0;
+ bool safe;
+ const char *ptr;
+ char *optr;
+ char ident[NAMEDATALEN + 1];
+
+ /* expect idents be not bigger than NAMEDATALEN */
+ if (srclen > NAMEDATALEN)
+ srclen = NAMEDATALEN;
+ memcpy(ident, src, srclen);
+ ident[srclen] = 0;
+
+ /*
+ * would like to use <ctype.h> macros here, but they might yield
+ * unwanted locale-specific results...
+ */
+ safe = ((ident[0] >= 'a' && ident[0] <= 'z') || ident[0] == '_');
+
+ for (ptr = ident; *ptr; ptr++)
+ {
+ char ch = *ptr;
+
+ if ((ch >= 'a' && ch <= 'z') ||
+ (ch >= '0' && ch <= '9') ||
+ (ch == '_'))
+ continue; /* okay */
+
+ safe = false;
+ if (ch == '"')
+ nquotes++;
+ }
+
+ if (safe)
+ {
+ /*
+ * Check for keyword. This test is overly strong, since many of
+ * the "keywords" known to the parser are usable as column names,
+ * but the parser doesn't provide any easy way to test for whether
+ * an identifier is safe or not... so be safe not sorry.
+ *
+ * Note: ScanKeywordLookup() does case-insensitive comparison, but
+ * that's fine, since we already know we have all-lower-case.
+ */
+ if (ScanKeywordLookup(ident) != NULL)
+ safe = false;
+ }
+
+ optr = dst;
+ if (!safe)
+ *optr++ = '"';
+
+ for (ptr = ident; *ptr; ptr++)
+ {
+ char ch = *ptr;
+
+ if (ch == '"')
+ *optr++ = '"';
+ *optr++ = ch;
+ }
+ if (!safe)
+ *optr++ = '"';
+
+ return optr - dst;
+}
+
+
+void tbuf_encode_cstring(TBuf *tbuf,
+ const char *str,
+ const char *encoding)
+{
+ if (str == NULL)
+ elog(ERROR, "tbuf_encode_cstring: NULL");
+ tbuf_encode_data(tbuf, (const uint8 *)str, strlen(str), encoding);
+}
+
+void tbuf_encode_data(TBuf *tbuf,
+ const uint8 *data, int len,
+ const char *encoding)
+{
+ int dlen = 0;
+ char *dst;
+ if (strcmp(encoding, "url") == 0) {
+ request_avail(tbuf, len*3);
+ dst = get_endp(tbuf);
+ dlen = urlencode(dst, data, len);
+ } else if (strcmp(encoding, "base64") == 0) {
+ request_avail(tbuf, (len + 2) * 4 / 3);
+ dst = get_endp(tbuf);
+ dlen = b64encode(dst, data, len);
+ } else if (strcmp(encoding, "quote_literal") == 0) {
+ request_avail(tbuf, len * 2 + 2);
+ dst = get_endp(tbuf);
+ dlen = quote_literal(dst, data, len);
+ } else if (strcmp(encoding, "quote_ident") == 0) {
+ request_avail(tbuf, len * 2 + 2);
+ dst = get_endp(tbuf);
+ dlen = quote_ident(dst, data, len);
+ } else
+ elog(ERROR, "bad encoding");
+ inc_used(tbuf, dlen);
+}
+
--- /dev/null
+struct TBuf;
+
+typedef struct TBuf TBuf;
+
+TBuf *tbuf_alloc(int start_size);
+void tbuf_free(TBuf *tbuf);
+int tbuf_get_size(TBuf *tbuf);
+void tbuf_reset(TBuf *tbuf);
+
+const text *tbuf_look_text(TBuf *tbuf);
+const char *tbuf_look_cstring(TBuf *tbuf);
+
+void tbuf_append_cstring(TBuf *tbuf, const char *str);
+void tbuf_append_text(TBuf *tbuf, const text *str);
+void tbuf_append_char(TBuf *tbuf, char chr);
+
+text *tbuf_steal_text(TBuf *tbuf);
+
+void tbuf_encode_cstring(TBuf *tbuf,
+ const char *str,
+ const char *encoding);
+
+void tbuf_encode_data(TBuf *tbuf,
+ const uint8 *data, int len,
+ const char *encoding);
+
--- /dev/null
+
+DATA_built = londiste.sql londiste.upgrade.sql
+DOCS = README.londiste
+
+FUNCS = $(wildcard functions/*.sql)
+SRCS = structure/tables.sql structure/types.sql $(FUNCS)
+
+REGRESS = londiste_install londiste_denytrigger londiste_provider londiste_subscriber
+REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql
+
+include ../../config.mak
+
+include $(PGXS)
+
+londiste.sql: $(SRCS)
+ cat $(SRCS) > $@
+
+londiste.upgrade.sql: $(FUNCS)
+ cat $(FUNCS) > $@
+
--- /dev/null
+
+londiste database backend
+--------------------------
+
+Provider side:
+--------------
+
+londiste.provider_table
+londiste.provider_seq
+
+
+Subscriber side
+---------------
+
+table londiste.completed
+table londiste.subscriber_table
+table londiste.subscriber_seq
+
+
+Open issues
+------------
+
+- notify behaviour
+- should notify-s given to db for processing?
+- link init functions
+- switchover
+- are set_last_tick()/get_last_tick() functions needed anymore?
+- typecheck for add_table()?
+
--- /dev/null
+create table denytest ( val integer);
+insert into denytest values (1);
+create trigger xdeny after insert or update or delete
+on denytest for each row execute procedure londiste.deny_trigger();
+insert into denytest values (2);
+ERROR: ('Changes no allowed on this table',)
+update denytest set val = 2;
+ERROR: ('Changes no allowed on this table',)
+delete from denytest;
+ERROR: ('Changes no allowed on this table',)
+select londiste.disable_deny_trigger(true);
+ disable_deny_trigger
+----------------------
+ t
+(1 row)
+
+update denytest set val = 2;
+select londiste.disable_deny_trigger(true);
+ disable_deny_trigger
+----------------------
+ t
+(1 row)
+
+update denytest set val = 2;
+select londiste.disable_deny_trigger(false);
+ disable_deny_trigger
+----------------------
+ f
+(1 row)
+
+update denytest set val = 2;
+ERROR: ('Changes no allowed on this table',)
+select londiste.disable_deny_trigger(false);
+ disable_deny_trigger
+----------------------
+ f
+(1 row)
+
+update denytest set val = 2;
+ERROR: ('Changes no allowed on this table',)
--- /dev/null
+\set ECHO off
--- /dev/null
+set client_min_messages = 'warning';
+--
+-- tables
+--
+create table testdata (
+ id serial primary key,
+ data text
+);
+create table testdata_nopk (
+ id serial,
+ data text
+);
+select londiste.provider_add_table('pqueue', 'public.testdata_nopk');
+ERROR: need key column
+CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return
+select londiste.provider_add_table('pqueue', 'public.testdata');
+ERROR: no such event queue
+CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return
+select pgq.create_queue('pqueue');
+ create_queue
+--------------
+ 1
+(1 row)
+
+select londiste.provider_add_table('pqueue', 'public.testdata');
+ provider_add_table
+--------------------
+ 1
+(1 row)
+
+select londiste.provider_add_table('pqueue', 'public.testdata');
+ERROR: duplicate key violates unique constraint "provider_table_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.provider_table (queue_name, table_name, trigger_name) values ( $1 , $2 , $3 )"
+PL/pgSQL function "provider_add_table" line 23 at SQL statement
+PL/pgSQL function "provider_add_table" line 2 at return
+select londiste.provider_refresh_trigger('pqueue', 'public.testdata');
+ provider_refresh_trigger
+--------------------------
+ 1
+(1 row)
+
+select * from londiste.provider_get_table_list('pqueue');
+ table_name | trigger_name
+-----------------+---------------
+ public.testdata | pqueue_logger
+(1 row)
+
+select londiste.provider_remove_table('pqueue', 'public.nonexist');
+ERROR: no such table registered
+select londiste.provider_remove_table('pqueue', 'public.testdata');
+ provider_remove_table
+-----------------------
+ 1
+(1 row)
+
+select * from londiste.provider_get_table_list('pqueue');
+ table_name | trigger_name
+------------+--------------
+(0 rows)
+
+--
+-- seqs
+--
+select * from londiste.provider_get_seq_list('pqueue');
+ provider_get_seq_list
+-----------------------
+(0 rows)
+
+select londiste.provider_add_seq('pqueue', 'public.no_seq');
+ERROR: seq not found
+CONTEXT: PL/pgSQL function "find_seq_oid" line 2 at return
+SQL statement "SELECT 1 from pg_class where oid = londiste.find_seq_oid( $1 )"
+PL/pgSQL function "provider_add_seq" line 10 at perform
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+ provider_add_seq
+------------------
+ 0
+(1 row)
+
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+ERROR: duplicate key violates unique constraint "provider_seq_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.provider_seq (queue_name, seq_name) values ( $1 , $2 )"
+PL/pgSQL function "provider_add_seq" line 16 at SQL statement
+select * from londiste.provider_get_seq_list('pqueue');
+ provider_get_seq_list
+------------------------
+ public.testdata_id_seq
+(1 row)
+
+select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq');
+ provider_remove_seq
+---------------------
+ 0
+(1 row)
+
+select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq');
+ERROR: seq not attached
+select * from londiste.provider_get_seq_list('pqueue');
+ provider_get_seq_list
+-----------------------
+(0 rows)
+
+--
+-- linked queue
+--
+select londiste.provider_add_table('pqueue', 'public.testdata');
+ provider_add_table
+--------------------
+ 1
+(1 row)
+
+insert into londiste.link (source, dest) values ('mqueue', 'pqueue');
+select londiste.provider_add_table('pqueue', 'public.testdata');
+ERROR: Linked queue, manipulation not allowed
+CONTEXT: PL/pgSQL function "provider_add_table" line 2 at return
+select londiste.provider_remove_table('pqueue', 'public.testdata');
+ERROR: Linked queue, manipulation not allowed
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+ERROR: Linked queue, cannot modify
+select londiste.provider_remove_seq('pqueue', 'public.testdata_seq');
+ERROR: Linked queue, cannot modify
+--
+-- cleanup
+--
+delete from londiste.link;
+drop table testdata;
+drop table testdata_nopk;
+delete from londiste.provider_seq;
+delete from londiste.provider_table;
+select pgq.drop_queue('pqueue');
+ drop_queue
+------------
+ 1
+(1 row)
+
--- /dev/null
+set client_min_messages = 'warning';
+create table testdata (
+ id serial primary key,
+ data text
+);
+--
+-- tables
+--
+select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk');
+ subscriber_add_table
+----------------------
+ 0
+(1 row)
+
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+ subscriber_add_table
+----------------------
+ 0
+(1 row)
+
+select pgq.create_queue('pqueue');
+ create_queue
+--------------
+ 1
+(1 row)
+
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+ERROR: duplicate key violates unique constraint "subscriber_table_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )"
+PL/pgSQL function "subscriber_add_table" line 2 at SQL statement
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+ERROR: duplicate key violates unique constraint "subscriber_table_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )"
+PL/pgSQL function "subscriber_add_table" line 2 at SQL statement
+select * from londiste.subscriber_get_table_list('pqueue');
+ table_name | merge_state | snapshot | trigger_name
+----------------------+-------------+----------+--------------
+ public.testdata_nopk | | |
+ public.testdata | | |
+(2 rows)
+
+select londiste.subscriber_remove_table('pqueue', 'public.nonexist');
+ERROR: no such table
+select londiste.subscriber_remove_table('pqueue', 'public.testdata');
+ subscriber_remove_table
+-------------------------
+ 0
+(1 row)
+
+select * from londiste.subscriber_get_table_list('pqueue');
+ table_name | merge_state | snapshot | trigger_name
+----------------------+-------------+----------+--------------
+ public.testdata_nopk | | |
+(1 row)
+
+--
+-- seqs
+--
+select * from londiste.subscriber_get_seq_list('pqueue');
+ subscriber_get_seq_list
+-------------------------
+(0 rows)
+
+select londiste.subscriber_add_seq('pqueue', 'public.no_seq');
+ subscriber_add_seq
+--------------------
+ 0
+(1 row)
+
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+ subscriber_add_seq
+--------------------
+ 0
+(1 row)
+
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+ERROR: duplicate key violates unique constraint "subscriber_seq_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.subscriber_seq (queue_name, seq_name) values ( $1 , $2 )"
+PL/pgSQL function "subscriber_add_seq" line 4 at SQL statement
+select * from londiste.subscriber_get_seq_list('pqueue');
+ subscriber_get_seq_list
+-------------------------
+ public.no_seq
+ public.testdata_id_seq
+(2 rows)
+
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq');
+ subscriber_remove_seq
+-----------------------
+ 0
+(1 row)
+
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq');
+ERROR: no such seq?
+select * from londiste.subscriber_get_seq_list('pqueue');
+ subscriber_get_seq_list
+-------------------------
+ public.no_seq
+(1 row)
+
+--
+-- linked queue
+--
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+ subscriber_add_table
+----------------------
+ 0
+(1 row)
+
+insert into londiste.link (source, dest) values ('mqueue', 'pqueue');
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+ERROR: duplicate key violates unique constraint "subscriber_table_pkey"
+CONTEXT: SQL statement "INSERT INTO londiste.subscriber_table (queue_name, table_name) values ( $1 , $2 )"
+PL/pgSQL function "subscriber_add_table" line 2 at SQL statement
+select londiste.subscriber_remove_table('pqueue', 'public.testdata');
+ subscriber_remove_table
+-------------------------
+ 0
+(1 row)
+
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+ subscriber_add_seq
+--------------------
+ 0
+(1 row)
+
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq');
+ERROR: no such seq?
--- /dev/null
+
+create or replace function londiste.deny_trigger()
+returns trigger as $$
+ if 'undeny' in GD:
+ return 'OK'
+ plpy.error('Changes no allowed on this table')
+$$ language plpythonu;
+
+create or replace function londiste.disable_deny_trigger(i_allow boolean)
+returns boolean as $$
+ if args[0]:
+ GD['undeny'] = 1
+ return True
+ else:
+ if 'undeny' in GD:
+ del GD['undeny']
+ return False
+$$ language plpythonu;
+
--- /dev/null
+create or replace function londiste.find_column_types(tbl text)
+returns text as $$
+declare
+ res text;
+ col record;
+ tbl_oid oid;
+begin
+ tbl_oid := londiste.find_table_oid(tbl);
+ res := '';
+ for col in
+ SELECT CASE WHEN k.attname IS NOT NULL THEN 'k' ELSE 'v' END AS type
+ FROM pg_attribute a LEFT JOIN (
+ SELECT k.attname FROM pg_index i, pg_attribute k
+ WHERE i.indrelid = tbl_oid AND k.attrelid = i.indexrelid
+ AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped
+ ) k ON (k.attname = a.attname)
+ WHERE a.attrelid = tbl_oid AND a.attnum > 0 AND NOT a.attisdropped
+ ORDER BY a.attnum
+ loop
+ res := res || col.type;
+ end loop;
+
+ return res;
+end;
+$$ language plpgsql;
+
--- /dev/null
+create or replace function londiste.find_rel_oid(tbl text, kind text)
+returns oid as $$
+declare
+ res oid;
+ pos integer;
+ schema text;
+ name text;
+begin
+ pos := position('.' in tbl);
+ if pos > 0 then
+ schema := substring(tbl for pos - 1);
+ name := substring(tbl from pos + 1);
+ else
+ schema := 'public';
+ name := tbl;
+ end if;
+ select c.oid into res
+ from pg_namespace n, pg_class c
+ where c.relnamespace = n.oid
+ and c.relkind = kind
+ and n.nspname = schema and c.relname = name;
+ if not found then
+ if kind = 'r' then
+ raise exception 'table not found';
+ elsif kind = 'S' then
+ raise exception 'seq not found';
+ else
+ raise exception 'weird relkind';
+ end if;
+ end if;
+
+ return res;
+end;
+$$ language plpgsql;
+
+create or replace function londiste.find_table_oid(tbl text)
+returns oid as $$
+begin
+ return londiste.find_rel_oid(tbl, 'r');
+end;
+$$ language plpgsql;
+
+create or replace function londiste.find_seq_oid(tbl text)
+returns oid as $$
+begin
+ return londiste.find_rel_oid(tbl, 'S');
+end;
+$$ language plpgsql;
+
--- /dev/null
+
+create or replace function londiste.get_last_tick(i_consumer text)
+returns bigint as $$
+declare
+ res bigint;
+begin
+ select last_tick_id into res
+ from londiste.completed
+ where consumer_id = i_consumer;
+ return res;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.link_source(i_dst_name text)
+returns text as $$
+declare
+ res text;
+begin
+ select source into res from londiste.link
+ where dest = i_dst_name;
+ return res;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.link_dest(i_source_name text)
+returns text as $$
+declare
+ res text;
+begin
+ select dest into res from londiste.link
+ where source = i_source_name;
+ return res;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.cmp_list(list1 text, a_queue text, a_table text, a_field text)
+returns boolean as $$
+declare
+ sql text;
+ tmp record;
+ list2 text;
+begin
+ sql := 'select ' || a_field || ' as name from ' || a_table
+ || ' where queue_name = ' || quote_literal(a_queue)
+ || ' order by 1';
+ list2 := '';
+ for tmp in execute sql loop
+ if list2 = '' then
+ list2 := tmp.name;
+ else
+ list2 := list2 || ',' || tmp.name;
+ end if;
+ end loop;
+ return list1 = list2;
+end;
+$$ language plpgsql;
+
+create or replace function londiste.link(i_source_name text, i_dest_name text, prov_tick_id bigint, prov_tbl_list text, prov_seq_list text)
+returns text as $$
+declare
+ tmp text;
+ list text;
+ tick_seq text;
+ external boolean;
+ last_tick bigint;
+begin
+ -- check if all matches
+ if not londiste.cmp_list(prov_tbl_list, i_source_name,
+ 'londiste.subscriber_table', 'table_name')
+ then
+ raise exception 'not all tables copied into subscriber';
+ end if;
+ if not londiste.cmp_list(prov_seq_list, i_source_name,
+ 'londiste.subscriber_seq', 'seq_name')
+ then
+ raise exception 'not all seqs copied into subscriber';
+ end if;
+ if not londiste.cmp_list(prov_seq_list, i_dest_name,
+ 'londiste.provider_table', 'table_name')
+ then
+ raise exception 'linked provider queue does not have all tables';
+ end if;
+ if not londiste.cmp_list(prov_seq_list, i_dest_name,
+ 'londiste.provider_seq', 'seq_name')
+ then
+ raise exception 'linked provider queue does not have all seqs';
+ end if;
+
+ -- check pgq
+ select queue_external_ticker, queue_tick_seq into external, tick_seq
+ from pgq.queue where queue_name = i_dest_name;
+ if not found then
+ raise exception 'dest queue does not exist';
+ end if;
+ if external then
+ raise exception 'dest queue has already external_ticker turned on?';
+ end if;
+
+ if nextval(tick_seq) >= prov_tick_id then
+ raise exception 'dest queue ticks larger';
+ end if;
+
+ update pgq.queue set queue_external_ticker = true
+ where queue_name = i_dest_name;
+
+ insert into londiste.link (source, dest) values (i_source_name, i_dest_name);
+
+ return null;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.link_del(i_source_name text, i_dest_name text)
+returns text as $$
+begin
+ delete from londiste.link
+ where source = i_source_name
+ and dest = i_dest_name;
+ if not found then
+ raise exception 'no suck link';
+ end if;
+ return null;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.provider_add_seq(
+ i_queue_name text, i_seq_name text)
+returns integer as $$
+declare
+ link text;
+begin
+ -- check if linked queue
+ link := londiste.link_source(i_queue_name);
+ if link is not null then
+ raise exception 'Linked queue, cannot modify';
+ end if;
+
+ perform 1 from pg_class
+ where oid = londiste.find_seq_oid(i_seq_name);
+ if not found then
+ raise exception 'seq not found';
+ end if;
+
+ insert into londiste.provider_seq (queue_name, seq_name)
+ values (i_queue_name, i_seq_name);
+ perform londiste.provider_notify_change(i_queue_name);
+
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function londiste.provider_add_table(
+ i_queue_name text,
+ i_table_name text,
+ i_col_types text
+) returns integer strict as $$
+declare
+ tgname text;
+ sql text;
+begin
+ if londiste.link_source(i_queue_name) is not null then
+ raise exception 'Linked queue, manipulation not allowed';
+ end if;
+
+ if position('k' in i_col_types) < 1 then
+ raise exception 'need key column';
+ end if;
+ if position('.' in i_table_name) < 1 then
+ raise exception 'need fully-qualified table name';
+ end if;
+ select queue_name into tgname
+ from pgq.queue where queue_name = i_queue_name;
+ if not found then
+ raise exception 'no such event queue';
+ end if;
+
+ tgname := i_queue_name || '_logger';
+ tgname := replace(lower(tgname), '.', '_');
+ insert into londiste.provider_table
+ (queue_name, table_name, trigger_name)
+ values (i_queue_name, i_table_name, tgname);
+
+ perform londiste.provider_create_trigger(
+ i_queue_name, i_table_name, i_col_types);
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.provider_add_table(
+ i_queue_name text,
+ i_table_name text
+) returns integer as $$
+begin
+ return londiste.provider_add_table(i_queue_name, i_table_name,
+ londiste.find_column_types(i_table_name));
+end;
+$$ language plpgsql;
+
--- /dev/null
+
+create or replace function londiste.provider_create_trigger(
+ i_queue_name text,
+ i_table_name text,
+ i_col_types text
+) returns integer strict as $$
+declare
+ tgname text;
+ sql text;
+begin
+ select trigger_name into tgname
+ from londiste.provider_table
+ where queue_name = i_queue_name
+ and table_name = i_table_name;
+ if not found then
+ raise exception 'table not found';
+ end if;
+
+ sql := 'select pgq.insert_event('
+ || quote_literal(i_queue_name)
+ || ', $1, $2, '
+ || quote_literal(i_table_name)
+ || ', NULL, NULL, NULL)';
+ execute 'create trigger ' || tgname
+ || ' after insert or update or delete on '
+ || i_table_name
+ || ' for each row execute procedure logtriga($arg1$'
+ || i_col_types || '$arg1$, $arg2$' || sql || '$arg2$)';
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.provider_get_seq_list(i_queue_name text)
+returns setof text as $$
+declare
+ rec record;
+begin
+ for rec in
+ select seq_name from londiste.provider_seq
+ where queue_name = i_queue_name
+ order by nr
+ loop
+ return next rec.seq_name;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.provider_get_table_list(i_queue text)
+returns setof londiste.ret_provider_table_list as $$
+declare
+ rec londiste.ret_provider_table_list%rowtype;
+begin
+ for rec in
+ select table_name, trigger_name
+ from londiste.provider_table
+ where queue_name = i_queue
+ order by nr
+ loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.provider_notify_change(i_queue_name text)
+returns integer as $$
+declare
+ res text;
+ tbl record;
+begin
+ res := '';
+ for tbl in
+ select table_name from londiste.provider_table
+ where queue_name = i_queue_name
+ order by nr
+ loop
+ if res = '' then
+ res := tbl.table_name;
+ else
+ res := res || ',' || tbl.table_name;
+ end if;
+ end loop;
+
+ perform pgq.insert_event(i_queue_name, 'T', res);
+
+ return 1;
+end;
+$$ language plpgsql;
+
--- /dev/null
+
+create or replace function londiste.provider_refresh_trigger(
+ i_queue_name text,
+ i_table_name text,
+ i_col_types text
+) returns integer strict as $$
+declare
+ t_name text;
+ tbl_oid oid;
+begin
+ select trigger_name into t_name
+ from londiste.provider_table
+ where queue_name = i_queue_name
+ and table_name = i_table_name;
+ if not found then
+ raise exception 'table not found';
+ end if;
+
+ tbl_oid := londiste.find_table_oid(i_table_name);
+ perform 1 from pg_trigger
+ where tgrelid = tbl_oid
+ and tgname = t_name;
+ if found then
+ execute 'drop trigger ' || t_name || ' on ' || i_table_name;
+ end if;
+
+ perform londiste.provider_create_trigger(i_queue_name, i_table_name, i_col_types);
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.provider_refresh_trigger(
+ i_queue_name text,
+ i_table_name text
+) returns integer strict as $$
+begin
+ return londiste.provider_refresh_trigger(i_queue_name, i_table_name,
+ londiste.find_column_types(i_table_name));
+end;
+$$ language plpgsql security definer;
+
+
+
--- /dev/null
+
+create or replace function londiste.provider_remove_seq(
+ i_queue_name text, i_seq_name text)
+returns integer as $$
+declare
+ link text;
+begin
+ -- check if linked queue
+ link := londiste.link_source(i_queue_name);
+ if link is not null then
+ raise exception 'Linked queue, cannot modify';
+ end if;
+
+ delete from londiste.provider_seq
+ where queue_name = i_queue_name
+ and seq_name = i_seq_name;
+ if not found then
+ raise exception 'seq not attached';
+ end if;
+
+ perform londiste.provider_notify_change(i_queue_name);
+
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.provider_remove_table(
+ i_queue_name text,
+ i_table_name text
+) returns integer as $$
+declare
+ tgname text;
+begin
+ if londiste.link_source(i_queue_name) is not null then
+ raise exception 'Linked queue, manipulation not allowed';
+ end if;
+
+ select trigger_name into tgname from londiste.provider_table
+ where queue_name = i_queue_name
+ and table_name = i_table_name;
+ if not found then
+ raise exception 'no such table registered';
+ end if;
+
+ execute 'drop trigger ' || tgname || ' on ' || i_table_name;
+
+ delete from londiste.provider_table
+ where queue_name = i_queue_name
+ and table_name = i_table_name;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+
--- /dev/null
+
+create or replace function londiste.set_last_tick(
+ i_consumer text,
+ i_tick_id bigint)
+returns integer as $$
+begin
+ update londiste.completed
+ set last_tick_id = i_tick_id
+ where consumer_id = i_consumer;
+ if not found then
+ insert into londiste.completed (consumer_id, last_tick_id)
+ values (i_consumer, i_tick_id);
+ end if;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_add_seq(
+ i_queue_name text, i_seq_name text)
+returns integer as $$
+declare
+ link text;
+begin
+ insert into londiste.subscriber_seq (queue_name, seq_name)
+ values (i_queue_name, i_seq_name);
+
+ -- update linked queue if needed
+ link := londiste.link_dest(i_queue_name);
+ if link is not null then
+ insert into londiste.provider_seq
+ (queue_name, seq_name)
+ values (link, i_seq_name);
+ perform londiste.provider_notify_change(link);
+ end if;
+
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_add_table(
+ i_queue_name text, i_table text)
+returns integer as $$
+begin
+ insert into londiste.subscriber_table (queue_name, table_name)
+ values (i_queue_name, i_table);
+
+ -- linked queue is updated, when the table is copied
+
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_get_seq_list(i_queue_name text)
+returns setof text as $$
+declare
+ rec record;
+begin
+ for rec in
+ select seq_name from londiste.subscriber_seq
+ where queue_name = i_queue_name
+ order by nr
+ loop
+ return next rec.seq_name;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_get_table_list(i_queue_name text)
+returns setof londiste.ret_subscriber_table as $$
+declare
+ rec londiste.ret_subscriber_table%rowtype;
+begin
+ for rec in
+ select table_name, merge_state, snapshot, trigger_name
+ from londiste.subscriber_table
+ where queue_name = i_queue_name
+ order by nr
+ loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+-- compat
+create or replace function londiste.get_table_state(i_queue text)
+returns setof londiste.subscriber_table as $$
+declare
+ rec londiste.subscriber_table%rowtype;
+begin
+ for rec in
+ select * from londiste.subscriber_table
+ where queue_name = i_queue
+ order by nr
+ loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_remove_seq(
+ i_queue_name text, i_seq_name text)
+returns integer as $$
+declare
+ link text;
+begin
+ delete from londiste.subscriber_seq
+ where queue_name = i_queue_name
+ and seq_name = i_seq_name;
+ if not found then
+ raise exception 'no such seq?';
+ end if;
+
+ -- update linked queue if needed
+ link := londiste.link_dest(i_queue_name);
+ if link is not null then
+ delete from londiste.provider_seq
+ where queue_name = link
+ and seq_name = i_seq_name;
+ perform londiste.provider_notify_change(link);
+ end if;
+
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function londiste.subscriber_remove_table(
+ i_queue_name text, i_table text)
+returns integer as $$
+declare
+ link text;
+begin
+ delete from londiste.subscriber_table
+ where queue_name = i_queue_name
+ and table_name = i_table;
+ if not found then
+ raise exception 'no such table';
+ end if;
+
+ -- sync link
+ link := londiste.link_dest(i_queue_name);
+ if link is not null then
+ delete from londiste.provider_table
+ where queue_name = link
+ and table_name = i_table;
+ perform londiste.provider_notify_change(link);
+ end if;
+
+ return 0;
+end;
+$$ language plpgsql;
+
--- /dev/null
+
+create or replace function londiste.subscriber_set_table_state(
+ i_queue_name text,
+ i_table_name text,
+ i_snapshot text,
+ i_merge_state text)
+returns integer as $$
+declare
+ link text;
+ ok integer;
+begin
+ update londiste.subscriber_table
+ set snapshot = i_snapshot,
+ merge_state = i_merge_state
+ where queue_name = i_queue_name
+ and table_name = i_table_name;
+ if not found then
+ raise exception 'no such table';
+ end if;
+
+ -- sync link state also
+ link := londiste.link_dest(i_queue_name);
+ if link then
+ select * from londiste.provider_table
+ where queue_name = linkdst
+ and table_name = i_table_name;
+ if found then
+ if i_merge_state is null or i_merge_state <> 'ok' then
+ delete from londiste.provider_table
+ where queue_name = link
+ and table_name = i_table_name;
+ perform londiste.notify_change(link);
+ end if;
+ else
+ if i_merge_state = 'ok' then
+ insert into londiste.provider_table (queue_name, table_name)
+ values (link, i_table_name);
+ perform londiste.notify_change(link);
+ end if;
+ end if;
+ end if;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+create or replace function londiste.set_table_state(
+ i_queue_name text,
+ i_table_name text,
+ i_snapshot text,
+ i_merge_state text)
+returns integer as $$
+begin
+ return londiste.subscriber_set_table_state(i_queue_name, i_table_name, i_snapshot, i_merge_state);
+end;
+$$ language plpgsql security definer;
+
+
--- /dev/null
+
+create table denytest ( val integer);
+insert into denytest values (1);
+create trigger xdeny after insert or update or delete
+on denytest for each row execute procedure londiste.deny_trigger();
+
+insert into denytest values (2);
+update denytest set val = 2;
+delete from denytest;
+
+select londiste.disable_deny_trigger(true);
+update denytest set val = 2;
+select londiste.disable_deny_trigger(true);
+update denytest set val = 2;
+select londiste.disable_deny_trigger(false);
+update denytest set val = 2;
+select londiste.disable_deny_trigger(false);
+update denytest set val = 2;
+
--- /dev/null
+\set ECHO off
+set log_error_verbosity = 'terse';
+\i ../txid/txid.sql
+\i ../pgq/pgq.sql
+\i ../logtriga/logtriga.sql
+\i londiste.sql
+\set ECHO all
+
--- /dev/null
+
+set client_min_messages = 'warning';
+
+--
+-- tables
+--
+create table testdata (
+ id serial primary key,
+ data text
+);
+create table testdata_nopk (
+ id serial,
+ data text
+);
+
+select londiste.provider_add_table('pqueue', 'public.testdata_nopk');
+select londiste.provider_add_table('pqueue', 'public.testdata');
+
+select pgq.create_queue('pqueue');
+select londiste.provider_add_table('pqueue', 'public.testdata');
+select londiste.provider_add_table('pqueue', 'public.testdata');
+
+select londiste.provider_refresh_trigger('pqueue', 'public.testdata');
+
+select * from londiste.provider_get_table_list('pqueue');
+
+select londiste.provider_remove_table('pqueue', 'public.nonexist');
+select londiste.provider_remove_table('pqueue', 'public.testdata');
+
+select * from londiste.provider_get_table_list('pqueue');
+
+--
+-- seqs
+--
+
+select * from londiste.provider_get_seq_list('pqueue');
+select londiste.provider_add_seq('pqueue', 'public.no_seq');
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+select * from londiste.provider_get_seq_list('pqueue');
+select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq');
+select londiste.provider_remove_seq('pqueue', 'public.testdata_id_seq');
+select * from londiste.provider_get_seq_list('pqueue');
+
+--
+-- linked queue
+--
+select londiste.provider_add_table('pqueue', 'public.testdata');
+insert into londiste.link (source, dest) values ('mqueue', 'pqueue');
+
+
+select londiste.provider_add_table('pqueue', 'public.testdata');
+select londiste.provider_remove_table('pqueue', 'public.testdata');
+
+select londiste.provider_add_seq('pqueue', 'public.testdata_id_seq');
+select londiste.provider_remove_seq('pqueue', 'public.testdata_seq');
+
+--
+-- cleanup
+--
+
+delete from londiste.link;
+drop table testdata;
+drop table testdata_nopk;
+delete from londiste.provider_seq;
+delete from londiste.provider_table;
+select pgq.drop_queue('pqueue');
+
--- /dev/null
+
+set client_min_messages = 'warning';
+
+create table testdata (
+ id serial primary key,
+ data text
+);
+
+--
+-- tables
+--
+
+select londiste.subscriber_add_table('pqueue', 'public.testdata_nopk');
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+
+select pgq.create_queue('pqueue');
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+
+select * from londiste.subscriber_get_table_list('pqueue');
+
+select londiste.subscriber_remove_table('pqueue', 'public.nonexist');
+select londiste.subscriber_remove_table('pqueue', 'public.testdata');
+
+select * from londiste.subscriber_get_table_list('pqueue');
+
+--
+-- seqs
+--
+
+select * from londiste.subscriber_get_seq_list('pqueue');
+select londiste.subscriber_add_seq('pqueue', 'public.no_seq');
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+select * from londiste.subscriber_get_seq_list('pqueue');
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq');
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_id_seq');
+select * from londiste.subscriber_get_seq_list('pqueue');
+
+--
+-- linked queue
+--
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+insert into londiste.link (source, dest) values ('mqueue', 'pqueue');
+
+
+select londiste.subscriber_add_table('pqueue', 'public.testdata');
+select londiste.subscriber_remove_table('pqueue', 'public.testdata');
+
+select londiste.subscriber_add_seq('pqueue', 'public.testdata_id_seq');
+select londiste.subscriber_remove_seq('pqueue', 'public.testdata_seq');
+
+
--- /dev/null
+set default_with_oids = 'off';
+
+create schema londiste;
+grant usage on schema londiste to public;
+
+create table londiste.provider_table (
+ nr serial not null,
+ queue_name text not null,
+ table_name text not null,
+ trigger_name text,
+ primary key (queue_name, table_name)
+);
+
+create table londiste.provider_seq (
+ nr serial not null,
+ queue_name text not null,
+ seq_name text not null,
+ primary key (queue_name, seq_name)
+);
+
+create table londiste.completed (
+ consumer_id text not null,
+ last_tick_id bigint not null,
+
+ primary key (consumer_id)
+);
+
+create table londiste.link (
+ source text not null,
+ dest text not null,
+ primary key (source),
+ unique (dest)
+);
+
+create table londiste.subscriber_table (
+ nr serial not null,
+ queue_name text not null,
+ table_name text not null,
+ snapshot text,
+ merge_state text,
+ trigger_name text,
+
+ primary key (queue_name, table_name)
+);
+
+create table londiste.subscriber_seq (
+ nr serial not null,
+ queue_name text not null,
+ seq_name text not null,
+
+ primary key (queue_name, seq_name)
+);
+
--- /dev/null
+
+create type londiste.ret_provider_table_list as (
+ table_name text,
+ trigger_name text
+);
+
+create type londiste.ret_subscriber_table as (
+ table_name text,
+ merge_state text,
+ snapshot text,
+ trigger_name text
+);
+
--- /dev/null
+
+DOCS = README.pgq
+DATA_built = pgq.sql
+DATA = structure/uninstall_pgq.sql
+
+SRCS = $(wildcard structure/*.sql) \
+ $(wildcard functions/*.sql) \
+ $(wildcard triggers/*.sql)
+
+REGRESS = pgq_init logutriga sqltriga
+REGRESS_OPTS = --load-language=plpythonu --load-language=plpgsql
+
+PGXS = $(shell pg_config --pgxs)
+include $(PGXS)
+
+NDOC = NaturalDocs
+NDOCARGS = -r -o html docs/html -p docs -i docs/sql
+CATSQL = ../../scripts/catsql.py
+
+pgq.sql: $(SRCS)
+ $(CATSQL) structure/install.sql > $@
+
+dox: cleandox
+ mkdir -p docs/html
+ mkdir -p docs/sql
+ $(CATSQL) --ndoc structure/tables.sql structure/types.sql > docs/sql/schema.sql
+ $(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql
+ $(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql
+ $(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql
+ $(NDOC) $(NDOCARGS)
+
+cleandox:
+ rm -rf docs/html docs/Data docs/sql
+
+clean: cleandox
+
+test:
+ #-dropdb pgq
+ #createdb pgq
+ #psql -f structure/pgq.sql pgq
+ make installcheck || { less regression.diffs; exit 1; }
+
+upload: dox
+ rsync -az docs/html structure functions data1:public_html/pgq/
+ make cleandox
+ rsync -az catsql.py Makefile docs data1:public_html/pgq/
+ make dox
+
--- /dev/null
+
+Schema overview
+===============
+
+pgq.consumer consumer name <> id mapping
+pgq.queue queue information
+pgq.subscription consumer registrations
+pgq.tick snapshots that group events into batches
+pgq.retry_queue events to be retried
+pgq.failed_queue events that have failed
+pgq.event_* data tables
+
+Random ideas
+============
+
+- all ticker logic in DB (plpython)
+- more funcs in plpython
+- insert_event in C (way to get rid of plpython)
+
--- /dev/null
+Format: 1.35
+
+# This is the Natural Docs languages file for this project. If you change
+# anything here, it will apply to THIS PROJECT ONLY. If you'd like to change
+# something for all your projects, edit the Languages.txt in Natural Docs'
+# Config directory instead.
+
+
+# You can prevent certain file extensions from being scanned like this:
+# Ignore Extensions: [extension] [extension] ...
+
+
+#-------------------------------------------------------------------------------
+# SYNTAX:
+#
+# Unlike other Natural Docs configuration files, in this file all comments
+# MUST be alone on a line. Some languages deal with the # character, so you
+# cannot put comments on the same line as content.
+#
+# Also, all lists are separated with spaces, not commas, again because some
+# languages may need to use them.
+#
+# Language: [name]
+# Alter Language: [name]
+# Defines a new language or alters an existing one. Its name can use any
+# characters. If any of the properties below have an add/replace form, you
+# must use that when using Alter Language.
+#
+# The language Shebang Script is special. It's entry is only used for
+# extensions, and files with those extensions have their shebang (#!) lines
+# read to determine the real language of the file. Extensionless files are
+# always treated this way.
+#
+# The language Text File is also special. It's treated as one big comment
+# so you can put Natural Docs content in them without special symbols. Also,
+# if you don't specify a package separator, ignored prefixes, or enum value
+# behavior, it will copy those settings from the language that is used most
+# in the source tree.
+#
+# Extensions: [extension] [extension] ...
+# [Add/Replace] Extensions: [extension] [extension] ...
+# Defines the file extensions of the language's source files. You can
+# redefine extensions found in the main languages file. You can use * to
+# mean any undefined extension.
+#
+# Shebang Strings: [string] [string] ...
+# [Add/Replace] Shebang Strings: [string] [string] ...
+# Defines a list of strings that can appear in the shebang (#!) line to
+# designate that it's part of the language. You can redefine strings found
+# in the main languages file.
+#
+# Ignore Prefixes in Index: [prefix] [prefix] ...
+# [Add/Replace] Ignored Prefixes in Index: [prefix] [prefix] ...
+#
+# Ignore [Topic Type] Prefixes in Index: [prefix] [prefix] ...
+# [Add/Replace] Ignored [Topic Type] Prefixes in Index: [prefix] [prefix] ...
+# Specifies prefixes that should be ignored when sorting symbols in an
+# index. Can be specified in general or for a specific topic type.
+#
+#------------------------------------------------------------------------------
+# For basic language support only:
+#
+# Line Comments: [symbol] [symbol] ...
+# Defines a space-separated list of symbols that are used for line comments,
+# if any.
+#
+# Block Comments: [opening sym] [closing sym] [opening sym] [closing sym] ...
+# Defines a space-separated list of symbol pairs that are used for block
+# comments, if any.
+#
+# Package Separator: [symbol]
+# Defines the default package separator symbol. The default is a dot.
+#
+# [Topic Type] Prototype Enders: [symbol] [symbol] ...
+# When defined, Natural Docs will attempt to get a prototype from the code
+# immediately following the topic type. It stops when it reaches one of
+# these symbols. Use \n for line breaks.
+#
+# Line Extender: [symbol]
+# Defines the symbol that allows a prototype to span multiple lines if
+# normally a line break would end it.
+#
+# Enum Values: [global|under type|under parent]
+# Defines how enum values are referenced. The default is global.
+# global - Values are always global, referenced as 'value'.
+# under type - Values are under the enum type, referenced as
+# 'package.enum.value'.
+# under parent - Values are under the enum's parent, referenced as
+# 'package.value'.
+#
+# Perl Package: [perl package]
+# Specifies the Perl package used to fine-tune the language behavior in ways
+# too complex to do in this file.
+#
+#------------------------------------------------------------------------------
+# For full language support only:
+#
+# Full Language Support: [perl package]
+# Specifies the Perl package that has the parsing routines necessary for full
+# language support.
+#
+#-------------------------------------------------------------------------------
+
+# The following languages are defined in the main file, if you'd like to alter
+# them:
+#
+# Text File, Shebang Script, C/C++, C#, Java, JavaScript, Perl, Python,
+# PHP, SQL, Visual Basic, Pascal, Assembly, Ada, Tcl, Ruby, Makefile,
+# ActionScript, ColdFusion, R, Fortran
+
+# If you add a language that you think would be useful to other developers
+# and should be included in Natural Docs by default, please e-mail it to
+# languages [at] naturaldocs [dot] org.
--- /dev/null
+Format: 1.35
+
+
+Title: PgQ
+SubTitle: Database API
+
+# You can add a footer to your documentation like this:
+# Footer: [text]
+# If you want to add a copyright notice, this would be the place to do it.
+
+
+# --------------------------------------------------------------------------
+#
+# Cut and paste the lines below to change the order in which your files
+# appear on the menu. Don't worry about adding or removing files, Natural
+# Docs will take care of that.
+#
+# You can further organize the menu by grouping the entries. Add a
+# "Group: [name] {" line to start a group, and add a "}" to end it.
+#
+# You can add text and web links to the menu by adding "Text: [text]" and
+# "Link: [name] ([URL])" lines, respectively.
+#
+# The formatting and comments are auto-generated, so don't worry about
+# neatness when editing the file. Natural Docs will clean it up the next
+# time it is run. When working with groups, just deal with the braces and
+# forget about the indentation and comments.
+#
+# --------------------------------------------------------------------------
+
+
+File: Public Functions (external.sql)
+File: Public Triggers (triggers.sql)
+File: Internal Functions (internal.sql)
+File: Internal Tables (schema.sql)
+
+Group: Index {
+
+ Index: Everything
+ Database Table Index: Database Tables
+ Function Index: Functions
+ } # Group: Index
+
--- /dev/null
+Format: 1.35
+
+# This is the Natural Docs topics file for this project. If you change anything
+# here, it will apply to THIS PROJECT ONLY. If you'd like to change something
+# for all your projects, edit the Topics.txt in Natural Docs' Config directory
+# instead.
+
+
+# If you'd like to prevent keywords from being recognized by Natural Docs, you
+# can do it like this:
+# Ignore Keywords: [keyword], [keyword], ...
+#
+# Or you can use the list syntax like how they are defined:
+# Ignore Keywords:
+# [keyword]
+# [keyword], [plural keyword]
+# ...
+
+
+#-------------------------------------------------------------------------------
+# SYNTAX:
+#
+# Topic Type: [name]
+# Alter Topic Type: [name]
+# Creates a new topic type or alters one from the main file. Each type gets
+# its own index and behavior settings. Its name can have letters, numbers,
+# spaces, and these charaters: - / . '
+#
+# Plural: [name]
+# Sets the plural name of the topic type, if different.
+#
+# Keywords:
+# [keyword]
+# [keyword], [plural keyword]
+# ...
+# Defines or adds to the list of keywords for the topic type. They may only
+# contain letters, numbers, and spaces and are not case sensitive. Plural
+# keywords are used for list topics. You can redefine keywords found in the
+# main topics file.
+#
+# Index: [yes|no]
+# Whether the topics get their own index. Defaults to yes. Everything is
+# included in the general index regardless of this setting.
+#
+# Scope: [normal|start|end|always global]
+# How the topics affects scope. Defaults to normal.
+# normal - Topics stay within the current scope.
+# start - Topics start a new scope for all the topics beneath it,
+# like class topics.
+# end - Topics reset the scope back to global for all the topics
+# beneath it.
+# always global - Topics are defined as global, but do not change the scope
+# for any other topics.
+#
+# Class Hierarchy: [yes|no]
+# Whether the topics are part of the class hierarchy. Defaults to no.
+#
+# Variable Type: [yes|no]
+# Whether the topics can be a variable type. Defaults to no.
+#
+# Page Title If First: [yes|no]
+# Whether the topic's title becomes the page title if it's the first one in
+# a file. Defaults to no.
+#
+# Break Lists: [yes|no]
+# Whether list topics should be broken into individual topics in the output.
+# Defaults to no.
+#
+# Can Group With: [type], [type], ...
+# Defines a list of topic types that this one can possibly be grouped with.
+# Defaults to none.
+#-------------------------------------------------------------------------------
+
+# The following topics are defined in the main file, if you'd like to alter
+# their behavior or add keywords:
+#
+# Generic, Class, Interface, Section, File, Group, Function, Variable,
+# Property, Type, Constant, Enumeration, Event, Delegate, Macro,
+# Database, Database Table, Database View, Database Index, Database
+# Cursor, Database Trigger, Cookie, Build Target
+
+# If you add something that you think would be useful to other developers
+# and should be included in Natural Docs by default, please e-mail it to
+# topics [at] naturaldocs [dot] org.
+
+
+Topic Type: Schema
+
+ Plural: Schemas
+ Index: No
+ Scope: Start
+ Class Hierarchy: Yes
+
+ Keywords:
+ schema, schemas
+
+
+Alter Topic Type: Function
+
+ Add Keywords:
+ public function
+ internal function
+
+
+Alter Topic Type: File
+
+ Index: No
--- /dev/null
+create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text)
+returns bigint as $$
+begin
+ raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1;
+ return 1;
+end;
+$$ language plpgsql;
+create table udata (
+ id serial primary key,
+ txt text,
+ bin bytea
+);
+NOTICE: CREATE TABLE will create implicit sequence "udata_id_seq" for serial column "udata.id"
+NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "udata_pkey" for table "udata"
+create trigger utest AFTER insert or update or delete ON udata
+for each row execute procedure pgq.logutriga('udata_que');
+insert into udata (txt) values ('text1');
+ERROR: plpython: function "logutriga" failed
+DETAIL: exceptions.NameError: global name 'rowdata' is not defined
+insert into udata (bin) values (E'bi\tn\\000bin');
+ERROR: plpython: function "logutriga" failed
+DETAIL: exceptions.NameError: global name 'rowdata' is not defined
--- /dev/null
+\set ECHO none
+select * from pgq.maint_tables_to_vacuum();
+ maint_tables_to_vacuum
+------------------------
+ pgq.subscription
+ pgq.queue
+ pgq.tick
+ pgq.retry_queue
+(4 rows)
+
+select * from pgq.maint_retry_events();
+ maint_retry_events
+--------------------
+ 0
+(1 row)
+
+select pgq.create_queue('tmpqueue');
+ create_queue
+--------------
+ 1
+(1 row)
+
+select pgq.register_consumer('tmpqueue', 'consumer');
+ register_consumer
+-------------------
+ 1
+(1 row)
+
+select pgq.unregister_consumer('tmpqueue', 'consumer');
+ unregister_consumer
+---------------------
+ 1
+(1 row)
+
+select pgq.drop_queue('tmpqueue');
+ drop_queue
+------------
+ 1
+(1 row)
+
+select pgq.create_queue('myqueue');
+ create_queue
+--------------
+ 1
+(1 row)
+
+select pgq.register_consumer('myqueue', 'consumer');
+ register_consumer
+-------------------
+ 1
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+
+(1 row)
+
+select pgq.ticker();
+ ticker
+--------
+ 1
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+ 1
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+ 1
+(1 row)
+
+select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1);
+ queue_name | consumer_name | prev_tick_id | tick_id | lag
+------------+---------------+--------------+---------+-------------
+ myqueue | consumer | 1 | 2 | @ 0.00 secs
+(1 row)
+
+select queue_name from pgq.get_queue_info() limit 0;
+ queue_name
+------------
+(0 rows)
+
+select queue_name, consumer_name from pgq.get_consumer_info() limit 0;
+ queue_name | consumer_name
+------------+---------------
+(0 rows)
+
+select pgq.finish_batch(1);
+ finish_batch
+--------------
+ 1
+(1 row)
+
+select pgq.finish_batch(1);
+WARNING: finish_batch: batch 1 not found
+ finish_batch
+--------------
+ 0
+(1 row)
+
+select pgq.ticker();
+ ticker
+--------
+ 1
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+ 2
+(1 row)
+
+select * from pgq.batch_event_tables(2);
+ batch_event_tables
+--------------------
+ pgq.event_2_0
+(1 row)
+
+select * from pgq.get_batch_events(2);
+ ev_id | ev_time | ev_txid | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4
+-------+---------+---------+----------+---------+---------+-----------+-----------+-----------+-----------
+(0 rows)
+
+select pgq.finish_batch(2);
+ finish_batch
+--------------
+ 1
+(1 row)
+
+select pgq.insert_event('myqueue', 'r1', 'data');
+ insert_event
+--------------
+ 1
+(1 row)
+
+select pgq.insert_event('myqueue', 'r2', 'data');
+ insert_event
+--------------
+ 2
+(1 row)
+
+select pgq.insert_event('myqueue', 'r3', 'data');
+ insert_event
+--------------
+ 3
+(1 row)
+
+select pgq.current_event_table('myqueue');
+ current_event_table
+---------------------
+ pgq.event_2_0
+(1 row)
+
+select pgq.ticker();
+ ticker
+--------
+ 1
+(1 row)
+
+select pgq.next_batch('myqueue', 'consumer');
+ next_batch
+------------
+ 3
+(1 row)
+
+select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3);
+ ev_id | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4
+-------+----------+---------+---------+-----------+-----------+-----------+-----------
+ 1 | | r1 | data | | | |
+ 2 | | r2 | data | | | |
+ 3 | | r3 | data | | | |
+(3 rows)
+
+select * from pgq.failed_event_list('myqueue', 'consumer');
+ ev_failed_reason | ev_failed_time | ev_id | ev_time | ev_txid | ev_owner | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4
+------------------+----------------+-------+---------+---------+----------+----------+---------+---------+-----------+-----------+-----------+-----------
+(0 rows)
+
+select pgq.event_failed(3, 1, 'failure test');
+ event_failed
+--------------
+ 1
+(1 row)
+
+select pgq.event_failed(3, 1, 'failure test');
+ event_failed
+--------------
+ 0
+(1 row)
+
+select pgq.event_retry(3, 2, 0);
+ event_retry
+-------------
+ 1
+(1 row)
+
+select pgq.event_retry(3, 2, 0);
+ event_retry
+-------------
+ 0
+(1 row)
+
+select pgq.finish_batch(3);
+ finish_batch
+--------------
+ 1
+(1 row)
+
+select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
+ from pgq.failed_event_list('myqueue', 'consumer');
+ ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data
+------------------+-------+---------+----------+---------+---------
+ failure test | 1 | | 0 | r1 | data
+(1 row)
+
+select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
+ from pgq.failed_event_list('myqueue', 'consumer', 0, 1);
+ ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data
+------------------+-------+---------+----------+---------+---------
+(0 rows)
+
+select * from pgq.failed_event_count('myqueue', 'consumer');
+ failed_event_count
+--------------------
+ 1
+(1 row)
+
+select * from pgq.failed_event_delete('myqueue', 'consumer', 0);
+ERROR: event not found
+select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0,
+ 'rawtest', 'data', null, null, null, null);
+ event_retry_raw
+-----------------
+ 666
+(1 row)
+
+select pgq.ticker();
+ ticker
+--------
+ 1
+(1 row)
+
--- /dev/null
+-- start testing
+create table rtest (
+ id integer primary key,
+ dat text
+);
+NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "rtest_pkey" for table "rtest"
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que');
+-- simple test
+insert into rtest values (1, 'value1');
+NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 'value2';
+NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest;
+NOTICE: insert_event(que, D, id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+-- test new fields
+alter table rtest add column dat2 text;
+insert into rtest values (1, 'value1');
+NOTICE: insert_event(que, I, (dat,id) values ('value1','1'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 'value2';
+NOTICE: insert_event(que, U, dat='value2' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest;
+NOTICE: insert_event(que, D, id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+-- test field ignore
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2');
+insert into rtest values (1, '666', 'newdat');
+NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 5, dat2 = 'newdat2';
+NOTICE: insert_event(que2, U, dat='5' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 6;
+NOTICE: insert_event(que2, U, dat='6' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest;
+NOTICE: insert_event(que2, D, id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+-- test hashed pkey
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)');
+insert into rtest values (1, '666', 'newdat');
+NOTICE: insert_event(que2, I, (dat,id) values ('666','1'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 5, dat2 = 'newdat2';
+NOTICE: insert_event(que2, U, dat='5' where dat='5' and hashtext(dat) = hashtext('5'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat = 6;
+NOTICE: insert_event(que2, U, dat='6' where dat='6' and hashtext(dat) = hashtext('6'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest;
+NOTICE: insert_event(que2, D, dat='6' and hashtext(dat) = hashtext('6'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+-- test wrong key
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que3');
+insert into rtest values (1, 0, 'non-null');
+NOTICE: insert_event(que3, I, (dat,id) values ('0','1'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+insert into rtest values (2, 0, NULL);
+NOTICE: insert_event(que3, I, (dat,id) values ('0','2'), public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat2 = 'non-null2' where id=1;
+NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat2 = NULL where id=1;
+NOTICE: insert_event(que3, U, id='1' where id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+update rtest set dat2 = 'new-nonnull' where id=2;
+NOTICE: insert_event(que3, U, id='2' where id='2', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest where id=1;
+NOTICE: insert_event(que3, D, id='1', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+delete from rtest where id=2;
+NOTICE: insert_event(que3, D, id='2', public.rtest)
+CONTEXT: SQL statement "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
--- /dev/null
+create or replace function pgq.batch_event_sql(x_batch_id bigint)
+returns text as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.batch_event_sql(1)
+-- Creates SELECT statement that fetches events for this batch.
+--
+-- Parameters:
+-- x_batch_id - ID of a active batch.
+--
+-- Returns:
+-- SQL statement
+-- ----------------------------------------------------------------------
+declare
+ rec record;
+ sql text;
+ tbl text;
+ arr text;
+ part text;
+ select_fields text;
+ retry_expr text;
+ batch record;
+begin
+ select s.sub_last_tick, s.sub_next_tick, s.sub_id, s.sub_queue,
+ get_snapshot_xmax(last.tick_snapshot) as tx_start,
+ get_snapshot_xmax(cur.tick_snapshot) as tx_end,
+ last.tick_snapshot as last_snapshot,
+ cur.tick_snapshot as cur_snapshot
+ into batch
+ from pgq.subscription s, pgq.tick last, pgq.tick cur
+ where s.sub_batch = x_batch_id
+ and last.tick_queue = s.sub_queue
+ and last.tick_id = s.sub_last_tick
+ and cur.tick_queue = s.sub_queue
+ and cur.tick_id = s.sub_next_tick;
+ if not found then
+ raise exception 'batch not found';
+ end if;
+
+ -- load older transactions
+ arr := '';
+ for rec in
+ -- active tx-es in prev_snapshot that were committed in cur_snapshot
+ select id1 from
+ get_snapshot_active(batch.last_snapshot) id1 left join
+ get_snapshot_active(batch.cur_snapshot) id2 on (id1 = id2)
+ where id2 is null
+ order by 1 desc
+ loop
+ -- try to avoid big IN expression, so try to include nearby
+ -- tx'es into range
+ if batch.tx_start - 100 <= rec.id1 then
+ batch.tx_start := rec.id1;
+ else
+ if arr = '' then
+ arr := rec.id1;
+ else
+ arr := arr || ',' || rec.id1;
+ end if;
+ end if;
+ end loop;
+
+ -- must match pgq.event_template
+ select_fields := 'select ev_id, ev_time, ev_txid, ev_retry, ev_type,'
+ || ' ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4';
+ retry_expr := ' and (ev_owner is null or ev_owner = '
+ || batch.sub_id || ')';
+
+ -- now generate query that goes over all potential tables
+ sql := '';
+ for rec in
+ select xtbl from pgq.batch_event_tables(x_batch_id) xtbl
+ loop
+ tbl := rec.xtbl;
+ -- this gets newer queries that definitely are not in prev_snapshot
+ part := select_fields
+ || ' from pgq.tick cur, pgq.tick last, ' || tbl || ' ev '
+ || ' where cur.tick_id = ' || batch.sub_next_tick
+ || ' and cur.tick_queue = ' || batch.sub_queue
+ || ' and last.tick_id = ' || batch.sub_last_tick
+ || ' and last.tick_queue = ' || batch.sub_queue
+ || ' and ev.ev_txid >= ' || batch.tx_start
+ || ' and ev.ev_txid <= ' || batch.tx_end
+ || ' and txid_in_snapshot(ev.ev_txid, cur.tick_snapshot)'
+ || ' and not txid_in_snapshot(ev.ev_txid, last.tick_snapshot)'
+ || retry_expr;
+ -- now include older tx-es, that were ongoing
+ -- at the time of prev_snapshot
+ if arr <> '' then
+ part := part || ' union all '
+ || select_fields || ' from ' || tbl || ' ev '
+ || ' where ev.ev_txid in (' || arr || ')'
+ || retry_expr;
+ end if;
+ if sql = '' then
+ sql := part;
+ else
+ sql := sql || ' union all ' || part;
+ end if;
+ end loop;
+ if sql = '' then
+ raise exception 'could not construct sql for batch %', x_batch_id;
+ end if;
+ return sql || ' order by 1';
+end;
+$$ language plpgsql; -- no perms needed
+
--- /dev/null
+create or replace function pgq.batch_event_tables(x_batch_id bigint)
+returns setof text as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.batch_event_tables(1)
+--
+-- Returns set of table names where this batch events may reside.
+--
+-- Parameters:
+-- x_batch_id - ID of a active batch.
+-- ----------------------------------------------------------------------
+declare
+ nr integer;
+ tbl text;
+ use_prev integer;
+ use_next integer;
+ batch record;
+begin
+ select
+ get_snapshot_xmin(last.tick_snapshot) as tx_min, -- absolute minimum
+ get_snapshot_xmax(cur.tick_snapshot) as tx_max, -- absolute maximum
+ q.queue_data_pfx, q.queue_ntables,
+ q.queue_cur_table, q.queue_switch_step1, q.queue_switch_step2
+ into batch
+ from pgq.tick last, pgq.tick cur, pgq.subscription s, pgq.queue q
+ where cur.tick_id = s.sub_next_tick
+ and cur.tick_queue = s.sub_queue
+ and last.tick_id = s.sub_last_tick
+ and last.tick_queue = s.sub_queue
+ and s.sub_batch = x_batch_id
+ and q.queue_id = s.sub_queue;
+ if not found then
+ raise exception 'Cannot find data for batch %', x_batch_id;
+ end if;
+
+ -- if its definitely not in one or other, look into both
+ if batch.tx_max < batch.queue_switch_step1 then
+ use_prev := 1;
+ use_next := 0;
+ elsif batch.queue_switch_step2 is not null
+ and (batch.tx_min > batch.queue_switch_step2)
+ then
+ use_prev := 0;
+ use_next := 1;
+ else
+ use_prev := 1;
+ use_next := 1;
+ end if;
+
+ if use_prev then
+ nr := batch.queue_cur_table - 1;
+ if nr < 0 then
+ nr := batch.queue_ntables - 1;
+ end if;
+ tbl := batch.queue_data_pfx || '_' || nr;
+ return next tbl;
+ end if;
+
+ if use_next then
+ tbl := batch.queue_data_pfx || '_' || batch.queue_cur_table;
+ return next tbl;
+ end if;
+
+ return;
+end;
+$$ language plpgsql; -- no perms needed
+
+
--- /dev/null
+create or replace function pgq.create_queue(i_queue_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.create_queue(1)
+--
+-- Creates new queue with given name.
+--
+-- Returns:
+-- 0 - queue already exists
+-- 1 - queue created
+-- ----------------------------------------------------------------------
+declare
+ tblpfx text;
+ tblname text;
+ idxpfx text;
+ idxname text;
+ sql text;
+ id integer;
+ tick_seq text;
+ ev_seq text;
+ n_tables integer;
+begin
+ if i_queue_name is null then
+ raise exception 'Invalid NULL value';
+ end if;
+
+ -- check if exists
+ perform 1 from pgq.queue where queue_name = i_queue_name;
+ if found then
+ return 0;
+ end if;
+
+ -- insert event
+ id := nextval('pgq.queue_queue_id_seq');
+ tblpfx := 'pgq.event_' || id;
+ idxpfx := 'event_' || id;
+ tick_seq := 'pgq.event_' || id || '_tick_seq';
+ ev_seq := 'pgq.event_' || id || '_id_seq';
+ insert into pgq.queue (queue_id, queue_name,
+ queue_data_pfx, queue_event_seq, queue_tick_seq)
+ values (id, i_queue_name, tblpfx, ev_seq, tick_seq);
+
+ select queue_ntables into n_tables from pgq.queue
+ where queue_id = id;
+
+ -- create seqs
+ execute 'CREATE SEQUENCE ' || tick_seq;
+ execute 'CREATE SEQUENCE ' || ev_seq;
+
+ -- create data tables
+ execute 'CREATE TABLE ' || tblpfx || ' () '
+ || ' INHERITS (pgq.event_template)';
+ for i in 0 .. (n_tables - 1) loop
+ tblname := tblpfx || '_' || i;
+ idxname := idxpfx || '_' || i;
+ execute 'CREATE TABLE ' || tblname || ' () '
+ || ' INHERITS (' || tblpfx || ')';
+ execute 'ALTER TABLE ' || tblname || ' ALTER COLUMN ev_id '
+ || ' SET DEFAULT nextval(' || quote_literal(ev_seq) || ')';
+ execute 'create index ' || idxname || '_txid_idx on '
+ || tblname || ' (ev_txid)';
+ end loop;
+
+ perform pgq.grant_perms(i_queue_name);
+
+ perform pgq.ticker(i_queue_name);
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.current_event_table(x_queue_name text)
+returns text as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.current_event_table(1)
+--
+-- Return active event table for particular queue.
+--
+-- Note:
+-- The result is valid only during current transaction.
+--
+-- Parameters:
+-- x_queue_name - Queue name.
+-- ----------------------------------------------------------------------
+declare
+ res text;
+begin
+ select queue_data_pfx || '_' || queue_cur_table into res
+ from pgq.queue where queue_name = x_queue_name;
+ if not found then
+ raise exception 'Event queue not found';
+ end if;
+ return res;
+end;
+$$ language plpgsql; -- no perms needed
+
--- /dev/null
+create or replace function pgq.drop_queue(x_queue_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.drop_queue(1)
+--
+-- Drop queue and all associated tables.
+-- No consumers must be listening on the queue.
+--
+-- ----------------------------------------------------------------------
+declare
+ tblname text;
+ q record;
+ num integer;
+begin
+ -- check ares
+ if x_queue_name is null then
+ raise exception 'Invalid NULL value';
+ end if;
+
+ -- check if exists
+ select * into q from pgq.queue
+ where queue_name = x_queue_name;
+ if not found then
+ raise exception 'No such event queue';
+ end if;
+
+ -- check if no consumers
+ select count(*) into num from pgq.subscription
+ where sub_queue = q.queue_id;
+ if num > 0 then
+ raise exception 'cannot drop queue, consumers still attached';
+ end if;
+
+ -- drop data tables
+ for i in 0 .. (q.queue_ntables - 1) loop
+ tblname := q.queue_data_pfx || '_' || i;
+ execute 'DROP TABLE ' || tblname;
+ end loop;
+ execute 'DROP TABLE ' || q.queue_data_pfx;
+
+ -- delete ticks
+ delete from pgq.tick where tick_queue = q.queue_id;
+
+ -- drop seqs
+ -- FIXME: any checks needed here?
+ execute 'DROP SEQUENCE ' || q.queue_tick_seq;
+ execute 'DROP SEQUENCE ' || q.queue_event_seq;
+
+ -- delete event
+ delete from pgq.queue
+ where queue_name = x_queue_name;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.event_failed(
+ x_batch_id bigint,
+ x_event_id bigint,
+ x_reason text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.event_failed(3)
+--
+-- Copies the event to failed queue. Can be looked later.
+--
+-- Parameters:
+-- x_batch_id - ID of active batch.
+-- x_event_id - Event id
+-- x_reason - Text to associate with event.
+--
+-- Returns:
+-- 0 if event was already in queue, 1 otherwise.
+-- ----------------------------------------------------------------------
+begin
+ insert into pgq.failed_queue (ev_failed_reason, ev_failed_time,
+ ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4)
+ select x_reason, now(),
+ ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0),
+ ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4
+ from pgq.get_batch_events(x_batch_id),
+ pgq.subscription
+ where sub_batch = x_batch_id
+ and ev_id = x_event_id;
+ if not found then
+ raise exception 'event not found';
+ end if;
+ return 1;
+
+-- dont worry if the event is already in queue
+exception
+ when unique_violation then
+ return 0;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.event_retry(
+ x_batch_id bigint,
+ x_event_id bigint,
+ x_retry_time timestamptz)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.event_retry(3)
+--
+-- Put the event into retry queue, to be processed later again.
+--
+-- Parameters:
+-- x_batch_id - ID of active batch.
+-- x_event_id - event id
+-- x_retry_time - Time when the event should be put back into queue
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+begin
+ insert into pgq.retry_queue (ev_retry_after,
+ ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4)
+ select x_retry_time,
+ ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0) + 1,
+ ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4
+ from pgq.get_batch_events(x_batch_id),
+ pgq.subscription
+ where sub_batch = x_batch_id
+ and ev_id = x_event_id;
+ if not found then
+ raise exception 'event not found';
+ end if;
+ return 1;
+
+-- dont worry if the event is already in queue
+exception
+ when unique_violation then
+ return 0;
+end;
+$$ language plpgsql security definer;
+
+
+create or replace function pgq.event_retry(
+ x_batch_id bigint,
+ x_event_id bigint,
+ x_retry_seconds integer)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.event_retry(3)
+--
+-- Put the event into retry queue, to be processed later again.
+--
+-- Parameters:
+-- x_batch_id - ID of active batch.
+-- x_event_id - event id
+-- x_retry_seconds - Time when the event should be put back into queue
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ new_retry timestamptz;
+begin
+ new_retry := current_timestamp + ((x_retry_seconds || ' seconds')::interval);
+ return pgq.event_retry(x_batch_id, x_event_id, new_retry);
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.event_retry_raw(
+ x_queue text,
+ x_consumer text,
+ x_retry_after timestamptz,
+ x_ev_id bigint,
+ x_ev_time timestamptz,
+ x_ev_retry integer,
+ x_ev_type text,
+ x_ev_data text,
+ x_ev_extra1 text,
+ x_ev_extra2 text,
+ x_ev_extra3 text,
+ x_ev_extra4 text)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.event_retry_raw(12)
+--
+-- Allows full control over what goes to retry queue.
+--
+-- Parameters:
+-- x_queue - name of the queue
+-- x_consumer - name of the consumer
+-- x_retry_after - when the event should be processed again
+-- x_ev_id - event id
+-- x_ev_time - creation time
+-- x_ev_retry - retry count
+-- x_ev_type - user data
+-- x_ev_data - user data
+-- x_ev_extra1 - user data
+-- x_ev_extra2 - user data
+-- x_ev_extra3 - user data
+-- x_ev_extra4 - user data
+--
+-- Returns:
+-- Event ID.
+-- ----------------------------------------------------------------------
+declare
+ q record;
+ id bigint;
+begin
+ select sub_id, queue_event_seq into q
+ from pgq.consumer, pgq.queue, pgq.subscription
+ where queue_name = x_queue
+ and co_name = x_consumer
+ and sub_consumer = co_id
+ and sub_queue = queue_id;
+ if not found then
+ raise exception 'consumer not registered';
+ end if;
+
+ id := x_ev_id;
+ if id is null then
+ id := nextval(q.queue_event_seq);
+ end if;
+
+ insert into pgq.retry_queue (ev_retry_after,
+ ev_id, ev_time, ev_owner, ev_retry,
+ ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)
+ values (x_retry_after, x_ev_id, x_ev_time, q.sub_id, x_ev_retry,
+ x_ev_type, x_ev_data, x_ev_extra1, x_ev_extra2,
+ x_ev_extra3, x_ev_extra4);
+
+ return id;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function pgq.failed_event_list(
+ x_queue_name text,
+ x_consumer_name text)
+returns setof pgq.failed_queue as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.failed_event_list(2)
+--
+-- Get list of all failed events for one consumer.
+--
+-- Parameters:
+-- x_queue_name - Queue name
+-- x_consumer_name - Consumer name
+--
+-- Returns:
+-- List of failed events.
+-- ----------------------------------------------------------------------
+declare
+ rec pgq.failed_queue%rowtype;
+begin
+ for rec in
+ select fq.*
+ from pgq.failed_queue fq, pgq.consumer,
+ pgq.queue, pgq.subscription
+ where queue_name = x_queue_name
+ and co_name = x_consumer_name
+ and sub_consumer = co_id
+ and sub_queue = queue_id
+ and ev_owner = sub_id
+ order by ev_id
+ loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq.failed_event_list(
+ x_queue_name text,
+ x_consumer_name text,
+ x_count integer,
+ x_offset integer)
+returns setof pgq.failed_queue as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.failed_event_list(4)
+--
+-- Get list of failed events, from offset and specific count.
+--
+-- Parameters:
+-- x_queue_name - Queue name
+-- x_consumer_name - Consumer name
+-- x_count - Max amount of events to fetch
+-- x_offset - From this offset
+--
+-- Returns:
+-- List of failed events.
+-- ----------------------------------------------------------------------
+declare
+ rec pgq.failed_queue%rowtype;
+begin
+ for rec in
+ select fq.*
+ from pgq.failed_queue fq, pgq.consumer,
+ pgq.queue, pgq.subscription
+ where queue_name = x_queue_name
+ and co_name = x_consumer_name
+ and sub_consumer = co_id
+ and sub_queue = queue_id
+ and ev_owner = sub_id
+ order by ev_id
+ limit x_count
+ offset x_offset
+ loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq.failed_event_count(
+ x_queue_name text,
+ x_consumer_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.failed_event_count(2)
+--
+-- Get size of failed event queue.
+--
+-- Parameters:
+-- x_queue_name - Queue name
+-- x_consumer_name - Consumer name
+--
+-- Returns:
+-- Number of failed events in failed event queue.
+-- ----------------------------------------------------------------------
+declare
+ ret integer;
+begin
+ select count(1) into ret
+ from pgq.failed_queue, pgq.consumer, pgq.queue, pgq.subscription
+ where queue_name = x_queue_name
+ and co_name = x_consumer_name
+ and sub_queue = queue_id
+ and sub_consumer = co_id
+ and ev_owner = sub_id;
+ return ret;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq.failed_event_delete(
+ x_queue_name text,
+ x_consumer_name text,
+ x_event_id bigint)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.failed_event_delete(3)
+--
+-- Delete specific event from failed event queue.
+--
+-- Parameters:
+-- x_queue_name - Queue name
+-- x_consumer_name - Consumer name
+-- x_event_id - Event ID
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ x_sub_id integer;
+begin
+ select sub_id into x_sub_id
+ from pgq.subscription, pgq.consumer, pgq.queue
+ where queue_name = x_queue_name
+ and co_name = x_consumer_name
+ and sub_consumer = co_id
+ and sub_queue = queue_id;
+ if not found then
+ raise exception 'no such queue/consumer';
+ end if;
+
+ delete from pgq.failed_queue
+ where ev_owner = x_sub_id
+ and ev_id = x_event_id;
+ if not found then
+ raise exception 'event not found';
+ end if;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq.failed_event_retry(
+ x_queue_name text,
+ x_consumer_name text,
+ x_event_id bigint)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.failed_event_retry(3)
+--
+-- Insert specific event from failed queue to main queue.
+--
+-- Parameters:
+-- x_queue_name - Queue name
+-- x_consumer_name - Consumer name
+-- x_event_id - Event ID
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ ret bigint;
+ x_sub_id integer;
+begin
+ select sub_id into x_sub_id
+ from pgq.subscription, pgq.consumer, pgq.queue
+ where queue_name = x_queue_name
+ and co_name = x_consumer_name
+ and sub_consumer = co_id
+ and sub_queue = queue_id;
+ if not found then
+ raise exception 'no such queue/consumer';
+ end if;
+
+ select pgq.insert_event_raw(x_queue_name, ev_id, ev_time,
+ ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4)
+ into ret
+ from pgq.failed_queue, pgq.consumer, pgq.queue
+ where ev_owner = x_sub_id
+ and ev_id = x_event_id;
+ if not found then
+ raise exception 'event not found';
+ end if;
+
+ perform pgq.failed_event_delete(x_queue_name, x_consumer_name, x_event_id);
+
+ return ret;
+end;
+$$ language plpgsql security definer;
+
+
--- /dev/null
+
+create or replace function pgq.finish_batch(
+ x_batch_id bigint)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.finish_batch(1)
+--
+-- Closes a batch. No more operations can be done with events
+-- of this batch.
+--
+-- Parameters:
+-- x_batch_id - id of batch.
+--
+-- Returns:
+-- If batch 1 if batch was found, 0 otherwise.
+-- ----------------------------------------------------------------------
+begin
+ update pgq.subscription
+ set sub_active = now(),
+ sub_last_tick = sub_next_tick,
+ sub_next_tick = null,
+ sub_batch = null
+ where sub_batch = x_batch_id;
+ if not found then
+ raise warning 'finish_batch: batch % not found', x_batch_id;
+ return 0;
+ end if;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.get_batch_events(x_batch_id bigint)
+returns setof pgq.ret_batch_event as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_batch_events(1)
+--
+-- Get all events in batch.
+--
+-- Parameters:
+-- x_batch_id - ID of active batch.
+--
+-- Returns:
+-- List of events.
+-- ----------------------------------------------------------------------
+declare
+ rec pgq.ret_batch_event%rowtype;
+ sql text;
+begin
+ sql := pgq.batch_event_sql(x_batch_id);
+ for rec in execute sql loop
+ return next rec;
+ end loop;
+ return;
+end;
+$$ language plpgsql; -- no perms needed
+
+
--- /dev/null
+
+create or replace function pgq.get_batch_info(x_batch_id bigint)
+returns pgq.ret_batch_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_batch_info(1)
+--
+-- Returns detailed info about a batch.
+--
+-- Parameters:
+-- x_batch_id - id of a active batch.
+--
+-- Returns:
+-- Info
+-- ----------------------------------------------------------------------
+declare
+ ret pgq.ret_batch_info%rowtype;
+begin
+ select queue_name, co_name,
+ prev.tick_time as batch_start,
+ cur.tick_time as batch_end,
+ sub_last_tick, sub_next_tick,
+ current_timestamp - cur.tick_time as lag
+ into ret
+ from pgq.subscription, pgq.tick cur, pgq.tick prev,
+ pgq.queue, pgq.consumer
+ where sub_batch = x_batch_id
+ and prev.tick_id = sub_last_tick
+ and prev.tick_queue = sub_queue
+ and cur.tick_id = sub_next_tick
+ and cur.tick_queue = sub_queue
+ and queue_id = sub_queue
+ and co_id = sub_consumer;
+ return ret;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+-------------------------------------------------------------------------
+create or replace function pgq.get_consumer_info()
+returns setof pgq.ret_consumer_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_consumer_info(0)
+--
+-- Desc
+--
+-- Parameters:
+-- arg - desc
+--
+-- Returns:
+-- desc
+-- ----------------------------------------------------------------------
+declare
+ ret pgq.ret_consumer_info%rowtype;
+ i record;
+begin
+ for i in select queue_name from pgq.queue order by 1
+ loop
+ for ret in
+ select * from pgq.get_consumer_info(i.queue_name)
+ loop
+ return next ret;
+ end loop;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+
+-------------------------------------------------------------------------
+create or replace function pgq.get_consumer_info(x_queue_name text)
+returns setof pgq.ret_consumer_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_consumer_info(1)
+--
+-- Desc
+--
+-- Parameters:
+-- arg - desc
+--
+-- Returns:
+-- desc
+-- ----------------------------------------------------------------------
+declare
+ ret pgq.ret_consumer_info%rowtype;
+ tmp record;
+begin
+ for tmp in
+ select queue_name, co_name
+ from pgq.queue, pgq.consumer, pgq.subscription
+ where queue_id = sub_queue
+ and co_id = sub_consumer
+ and queue_name = x_queue_name
+ order by 1, 2
+ loop
+ for ret in
+ select * from pgq.get_consumer_info(tmp.queue_name, tmp.co_name)
+ loop
+ return next ret;
+ end loop;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+
+------------------------------------------------------------------------
+create or replace function pgq.get_consumer_info(
+ x_queue_name text,
+ x_consumer_name text)
+returns setof pgq.ret_consumer_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_consumer_info(2)
+--
+-- Get info about particular consumer on particular queue.
+--
+-- Parameters:
+-- x_queue_name - name of a queue.
+-- x_consumer_name - name of a consumer
+--
+-- Returns:
+-- info
+-- ----------------------------------------------------------------------
+declare
+ ret pgq.ret_consumer_info%rowtype;
+begin
+ for ret in
+ select queue_name, co_name,
+ current_timestamp - tick_time as lag,
+ current_timestamp - sub_active as last_seen
+ from pgq.subscription, pgq.tick, pgq.queue, pgq.consumer
+ where tick_id = sub_last_tick
+ and queue_id = sub_queue
+ and tick_queue = sub_queue
+ and co_id = sub_consumer
+ and queue_name = x_queue_name
+ and co_name = x_consumer_name
+ order by 1,2
+ loop
+ return next ret;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.get_queue_info()
+returns setof pgq.ret_queue_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_queue_info(0)
+--
+-- Get info about all queues.
+--
+-- Returns:
+-- List of pgq.ret_queue_info records.
+-- ----------------------------------------------------------------------
+declare
+ qname text;
+ ret pgq.ret_queue_info%rowtype;
+begin
+ for qname in
+ select queue_name from pgq.queue order by 1
+ loop
+ select * into ret from pgq.get_queue_info(qname);
+ return next ret;
+ end loop;
+ return;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq.get_queue_info(qname text)
+returns pgq.ret_queue_info as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.get_queue_info(1)
+--
+-- Get info about particular queue.
+--
+-- Returns:
+-- One pgq.ret_queue_info record.
+-- ----------------------------------------------------------------------
+declare
+ ret pgq.ret_queue_info%rowtype;
+begin
+ select queue_name, queue_ntables, queue_cur_table,
+ queue_rotation_period, queue_switch_time,
+ queue_external_ticker,
+ queue_ticker_max_count, queue_ticker_max_lag,
+ queue_ticker_idle_period,
+ (select current_timestamp - tick_time
+ from pgq.tick where tick_queue = queue_id
+ order by tick_queue desc, tick_id desc limit 1
+ ) as ticker_lag
+ into ret from pgq.queue where queue_name = qname;
+ return ret;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.grant_perms(x_queue_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.grant_perms(1)
+--
+-- Make event tables readable by public.
+--
+-- Parameters:
+-- x_queue_name - Name of the queue.
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ q record;
+ i integer;
+begin
+ select * from pgq.queue into q
+ where queue_name = x_queue_name;
+ if not found then
+ raise exception 'Queue not found';
+ end if;
+ execute 'grant select, update on '
+ || q.queue_event_seq || ',' || q.queue_tick_seq
+ || ' to public';
+ execute 'grant select on '
+ || q.queue_data_pfx
+ || ' to public';
+ for i in 0 .. q.queue_ntables - 1 loop
+ execute 'grant select, insert on '
+ || q.queue_data_pfx || '_' || i
+ || ' to public';
+ end loop;
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+create or replace function pgq.insert_event(queue_name text, ev_type text, ev_data text)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.insert_event(3)
+--
+-- Insert a event into queue.
+--
+-- Parameters:
+-- queue_name - Name of the queue
+-- ev_type - User-specified type for the event
+-- ev_data - User data for the event
+--
+-- Returns:
+-- Event ID
+-- ----------------------------------------------------------------------
+begin
+ return pgq.insert_event(queue_name, ev_type, ev_data, null, null, null, null);
+end;
+$$ language plpgsql; -- event inserting needs no special perms
+
+
+
+create or replace function pgq.insert_event(
+ queue_name text, ev_type text, ev_data text,
+ ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.insert_event(7)
+--
+-- Insert a event into queue with all the extra fields.
+--
+-- Parameters:
+-- queue_name - Name of the queue
+-- ev_type - User-specified type for the event
+-- ev_data - User data for the event
+-- ev_extra1 - Extra data field for the event
+-- ev_extra2 - Extra data field for the event
+-- ev_extra3 - Extra data field for the event
+-- ev_extra4 - Extra data field for the event
+--
+-- Returns:
+-- Event ID
+-- ----------------------------------------------------------------------
+begin
+ return pgq.insert_event_raw(queue_name, null, now(), null, null,
+ ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4);
+end;
+$$ language plpgsql; -- event inserting needs no special perms
+
--- /dev/null
+create or replace function pgq.insert_event_raw(
+ queue_name text, ev_id bigint, ev_time timestamptz,
+ ev_owner integer, ev_retry integer, ev_type text, ev_data text,
+ ev_extra1 text, ev_extra2 text, ev_extra3 text, ev_extra4 text)
+returns bigint as $$
+# -- ----------------------------------------------------------------------
+# -- Function: pgq.insert_event_raw(11)
+# --
+# -- Actual event insertion. Used also by retry queue maintenance.
+# --
+# -- Parameters:
+# -- queue_name - Name of the queue
+# -- ev_id - Event ID. If NULL, will be taken from seq.
+# -- ev_time - Event creation time.
+# -- ev_type - user data
+# -- ev_data - user data
+# -- ev_extra1 - user data
+# -- ev_extra2 - user data
+# -- ev_extra3 - user data
+# -- ev_extra4 - user data
+# --
+# -- Returns:
+# -- Event ID.
+# -- ----------------------------------------------------------------------
+
+ # load args
+ queue_name = args[0]
+ ev_id = args[1]
+ ev_time = args[2]
+ ev_owner = args[3]
+ ev_retry = args[4]
+ ev_type = args[5]
+ ev_data = args[6]
+ ev_extra1 = args[7]
+ ev_extra2 = args[8]
+ ev_extra3 = args[9]
+ ev_extra4 = args[10]
+
+ if not "cf_plan" in SD:
+ # get current event table
+ q = "select queue_data_pfx, queue_cur_table, queue_event_seq "\
+ " from pgq.queue where queue_name = $1"
+ SD["cf_plan"] = plpy.prepare(q, ["text"])
+
+ # get next id
+ q = "select nextval($1) as id"
+ SD["seq_plan"] = plpy.prepare(q, ["text"])
+
+ # get queue config
+ res = plpy.execute(SD["cf_plan"], [queue_name])
+ if len(res) != 1:
+ plpy.error("Unknown event queue: %s" % (queue_name))
+ tbl_prefix = res[0]["queue_data_pfx"]
+ cur_nr = res[0]["queue_cur_table"]
+ id_seq = res[0]["queue_event_seq"]
+
+ # get id - bump seq even if id is given
+ res = plpy.execute(SD['seq_plan'], [id_seq])
+ if ev_id is None:
+ ev_id = res[0]["id"]
+
+ # create plan for insertion
+ ins_plan = None
+ ins_key = "ins.%s" % (queue_name)
+ if ins_key in SD:
+ nr, ins_plan = SD[ins_key]
+ if nr != cur_nr:
+ ins_plan = None
+ if ins_plan == None:
+ q = "insert into %s_%s (ev_id, ev_time, ev_owner, ev_retry,"\
+ " ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4)"\
+ " values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" % (
+ tbl_prefix, cur_nr)
+ types = ["int8", "timestamptz", "int4", "int4", "text",
+ "text", "text", "text", "text", "text"]
+ ins_plan = plpy.prepare(q, types)
+ SD[ins_key] = (cur_nr, ins_plan)
+
+ # insert the event
+ plpy.execute(ins_plan, [ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4])
+
+ # done
+ return ev_id
+
+$$ language plpythonu; -- event inserting needs no special perms
+
--- /dev/null
+create or replace function pgq.maint_retry_events()
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.maint_retry_events(0)
+--
+-- Moves retry events back to main queue.
+--
+-- It moves small amount at a time. It should be called
+-- until it returns 0
+--
+-- Parameters:
+-- arg - desc
+--
+-- Returns:
+-- Number of events processed.
+-- ----------------------------------------------------------------------
+declare
+ cnt integer;
+ rec record;
+begin
+ cnt := 0;
+ for rec in
+ select pgq.insert_event_raw(queue_name,
+ ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4),
+ ev_owner, ev_id
+ from pgq.retry_queue, pgq.queue, pgq.subscription
+ where ev_retry_after <= current_timestamp
+ and sub_id = ev_owner
+ and queue_id = sub_queue
+ order by ev_retry_after
+ limit 10
+ loop
+ cnt := cnt + 1;
+ delete from pgq.retry_queue
+ where ev_owner = rec.ev_owner
+ and ev_id = rec.ev_id;
+ end loop;
+ return cnt;
+end;
+$$ language plpgsql; -- need admin access
+
--- /dev/null
+create or replace function pgq.maint_rotate_tables_step1(i_queue_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.maint_rotate_tables_step1(1)
+--
+-- Rotate tables for one queue.
+--
+-- Parameters:
+-- i_queue_name - Name of the queue
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ badcnt integer;
+ cf record;
+ nr integer;
+ tbl text;
+begin
+ -- check if needed and load record
+ select * from pgq.queue into cf
+ where queue_name = i_queue_name
+ and queue_rotation_period is not null
+ and queue_switch_step2 is not null
+ and queue_switch_time + queue_rotation_period < current_timestamp
+ for update;
+ if not found then
+ return 0;
+ end if;
+
+ -- check if any consumer is on previous table
+ select coalesce(count(*), 0) into badcnt
+ from pgq.subscription, pgq.tick
+ where get_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2
+ and sub_queue = cf.queue_id
+ and tick_queue = cf.queue_id
+ and tick_id = (select tick_id from pgq.tick
+ where tick_id < sub_last_tick
+ and tick_queue = sub_queue
+ order by tick_queue desc, tick_id desc
+ limit 1);
+ if badcnt > 0 then
+ return 0;
+ end if;
+
+ -- all is fine, calc next table number
+ nr := cf.queue_cur_table + 1;
+ if nr = cf.queue_ntables then
+ nr := 0;
+ end if;
+ tbl := cf.table_name || '_' || nr;
+
+ -- there may be long lock on the table from pg_dump,
+ -- detect it and skip rotate then
+ begin
+ execute 'lock table ' || tbl || ' nowait';
+ execute 'truncate ' || tbl;
+ exception
+ when lock_not_available then
+ raise warning 'truncate of % failed, skipping rotate', tbl;
+ return 0;
+ end;
+
+ -- remember the moment
+ update pgq.queue
+ set queue_cur_table = nr,
+ queue_switch_time = current_timestamp,
+ queue_switch_step1 = get_current_txid(),
+ queue_switch_step2 = NULL
+ where queue_id = cf.queue_id;
+
+ -- clean ticks - avoid partial batches
+ delete from pgq.tick
+ where tick_queue = cf.queue_id
+ and get_snapshot_xmin(snapshot) < cf.last_switch_step2;
+
+ return 1;
+end;
+$$ language plpgsql; -- need admin access
+
+-- ----------------------------------------------------------------------
+-- Function: pgq.maint_rotate_tables_step2(0)
+--
+-- It tag rotation as finished where needed. It should be
+-- called in separate transaction than pgq.maint_rotate_tables_step1()
+-- ----------------------------------------------------------------------
+create or replace function pgq.maint_rotate_tables_step2()
+returns integer as $$
+-- visibility tracking. this should run in separate
+-- tranaction than step1
+begin
+ update pgq.queue
+ set queue_switch_step2 = get_current_txid()
+ where queue_switch_step2 is null;
+ return 1;
+end;
+$$ language plpgsql; -- need admin access
+
--- /dev/null
+create or replace function pgq.maint_tables_to_vacuum()
+returns setof text as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.maint_tables_to_vacuum(0)
+--
+-- Returns list of tablenames that need frequent vacuuming.
+--
+-- The goal is to avoid hardcoding them into maintenance process.
+--
+-- Returns:
+-- List of table names.
+-- ----------------------------------------------------------------------
+begin
+ return next 'pgq.subscription';
+ return next 'pgq.consumer';
+ return next 'pgq.queue';
+ return next 'pgq.tick';
+ return next 'pgq.retry_queue';
+
+ -- vacuum also txid.epoch, if exists
+ perform 1 from pg_class t, pg_namespace n
+ where t.relname = 'epoch'
+ and n.nspname = 'txid'
+ and n.oid = t.relnamespace;
+ if found then
+ return next 'txid.epoch';
+ end if;
+
+ return;
+end;
+$$ language plpgsql;
+
+
--- /dev/null
+create or replace function pgq.next_batch(x_queue_name text, x_consumer_name text)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.next_batch(2)
+--
+-- Makes next block of events active.
+--
+-- If it returns NULL, there is no events available in queue.
+-- Consumer should sleep a bith then.
+--
+-- Parameters:
+-- x_queue_name - Name of the queue
+-- x_consumer_name - Name of the consumer
+--
+-- Returns:
+-- Batch ID or NULL if there are no more events available.
+-- ----------------------------------------------------------------------
+declare
+ next_tick bigint;
+ next_batch bigint;
+ errmsg text;
+ sub record;
+begin
+ select sub_queue, sub_id, sub_last_tick, sub_batch into sub
+ from pgq.queue q, pgq.consumer c, pgq.subscription s
+ where q.queue_name = x_queue_name
+ and c.co_name = x_consumer_name
+ and s.sub_queue = q.queue_id
+ and s.sub_consumer = c.co_id;
+ if not found then
+ errmsg := 'Not subscriber to queue: '
+ || coalesce(x_queue_name, 'NULL')
+ || '/'
+ || coalesce(x_consumer_name, 'NULL');
+ raise exception '%', errmsg;
+ end if;
+
+ -- has already active batch
+ if sub.sub_batch is not null then
+ return sub.sub_batch;
+ end if;
+
+ -- find next tick
+ select tick_id into next_tick
+ from pgq.tick
+ where tick_id > sub.sub_last_tick
+ and tick_queue = sub.sub_queue
+ order by tick_queue asc, tick_id asc
+ limit 1;
+ if not found then
+ -- nothing to do
+ return null;
+ end if;
+
+ -- get next batch
+ next_batch := nextval('pgq.batch_id_seq');
+ update pgq.subscription
+ set sub_batch = next_batch,
+ sub_next_tick = next_tick,
+ sub_active = now()
+ where sub_id = sub.sub_id;
+ return next_batch;
+end;
+$$ language plpgsql security definer;
+
+
--- /dev/null
+create or replace function pgq.register_consumer(
+ x_queue_name text,
+ x_consumer_id text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.register_consumer(2)
+--
+-- Subscribe consumer on a queue.
+--
+-- From this moment forward, consumer will see all events in the queue.
+--
+-- Parameters:
+-- x_queue_name - Name of queue
+-- x_consumer_name - Name of consumer
+--
+-- Returns:
+-- 0 - if already registered
+-- 1 - if new registration
+-- ----------------------------------------------------------------------
+begin
+ return pgq.register_consumer(x_queue_name, x_consumer_id, NULL);
+end;
+$$ language plpgsql; -- no perms needed
+
+
+create or replace function pgq.register_consumer(
+ x_queue_name text,
+ x_consumer_name text,
+ x_tick_pos bigint)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.register_consumer(3)
+--
+-- Extended registration, allows to specify tick_id.
+--
+-- Note:
+-- For usage in special situations.
+--
+-- Parameters:
+-- x_queue_name - Name of a queue
+-- x_consumer_name - Name of consumer
+-- x_tick_pos - Tick ID
+--
+-- Returns:
+-- 0/1 whether consumer has already registered.
+-- ----------------------------------------------------------------------
+declare
+ tmp text;
+ last_tick bigint;
+ x_queue_id integer;
+ x_consumer_id integer;
+ queue integer;
+ sub record;
+begin
+ select queue_id into x_queue_id from pgq.queue
+ where queue_name = x_queue_name;
+ if not found then
+ raise exception 'Event queue not created yet';
+ end if;
+
+ -- get consumer and create if new
+ select co_id into x_consumer_id from pgq.consumer
+ where co_name = x_consumer_name;
+ if not found then
+ insert into pgq.consumer (co_name) values (x_consumer_name);
+ x_consumer_id := currval('pgq.consumer_co_id_seq');
+ end if;
+
+ -- if particular tick was requested, check if it exists
+ if x_tick_pos is not null then
+ perform 1 from pgq.tick
+ where tick_queue = x_queue_id
+ and tick_id = x_tick_pos;
+ if not found then
+ raise exception 'cannot reposition, tick not found: %', x_tick_pos;
+ end if;
+ end if;
+
+ -- check if already registered
+ select sub_last_tick, sub_batch into sub
+ from pgq.subscription
+ where sub_consumer = x_consumer_id
+ and sub_queue = x_queue_id;
+ if found then
+ if sub.sub_batch is not null then
+ raise exception 'reposition while active not allowed';
+ end if;
+ if x_tick_pos is not null then
+ -- update tick pos if requested
+ update pgq.subscription
+ set sub_last_tick = x_tick_pos
+ where sub_consumer = x_consumer_id
+ and sub_queue = x_queue_id;
+ end if;
+ -- already registered
+ return 0;
+ end if;
+
+ -- new registration
+ if x_tick_pos is null then
+ -- start from current tick
+ select tick_id into last_tick from pgq.tick
+ where tick_queue = x_queue_id
+ order by tick_queue desc, tick_id desc
+ limit 1;
+ if not found then
+ raise exception 'No ticks for this queue. Please run ticker on database.';
+ end if;
+ else
+ last_tick := x_tick_pos;
+ end if;
+
+ -- register
+ insert into pgq.subscription (sub_queue, sub_consumer, sub_last_tick)
+ values (x_queue_id, x_consumer_id, last_tick);
+ return 1;
+end;
+$$ language plpgsql security definer;
+
+
--- /dev/null
+create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.ticker(2)
+--
+-- Insert a tick with a particular tick_id.
+--
+-- For external tickers.
+--
+-- Parameters:
+-- i_queue_name - Name of the queue
+-- i_tick_id - Id of new tick.
+--
+-- Returns:
+-- Tick id.
+-- ----------------------------------------------------------------------
+begin
+ insert into pgq.tick (tick_queue, tick_id)
+ select queue_id, i_tick_id
+ from pgq.queue
+ where queue_name = i_queue_name
+ and queue_external_ticker;
+ if not found then
+ raise exception 'queue not found';
+ end if;
+ return i_tick_id;
+end;
+$$ language plpgsql security definer; -- unsure about access
+
+create or replace function pgq.ticker(i_queue_name text)
+returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.ticker(1)
+--
+-- Insert a tick with a tick_id from sequence.
+--
+-- For pgqadm usage.
+--
+-- Parameters:
+-- i_queue_name - Name of the queue
+--
+-- Returns:
+-- Tick id.
+-- ----------------------------------------------------------------------
+declare
+ res bigint;
+ ext boolean;
+ seq text;
+ q record;
+begin
+ select queue_id, queue_tick_seq, queue_external_ticker into q
+ from pgq.queue where queue_name = i_queue_name;
+ if not found then
+ raise exception 'no such queue';
+ end if;
+
+ if q.queue_external_ticker then
+ raise exception 'This queue has external tick source.';
+ end if;
+
+ insert into pgq.tick (tick_queue, tick_id)
+ values (q.queue_id, nextval(q.queue_tick_seq));
+
+ res = currval(q.queue_tick_seq);
+ return res;
+end;
+$$ language plpgsql security definer; -- unsure about access
+
+create or replace function pgq.ticker() returns bigint as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.ticker(0)
+--
+-- Creates ticks for all queues which dont have external ticker.
+--
+-- Returns:
+-- Number of queues that were processed.
+-- ----------------------------------------------------------------------
+declare
+ res bigint;
+begin
+ select count(pgq.ticker(queue_name)) into res
+ from pgq.queue where not queue_external_ticker;
+ return res;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function pgq.unregister_consumer(
+ x_queue_name text,
+ x_consumer_name text)
+returns integer as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.unregister_consumer(2)
+--
+-- Unsubscriber consumer from the queue. Also consumer's failed
+-- and retry events are deleted.
+--
+-- Parameters:
+-- x_queue_name - Name of the queue
+-- x_consumer_name - Name of the consumer
+--
+-- Returns:
+-- nothing
+-- ----------------------------------------------------------------------
+declare
+ x_sub_id integer;
+begin
+ select sub_id into x_sub_id
+ from pgq.subscription, pgq.consumer, pgq.queue
+ where sub_queue = queue_id
+ and sub_consumer = co_id
+ and queue_name = x_queue_name
+ and co_name = x_consumer_name;
+ if not found then
+ raise exception 'consumer not registered on queue';
+ end if;
+
+ delete from pgq.retry_queue
+ where ev_owner = x_sub_id;
+
+ delete from pgq.failed_queue
+ where ev_owner = x_sub_id;
+
+ delete from pgq.subscription
+ where sub_id = x_sub_id;
+
+ return 1;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+-- ----------------------------------------------------------------------
+-- Function: pgq.version(0)
+--
+-- Returns verison string for pgq.
+-- ----------------------------------------------------------------------
+create or replace function pgq.version()
+returns text as $$
+begin
+ return '2.1';
+end;
+$$ language plpgsql;
+
--- /dev/null
+
+create or replace function pgq.insert_event(que text, ev_type text, ev_data text, x1 text, x2 text, x3 text, x4 text)
+returns bigint as $$
+begin
+ raise notice 'insert_event(%, %, %, %)', que, ev_type, ev_data, x1;
+ return 1;
+end;
+$$ language plpgsql;
+
+create table udata (
+ id serial primary key,
+ txt text,
+ bin bytea
+);
+
+create trigger utest AFTER insert or update or delete ON udata
+for each row execute procedure pgq.logutriga('udata_que');
+
+insert into udata (txt) values ('text1');
+insert into udata (bin) values (E'bi\tn\\000bin');
+
+
--- /dev/null
+
+\set ECHO none
+\i ../txid/txid.sql
+\i structure/install.sql
+
+\set ECHO all
+
+select * from pgq.maint_tables_to_vacuum();
+select * from pgq.maint_retry_events();
+
+select pgq.create_queue('tmpqueue');
+select pgq.register_consumer('tmpqueue', 'consumer');
+select pgq.unregister_consumer('tmpqueue', 'consumer');
+select pgq.drop_queue('tmpqueue');
+
+select pgq.create_queue('myqueue');
+select pgq.register_consumer('myqueue', 'consumer');
+select pgq.next_batch('myqueue', 'consumer');
+select pgq.next_batch('myqueue', 'consumer');
+select pgq.ticker();
+select pgq.next_batch('myqueue', 'consumer');
+select pgq.next_batch('myqueue', 'consumer');
+
+select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1);
+select queue_name from pgq.get_queue_info() limit 0;
+select queue_name, consumer_name from pgq.get_consumer_info() limit 0;
+
+select pgq.finish_batch(1);
+select pgq.finish_batch(1);
+
+select pgq.ticker();
+select pgq.next_batch('myqueue', 'consumer');
+select * from pgq.batch_event_tables(2);
+select * from pgq.get_batch_events(2);
+select pgq.finish_batch(2);
+
+select pgq.insert_event('myqueue', 'r1', 'data');
+select pgq.insert_event('myqueue', 'r2', 'data');
+select pgq.insert_event('myqueue', 'r3', 'data');
+select pgq.current_event_table('myqueue');
+select pgq.ticker();
+
+select pgq.next_batch('myqueue', 'consumer');
+select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4 from pgq.get_batch_events(3);
+
+select * from pgq.failed_event_list('myqueue', 'consumer');
+
+select pgq.event_failed(3, 1, 'failure test');
+select pgq.event_failed(3, 1, 'failure test');
+select pgq.event_retry(3, 2, 0);
+select pgq.event_retry(3, 2, 0);
+select pgq.finish_batch(3);
+
+select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
+ from pgq.failed_event_list('myqueue', 'consumer');
+
+select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
+ from pgq.failed_event_list('myqueue', 'consumer', 0, 1);
+
+select * from pgq.failed_event_count('myqueue', 'consumer');
+select * from pgq.failed_event_delete('myqueue', 'consumer', 0);
+
+select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0,
+ 'rawtest', 'data', null, null, null, null);
+
+select pgq.ticker();
--- /dev/null
+
+-- start testing
+create table rtest (
+ id integer primary key,
+ dat text
+);
+
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que');
+
+-- simple test
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+
+-- test new fields
+alter table rtest add column dat2 text;
+insert into rtest values (1, 'value1');
+update rtest set dat = 'value2';
+delete from rtest;
+
+-- test field ignore
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2');
+
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5, dat2 = 'newdat2';
+update rtest set dat = 6;
+delete from rtest;
+
+-- test hashed pkey
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que2', 'ignore=dat2&pkey=dat,hashtext(dat)');
+
+insert into rtest values (1, '666', 'newdat');
+update rtest set dat = 5, dat2 = 'newdat2';
+update rtest set dat = 6;
+delete from rtest;
+
+
+-- test wrong key
+drop trigger rtest_triga on rtest;
+create trigger rtest_triga after insert or update or delete on rtest
+for each row execute procedure pgq.sqltriga('que3');
+
+insert into rtest values (1, 0, 'non-null');
+insert into rtest values (2, 0, NULL);
+update rtest set dat2 = 'non-null2' where id=1;
+update rtest set dat2 = NULL where id=1;
+update rtest set dat2 = 'new-nonnull' where id=2;
+
+delete from rtest where id=1;
+delete from rtest where id=2;
+
+
+
--- /dev/null
+-- Section: Internal Functions
+
+-- Group: Low-level event handling
+
+\i functions/pgq.batch_event_sql.sql
+\i functions/pgq.batch_event_tables.sql
+\i functions/pgq.event_retry_raw.sql
+\i functions/pgq.insert_event_raw.sql
+
+-- Group: Ticker
+
+\i functions/pgq.ticker.sql
+
+-- Group: Periodic maintenence
+
+\i functions/pgq.maint_retry_events.sql
+\i functions/pgq.maint_rotate_tables.sql
+\i functions/pgq.maint_tables_to_vacuum.sql
+
+-- Group: Random utility functions
+
+\i functions/pgq.grant_perms.sql
+
--- /dev/null
+-- Section: Public Functions
+
+-- Group: Queue creation
+
+\i functions/pgq.create_queue.sql
+\i functions/pgq.drop_queue.sql
+
+-- Group: Event publishing
+
+\i functions/pgq.insert_event.sql
+\i functions/pgq.current_event_table.sql
+
+-- Group: Subscribing to queue
+
+\i functions/pgq.register_consumer.sql
+\i functions/pgq.unregister_consumer.sql
+
+-- Group: Batch processing
+
+\i functions/pgq.next_batch.sql
+\i functions/pgq.get_batch_events.sql
+\i functions/pgq.event_failed.sql
+\i functions/pgq.event_retry.sql
+\i functions/pgq.finish_batch.sql
+
+-- Group: General info functions
+
+\i functions/pgq.get_queue_info.sql
+\i functions/pgq.get_consumer_info.sql
+\i functions/pgq.version.sql
+\i functions/pgq.get_batch_info.sql
+
+-- Group: Failed queue browsing
+
+\i functions/pgq.failed_queue.sql
+
--- /dev/null
+
+\i structure/tables.sql
+\i structure/types.sql
+\i structure/func_internal.sql
+\i structure/func_public.sql
+\i structure/triggers.sql
+
--- /dev/null
+-- ----------------------------------------------------------------------
+-- Section: Internal Tables
+--
+-- Map to Slony-I:
+-- sl_node - pgq.consumer
+-- sl_set - pgq.queue
+-- sl_subscriber + sl_confirm - pgq.subscription
+-- sl_event - pgq.tick
+-- sl_setsync - pgq_ext.completed_*
+-- sl_log_* - slony1 has per-cluster data tables,
+-- here we do redirection in pgq.queue
+-- to have per-queue data tables.
+-- ----------------------------------------------------------------------
+
+set client_min_messages = 'warning';
+
+-- drop schema if exists pgq cascade;
+create schema pgq;
+grant usage on schema pgq to public;
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.consumer
+--
+-- Name to id lookup for consumers
+--
+-- Columns:
+-- co_id - consumer's id for internal usage
+-- co_name - consumer's id for external usage
+-- ----------------------------------------------------------------------
+create table pgq.consumer (
+ co_id serial,
+ co_name text not null default 'fooz',
+
+ constraint consumer_pkey primary key (co_id),
+ constraint consumer_name_uq UNIQUE (co_name)
+);
+
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.queue
+--
+-- Information about available queues
+--
+-- Columns:
+-- queue_id - queue id for internal usage
+-- queue_name - queue name visible outside
+-- queue_data - parent table for actual data tables
+-- queue_switch_step1 - tx when rotation happened
+-- queue_switch_step2 - tx after rotation was committed
+-- queue_switch_time - time when switch happened
+-- queue_ticker_max_count - batch should not contain more events
+-- queue_ticker_max_lag - events should not age more
+-- queue_ticker_idle_period - how often to tick when no events happen
+-- ----------------------------------------------------------------------
+create table pgq.queue (
+ queue_id serial,
+ queue_name text not null,
+
+ queue_ntables integer not null default 3,
+ queue_cur_table integer not null default 0,
+ queue_rotation_period interval not null default '2 hours',
+ queue_switch_step1 bigint not null default get_current_txid(),
+ queue_switch_step2 bigint default get_current_txid(),
+ queue_switch_time timestamptz not null default now(),
+
+ queue_external_ticker boolean not null default false,
+ queue_ticker_max_count integer not null default 500,
+ queue_ticker_max_lag interval not null default '3 seconds',
+ queue_ticker_idle_period interval not null default '1 minute',
+
+ queue_data_pfx text not null,
+ queue_event_seq text not null,
+ queue_tick_seq text not null,
+
+ constraint queue_pkey primary key (queue_id),
+ constraint queue_name_uq unique (queue_name)
+);
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.tick
+--
+-- Snapshots for event batching
+--
+-- Columns:
+-- tick_queue - queue id whose tick it is
+-- tick_id - ticks id (per-queue)
+-- tick_time - time when tick happened
+-- tick_snapshot
+-- ----------------------------------------------------------------------
+create table pgq.tick (
+ tick_queue int4 not null,
+ tick_id bigint not null,
+ tick_time timestamptz not null default now(),
+ tick_snapshot txid_snapshot not null default get_current_snapshot(),
+
+ constraint tick_pkey primary key (tick_queue, tick_id),
+ constraint tick_queue_fkey foreign key (tick_queue)
+ references pgq.queue (queue_id)
+);
+
+-- ----------------------------------------------------------------------
+-- Sequence: pgq.batch_id_seq
+--
+-- Sequence for batch id's.
+-- ----------------------------------------------------------------------
+
+create sequence pgq.batch_id_seq;
+-- ----------------------------------------------------------------------
+-- Table: pgq.subscription
+--
+-- Consumer registration on a queue
+--
+-- Columns:
+--
+-- sub_id - subscription id for internal usage
+-- sub_queue - queue id
+-- sub_consumer - consumer's id
+-- sub_tick - last tick the consumer processed
+-- sub_batch - shortcut for queue_id/consumer_id/tick_id
+-- sub_next_tick -
+-- ----------------------------------------------------------------------
+create table pgq.subscription (
+ sub_id serial not null,
+ sub_queue int4 not null,
+ sub_consumer int4 not null,
+ sub_last_tick bigint not null,
+ sub_active timestamptz not null default now(),
+ sub_batch bigint,
+ sub_next_tick bigint,
+
+ constraint subscription_pkey primary key (sub_id),
+ constraint sub_queue_fkey foreign key (sub_queue)
+ references pgq.queue (queue_id),
+ constraint sub_consumer_fkey foreign key (sub_consumer)
+ references pgq.consumer (co_id)
+);
+
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.event_template
+--
+-- Parent table for all event tables
+--
+-- Columns:
+-- ev_id - event's id, supposed to be unique per queue
+-- ev_time - when the event was inserted
+-- ev_txid - transaction id which inserted the event
+-- ev_owner - subscription id that wanted to retry this
+-- ev_retry - how many times the event has been retried, NULL for new events
+-- ev_type - consumer/producer can specify what the data fields contain
+-- ev_data - data field
+-- ev_extra1 - extra data field
+-- ev_extra2 - extra data field
+-- ev_extra3 - extra data field
+-- ev_extra4 - extra data field
+-- ----------------------------------------------------------------------
+create table pgq.event_template (
+ ev_id bigint not null,
+ ev_time timestamptz not null,
+
+ ev_txid bigint not null default get_current_txid(),
+ ev_owner int4,
+ ev_retry int4,
+
+ ev_type text,
+ ev_data text,
+ ev_extra1 text,
+ ev_extra2 text,
+ ev_extra3 text,
+ ev_extra4 text
+);
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.retry_queue
+--
+-- Events to be retried
+--
+-- Columns:
+-- ev_retry_after - time when it should be re-inserted to main queue
+-- ----------------------------------------------------------------------
+create table pgq.retry_queue (
+ ev_retry_after timestamptz not null,
+
+ like pgq.event_template,
+
+ constraint rq_pkey primary key (ev_owner, ev_id),
+ constraint rq_owner_fkey foreign key (ev_owner)
+ references pgq.subscription (sub_id)
+);
+alter table pgq.retry_queue alter column ev_owner set not null;
+alter table pgq.retry_queue alter column ev_txid drop not null;
+create index rq_retry_idx on pgq.retry_queue (ev_retry_after);
+
+-- ----------------------------------------------------------------------
+-- Table: pgq.failed_queue
+--
+-- Events whose processing failed
+--
+-- Columns:
+-- ev_failed_reason - consumer's excuse for not processing
+-- ev_failed_time - when it was tagged failed
+-- ----------------------------------------------------------------------
+create table pgq.failed_queue (
+ ev_failed_reason text,
+ ev_failed_time timestamptz not null,
+
+ -- all event fields
+ like pgq.event_template,
+
+ constraint fq_pkey primary key (ev_owner, ev_id),
+ constraint fq_owner_fkey foreign key (ev_owner)
+ references pgq.subscription (sub_id)
+);
+alter table pgq.failed_queue alter column ev_owner set not null;
+alter table pgq.failed_queue alter column ev_txid drop not null;
+
+
--- /dev/null
+
+-- Section: Public Triggers
+
+-- Group: Trigger Functions
+
+\i triggers/pgq.logutriga.sql
+\i triggers/pgq.sqltriga.sql
+
--- /dev/null
+
+create type pgq.ret_queue_info as (
+ queue_name text,
+ queue_ntables integer,
+ queue_cur_table integer,
+ queue_rotation_period interval,
+ queue_switch_time timestamptz,
+ queue_external_ticker boolean,
+ queue_ticker_max_count integer,
+ queue_ticker_max_lag interval,
+ queue_ticker_idle_period interval,
+ ticker_lag interval
+);
+
+create type pgq.ret_consumer_info as (
+ queue_name text,
+ consumer_name text,
+ lag interval,
+ last_seen interval
+);
+
+create type pgq.ret_batch_info as (
+ queue_name text,
+ consumer_name text,
+ batch_start timestamptz,
+ batch_end timestamptz,
+ prev_tick_id bigint,
+ tick_id bigint,
+ lag interval
+);
+
+
+create type pgq.ret_batch_event as (
+ ev_id bigint,
+ ev_time timestamptz,
+
+ ev_txid bigint,
+ ev_retry int4,
+
+ ev_type text,
+ ev_data text,
+ ev_extra1 text,
+ ev_extra2 text,
+ ev_extra3 text,
+ ev_extra4 text
+);
+
--- /dev/null
+
+create or replace function pgq.logutriga()
+returns trigger as $$
+# -- ----------------------------------------------------------------------
+# -- Function: pgq.logutriga()
+# --
+# -- Trigger function that puts row data urlencoded into queue.
+# --
+# -- Trigger parameters:
+# -- arg1 - queue name
+# -- arg2 - optionally 'SKIP'
+# --
+# -- Queue event fields:
+# -- ev_type - I/U/D
+# -- ev_data - column values urlencoded
+# -- ev_extra1 - table name
+# -- ev_extra2 - primary key columns
+# --
+# -- Regular listen trigger example:
+# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer
+# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname');
+# --
+# -- Redirect trigger example:
+# -- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer
+# -- > FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('qname', 'SKIP');
+# -- ----------------------------------------------------------------------
+
+# this triger takes 1 or 2 args:
+# queue_name - destination queue
+# option return code (OK, SKIP) SKIP means op won't happen
+# copy-paste of db_urlencode from skytools.quoting
+from urllib import quote_plus
+def db_urlencode(dict):
+ elem_list = []
+ for k, v in dict.items():
+ if v is None:
+ elem = quote_plus(str(k))
+ else:
+ elem = quote_plus(str(k)) + '=' + quote_plus(str(v))
+ elem_list.append(elem)
+ return '&'.join(elem_list)
+
+# load args
+queue_name = TD['args'][0]
+if len(TD['args']) > 1:
+ ret_code = TD['args'][1]
+else:
+ ret_code = 'OK'
+table_oid = TD['relid']
+
+# on first call init plans
+if not 'init_done' in SD:
+ # find table name
+ q = "SELECT n.nspname || '.' || c.relname AS table_name"\
+ " FROM pg_namespace n, pg_class c"\
+ " WHERE n.oid = c.relnamespace AND c.oid = $1"
+ SD['name_plan'] = plpy.prepare(q, ['oid'])
+
+ # find key columns
+ q = "SELECT k.attname FROM pg_index i, pg_attribute k"\
+ " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\
+ " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\
+ " ORDER BY k.attnum"
+ SD['key_plan'] = plpy.prepare(q, ['oid'])
+
+ # insert data
+ q = "SELECT pgq.insert_event($1, $2, $3, $4, $5, null, null)"
+ SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text', 'text'])
+
+ # shorter tags
+ SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'}
+
+ # remember init
+ SD['init_done'] = 1
+
+# load & cache table data
+if table_oid in SD:
+ tbl_name, tbl_keys = SD[table_oid]
+else:
+ res = plpy.execute(SD['name_plan'], [table_oid])
+ tbl_name = res[0]['table_name']
+ res = plpy.execute(SD['key_plan'], [table_oid])
+ tbl_keys = ",".join(map(lambda x: x['attname'], res))
+
+ SD[table_oid] = (tbl_name, tbl_keys)
+
+# prepare args
+if TD['event'] == 'DELETE':
+ data = db_urlencode(TD['old'])
+else:
+ data = db_urlencode(TD['new'])
+
+# insert event
+plpy.execute(SD['ins_plan'], [
+ queue_name,
+ SD['op_map'][TD['event']],
+ data, tbl_name, tbl_keys])
+
+# done
+return ret_code
+
+$$ language plpythonu;
+
--- /dev/null
+
+-- listen trigger:
+-- create trigger triga_nimi after insert or update on customer
+-- for each row execute procedure pgq.sqltriga('qname');
+
+-- redirect trigger:
+-- create trigger triga_nimi after insert or update on customer
+-- for each row execute procedure pgq.sqltriga('qname', 'ret=SKIP');
+
+create or replace function pgq.sqltriga()
+returns trigger as $$
+# -- ----------------------------------------------------------------------
+# -- Function: pgq.sqltriga()
+# --
+# -- Trigger function that puts row data in partial SQL form into queue.
+# --
+# -- Parameters:
+# -- arg1 - queue name
+# -- arg2 - optional urlencoded options
+# --
+# -- Extra options:
+# --
+# -- ret - return value for function OK/SKIP
+# -- pkey - override pkey fields, can be functions
+# -- ignore - comma separated field names to ignore
+# --
+# -- Queue event fields:
+# -- ev_type - I/U/D
+# -- ev_data - partial SQL statement
+# -- ev_extra1 - table name
+# --
+# -- ----------------------------------------------------------------------
+# this triger takes 1 or 2 args:
+# queue_name - destination queue
+# args - urlencoded dict of options:
+# ret - return value: OK/SKIP
+# pkey - comma-separated col names or funcs on cols
+# simple: pkey=user,orderno
+# hashed: pkey=user,hashtext(user)
+# ignore - comma-separated col names to ignore
+
+# on first call init stuff
+if not 'init_done' in SD:
+ # find table name plan
+ q = "SELECT n.nspname || '.' || c.relname AS table_name"\
+ " FROM pg_namespace n, pg_class c"\
+ " WHERE n.oid = c.relnamespace AND c.oid = $1"
+ SD['name_plan'] = plpy.prepare(q, ['oid'])
+
+ # find key columns plan
+ q = "SELECT k.attname FROM pg_index i, pg_attribute k"\
+ " WHERE i.indrelid = $1 AND k.attrelid = i.indexrelid"\
+ " AND i.indisprimary AND k.attnum > 0 AND NOT k.attisdropped"\
+ " ORDER BY k.attnum"
+ SD['key_plan'] = plpy.prepare(q, ['oid'])
+
+ # data insertion
+ q = "SELECT pgq.insert_event($1, $2, $3, $4, null, null, null)"
+ SD['ins_plan'] = plpy.prepare(q, ['text', 'text', 'text', 'text'])
+
+ # shorter tags
+ SD['op_map'] = {'INSERT': 'I', 'UPDATE': 'U', 'DELETE': 'D'}
+
+ # quoting
+ from psycopg import QuotedString
+ def quote(s):
+ if s is None:
+ return "null"
+ s = str(s)
+ return str(QuotedString(s))
+ s = s.replace('\\', '\\\\')
+ s = s.replace("'", "''")
+ return "'%s'" % s
+
+ # TableInfo class
+ import re, urllib
+ class TableInfo:
+ func_rc = re.compile("([^(]+) [(] ([^)]+) [)]", re.I | re.X)
+ def __init__(self, table_oid, options_txt):
+ res = plpy.execute(SD['name_plan'], [table_oid])
+ self.name = res[0]['table_name']
+
+ self.parse_options(options_txt)
+ self.load_pkey()
+
+ def recheck(self, options_txt):
+ if self.options_txt == options_txt:
+ return
+ self.parse_options(options_txt)
+ self.load_pkey()
+
+ def parse_options(self, options_txt):
+ self.options = {'ret': 'OK'}
+ if options_txt:
+ for s in options_txt.split('&'):
+ k, v = s.split('=', 1)
+ self.options[k] = urllib.unquote_plus(v)
+ self.options_txt = options_txt
+
+ def load_pkey(self):
+ self.pkey_list = []
+ if not 'pkey' in self.options:
+ res = plpy.execute(SD['key_plan'], [table_oid])
+ for krow in res:
+ col = krow['attname']
+ expr = col + "=%s"
+ self.pkey_list.append( (col, expr) )
+ else:
+ for a_pk in self.options['pkey'].split(','):
+ m = self.func_rc.match(a_pk)
+ if m:
+ col = m.group(2)
+ fn = m.group(1)
+ expr = "%s(%s) = %s(%%s)" % (fn, col, fn)
+ else:
+ # normal case
+ col = a_pk
+ expr = col + "=%s"
+ self.pkey_list.append( (col, expr) )
+ if len(self.pkey_list) == 0:
+ plpy.error('sqltriga needs primary key on table')
+
+ def get_insert_stmt(self, new):
+ col_list = []
+ val_list = []
+ for k, v in new.items():
+ col_list.append(k)
+ val_list.append(quote(v))
+ return "(%s) values (%s)" % (",".join(col_list), ",".join(val_list))
+
+ def get_update_stmt(self, old, new):
+ chg_list = []
+ for k, v in new.items():
+ ov = old[k]
+ if v == ov:
+ continue
+ chg_list.append("%s=%s" % (k, quote(v)))
+ if len(chg_list) == 0:
+ pk = self.pkey_list[0][0]
+ chg_list.append("%s=%s" % (pk, quote(new[pk])))
+ return "%s where %s" % (",".join(chg_list), self.get_pkey_expr(new))
+
+ def get_pkey_expr(self, data):
+ exp_list = []
+ for col, exp in self.pkey_list:
+ exp_list.append(exp % quote(data[col]))
+ return " and ".join(exp_list)
+
+ SD['TableInfo'] = TableInfo
+
+ # cache some functions
+ def proc_insert(tbl):
+ return tbl.get_insert_stmt(TD['new'])
+ def proc_update(tbl):
+ return tbl.get_update_stmt(TD['old'], TD['new'])
+ def proc_delete(tbl):
+ return tbl.get_pkey_expr(TD['old'])
+ SD['event_func'] = {
+ 'I': proc_insert,
+ 'U': proc_update,
+ 'D': proc_delete,
+ }
+
+ # remember init
+ SD['init_done'] = 1
+
+
+# load args
+table_oid = TD['relid']
+queue_name = TD['args'][0]
+if len(TD['args']) > 1:
+ options_str = TD['args'][1]
+else:
+ options_str = ''
+
+# load & cache table data
+if table_oid in SD:
+ tbl = SD[table_oid]
+ tbl.recheck(options_str)
+else:
+ tbl = SD['TableInfo'](table_oid, options_str)
+ SD[table_oid] = tbl
+
+# generate payload
+op = SD['op_map'][TD['event']]
+data = SD['event_func'][op](tbl)
+
+# insert event
+plpy.execute(SD['ins_plan'], [queue_name, op, data, tbl.name])
+
+# done
+return tbl.options['ret']
+
+$$ language plpythonu;
+
--- /dev/null
+
+DOCS = README.pgq_ext
+DATA_built = pgq_ext.sql
+
+SRCS = structure/tables.sql functions/track_batch.sql functions/track_event.sql
+
+REGRESS = test_pgq_ext
+REGRESS_OPTS = --load-language=plpgsql
+
+include ../../config.mak
+
+include $(PGXS)
+
+pgq_ext.sql: $(SRCS)
+ cat $(SRCS) > $@
+
--- /dev/null
+
+Track processed batches and events in target DB
+================================================
+
+Batch tracking is OK.
+
+Event tracking is OK if consumer does not use retry queue.
+
+Batch tracking
+--------------
+
+is_batch_done(consumer, batch)
+
+returns:
+
+ true - batch is done already
+ false - batch is not done yet
+
+set_batch_done(consumer, batch)
+
+returns:
+
+ true - tagging successful, batch was not done yet
+ false - batch was done already
+
+Event tracking
+--------------
+
+is_batch_done(consumer, batch, event)
+
+returns:
+
+ true - event is done
+ false - event is not done yet
+
+
+set_batch_done(consumer, batch, event)
+
+returns:
+
+ true - tagging was successful, event was not done
+ false - event is done already
+
+
+Fastvacuum
+----------
+
+pgq.ext.completed_batch
+pgq.ext.completed_event
+pgq.ext.completed_tick
+pgq.ext.partial_batch
+
--- /dev/null
+\set ECHO off
+--
+-- test batch tracking
+--
+select pgq_ext.is_batch_done('c', 1);
+ is_batch_done
+---------------
+ f
+(1 row)
+
+select pgq_ext.set_batch_done('c', 1);
+ set_batch_done
+----------------
+ t
+(1 row)
+
+select pgq_ext.is_batch_done('c', 1);
+ is_batch_done
+---------------
+ t
+(1 row)
+
+select pgq_ext.set_batch_done('c', 1);
+ set_batch_done
+----------------
+ f
+(1 row)
+
+select pgq_ext.is_batch_done('c', 2);
+ is_batch_done
+---------------
+ f
+(1 row)
+
+select pgq_ext.set_batch_done('c', 2);
+ set_batch_done
+----------------
+ t
+(1 row)
+
+--
+-- test event tracking
+--
+select pgq_ext.is_batch_done('c', 3);
+ is_batch_done
+---------------
+ f
+(1 row)
+
+select pgq_ext.is_event_done('c', 3, 101);
+ is_event_done
+---------------
+ f
+(1 row)
+
+select pgq_ext.set_event_done('c', 3, 101);
+ set_event_done
+----------------
+ t
+(1 row)
+
+select pgq_ext.is_event_done('c', 3, 101);
+ is_event_done
+---------------
+ t
+(1 row)
+
+select pgq_ext.set_event_done('c', 3, 101);
+ set_event_done
+----------------
+ f
+(1 row)
+
+select pgq_ext.set_batch_done('c', 3);
+ set_batch_done
+----------------
+ t
+(1 row)
+
+select * from pgq_ext.completed_event order by 1,2;
+ consumer_id | batch_id | event_id
+-------------+----------+----------
+ c | 3 | 101
+(1 row)
+
--- /dev/null
+
+create or replace function pgq_ext.is_batch_done(
+ a_consumer text, a_batch_id bigint)
+returns boolean as $$
+declare
+ res boolean;
+begin
+ select last_batch_id = a_batch_id
+ into res from pgq_ext.completed_batch
+ where consumer_id = a_consumer;
+ if not found then
+ return false;
+ end if;
+ return res;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq_ext.set_batch_done(
+ a_consumer text, a_batch_id bigint)
+returns boolean as $$
+begin
+ if pgq_ext.is_batch_done(a_consumer, a_batch_id) then
+ return false;
+ end if;
+
+ if a_batch_id > 0 then
+ update pgq_ext.completed_batch
+ set last_batch_id = a_batch_id
+ where consumer_id = a_consumer;
+ if not found then
+ insert into pgq_ext.completed_batch (consumer_id, last_batch_id)
+ values (a_consumer, a_batch_id);
+ end if;
+ end if;
+
+ return true;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+
+create or replace function pgq_ext.is_event_done(
+ a_consumer text,
+ a_batch_id bigint, a_event_id bigint)
+returns boolean as $$
+declare
+ res bigint;
+begin
+ perform 1 from pgq_ext.completed_event
+ where consumer_id = a_consumer
+ and batch_id = a_batch_id
+ and event_id = a_event_id;
+ return found;
+end;
+$$ language plpgsql security definer;
+
+create or replace function pgq_ext.set_event_done(
+ a_consumer text, a_batch_id bigint, a_event_id bigint)
+returns boolean as $$
+declare
+ old_batch bigint;
+begin
+ -- check if done
+ perform 1 from pgq_ext.completed_event
+ where consumer_id = a_consumer
+ and batch_id = a_batch_id
+ and event_id = a_event_id;
+ if found then
+ return false;
+ end if;
+
+ -- if batch changed, do cleanup
+ select cur_batch_id into old_batch
+ from pgq_ext.partial_batch
+ where consumer_id = a_consumer;
+ if not found then
+ -- first time here
+ insert into pgq_ext.partial_batch
+ (consumer_id, cur_batch_id)
+ values (a_consumer, a_batch_id);
+ elsif old_batch <> a_batch_id then
+ -- batch changed, that means old is finished on queue db
+ -- thus the tagged events are not needed anymore
+ delete from pgq_ext.completed_event
+ where consumer_id = a_consumer
+ and batch_id = old_batch;
+ -- remember current one
+ update pgq_ext.partial_batch
+ set cur_batch_id = a_batch_id
+ where consumer_id = a_consumer;
+ end if;
+
+ -- tag as done
+ insert into pgq_ext.completed_event (consumer_id, batch_id, event_id)
+ values (a_consumer, a_batch_id, a_event_id);
+
+ return true;
+end;
+$$ language plpgsql security definer;
+
--- /dev/null
+\set ECHO off
+\i pgq_ext.sql
+\set ECHO all
+
+--
+-- test batch tracking
+--
+select pgq_ext.is_batch_done('c', 1);
+select pgq_ext.set_batch_done('c', 1);
+select pgq_ext.is_batch_done('c', 1);
+select pgq_ext.set_batch_done('c', 1);
+select pgq_ext.is_batch_done('c', 2);
+select pgq_ext.set_batch_done('c', 2);
+
+--
+-- test event tracking
+--
+select pgq_ext.is_batch_done('c', 3);
+select pgq_ext.is_event_done('c', 3, 101);
+select pgq_ext.set_event_done('c', 3, 101);
+select pgq_ext.is_event_done('c', 3, 101);
+select pgq_ext.set_event_done('c', 3, 101);
+select pgq_ext.set_batch_done('c', 3);
+select * from pgq_ext.completed_event order by 1,2;
+
+
--- /dev/null
+
+set client_min_messages = 'warning';
+set default_with_oids = 'off';
+
+create schema pgq_ext;
+grant usage on schema pgq_ext to public;
+
+
+--
+-- batch tracking
+--
+create table pgq_ext.completed_batch (
+ consumer_id text not null,
+ last_batch_id bigint not null,
+
+ primary key (consumer_id)
+);
+
+
+--
+-- event tracking
+--
+create table pgq_ext.completed_event (
+ consumer_id text not null,
+ batch_id bigint not null,
+ event_id bigint not null,
+
+ primary key (consumer_id, batch_id, event_id)
+);
+
+create table pgq_ext.partial_batch (
+ consumer_id text not null,
+ cur_batch_id bigint not null,
+
+ primary key (consumer_id)
+);
+
+--
+-- tick tracking for SerialConsumer()
+-- no access functions provided here
+--
+create table pgq_ext.completed_tick (
+ consumer_id text not null,
+ last_tick_id bigint not null,
+
+ primary key (consumer_id)
+);
+
--- /dev/null
+
+MODULE_big = txid
+SRCS = txid.c epoch.c
+OBJS = $(SRCS:.c=.o)
+
+DATA_built = txid.sql
+DATA = uninstall_txid.sql
+DOCS = README.txid
+EXTRA_CLEAN = txid.sql.in
+
+REGRESS = txid
+REGRESS_OPTS = --load-language=plpgsql
+
+include ../../config.mak
+include $(PGXS)
+
+# additional deps
+txid.o: txid.h
+epoch.o: txid.h
+
+# postgres >= manages epoch itself, so skip epoch tables
+pgnew = $(shell test $(VERSION) "<" "8.2" && echo "false" || echo "true")
+ifeq ($(pgnew),true)
+TXID_SQL = txid.std.sql
+else
+TXID_SQL = txid.std.sql txid.schema.sql
+endif
+txid.sql.in: $(TXID_SQL)
+ cat $(TXID_SQL) > $@
+
--- /dev/null
+
+txid - 8 byte transaction ID's
+==============================
+
+Based on xxid module from Slony-I.
+
+The goal is to make PostgreSQL internal transaction ID and snapshot
+data usable externally. They cannot be used directly as the
+internal 4-byte value wraps around and thus breaks indexing.
+
+This module extends the internal value with wraparound cound (epoch).
+It uses relaxed method for wraparound check. There is a table
+txid.epoch (epoch, last_value) which is used to check if the xid
+is in current, next or previous epoch. It requires only occasional
+read-write access - ca. after 100k - 500k transactions.
+
+Also it contains type 'txid_snapshot' and following functions:
+
+
+get_current_txid() returns int8
+
+ Current transaction ID
+
+get_current_snapshot() returns txid_snapshot
+
+ Current snapshot
+
+get_snapshot_xmin( snap ) returns int8 --
+
+ Smallest TXID in snapshot. TXID's smaller than this
+ are all visible in snapshot.
+
+get_snapshot_xmax( snap ) returns int8
+
+ Largest TXID in snapshot. TXID's starting from this one are
+ all invisible in snapshot.
+
+get_snapshot_values( snap ) setof int8
+
+ List of uncommitted TXID's in snapshot, that are invisible
+ in snapshot. Values are between xmin and xmax.
+
+txid_in_snapshot(id, snap) returns bool
+
+ Is TXID visible in snapshot?
+
+txid_not_in_snapshot(id, snap) returns bool
+
+ Is TXID invisible in snapshot?
+
+
+Problems
+--------
+
+- it breaks when there are more than 2G tx'es between calls.
+ Fixed in 8.2
+
+- functions that create new txid's should be 'security definers'
+ thus better protecting txid_epoch table.
+
+- After loading database from backup you should do:
+
+ UPDATE txid.epoch SET epoch = epoch + 1,
+ last_value = (get_current_txid() & 4294967295);
+
--- /dev/null
+/*-------------------------------------------------------------------------
+ * epoch.c
+ *
+ * Detect current epoch.
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <limits.h>
+
+#include "access/transam.h"
+#include "executor/spi.h"
+#include "miscadmin.h"
+#include "catalog/pg_control.h"
+#include "access/xlog.h"
+
+#include "txid.h"
+
+/*
+ * do a TransactionId -> txid conversion
+ */
+txid txid_convert_xid(TransactionId xid, TxidEpoch *state)
+{
+ uint64 epoch;
+
+ /* avoid issues with the the special meaning of 0 */
+ if (xid == InvalidTransactionId)
+ return MAX_INT64;
+
+ /* return special xid's as-is */
+ if (xid < FirstNormalTransactionId)
+ return xid;
+
+ /* xid can on both sides on wrap-around */
+ epoch = state->epoch;
+ if (TransactionIdPrecedes(xid, state->last_value)) {
+ if (xid > state->last_value)
+ epoch--;
+ } else if (TransactionIdFollows(xid, state->last_value)) {
+ if (xid < state->last_value)
+ epoch++;
+ }
+ return (epoch << 32) | xid;
+}
+
+#if PG_CONTROL_VERSION >= 820
+
+/*
+ * PostgreSQl 8.2 keeps track of epoch internally.
+ */
+
+void txid_load_epoch(TxidEpoch *state, int try_write)
+{
+ TransactionId xid;
+ uint32 epoch;
+
+ GetNextXidAndEpoch(&xid, &epoch);
+
+ state->epoch = epoch;
+ state->last_value = xid;
+}
+
+#else
+
+/*
+ * For older PostgreSQL keep epoch in table.
+ */
+
+/*
+ * this caches the txid_epoch table.
+ * The struct should be updated only together with the table.
+ */
+static TxidEpoch epoch_state = { 0, 0 };
+
+/*
+ * load values from txid_epoch table.
+ */
+static int load_epoch(void)
+{
+ HeapTuple row;
+ TupleDesc rdesc;
+ bool isnull = false;
+ Datum tmp;
+ int res;
+ uint64 db_epoch, db_value;
+
+ res = SPI_connect();
+ if (res < 0)
+ elog(ERROR, "cannot connect to SPI");
+
+ res = SPI_execute("select epoch, last_value from txid.epoch", true, 0);
+ if (res != SPI_OK_SELECT)
+ elog(ERROR, "load_epoch: select failed?");
+ if (SPI_processed != 1)
+ elog(ERROR, "load_epoch: there must be exactly 1 row");
+
+ row = SPI_tuptable->vals[0];
+ rdesc = SPI_tuptable->tupdesc;
+
+ tmp = SPI_getbinval(row, rdesc, 1, &isnull);
+ if (isnull)
+ elog(ERROR, "load_epoch: epoch is NULL");
+ db_epoch = DatumGetInt64(tmp);
+
+ tmp = SPI_getbinval(row, rdesc, 2, &isnull);
+ if (isnull)
+ elog(ERROR, "load_epoch: last_value is NULL");
+ db_value = DatumGetInt64(tmp);
+
+ SPI_finish();
+
+ /*
+ * If the db has lesser values, then some updates were lost.
+ *
+ * Should that be special-cased? ATM just use db values.
+ * Thus immidiate update.
+ */
+ epoch_state.epoch = db_epoch;
+ epoch_state.last_value = db_value;
+ return 1;
+}
+
+/*
+ * updates last_value and epoch, if needed
+ */
+static void save_epoch(void)
+{
+ int res;
+ char qbuf[200];
+ uint64 new_epoch, new_value;
+ TransactionId xid = GetTopTransactionId();
+ TransactionId old_value;
+
+ /* store old state */
+ MemoryContext oldcontext = CurrentMemoryContext;
+ ResourceOwner oldowner = CurrentResourceOwner;
+
+ /*
+ * avoid changing internal values.
+ */
+ new_value = xid;
+ new_epoch = epoch_state.epoch;
+ old_value = (TransactionId)epoch_state.last_value;
+ if (xid < old_value) {
+ if (TransactionIdFollows(xid, old_value))
+ new_epoch++;
+ else
+ return;
+ }
+ sprintf(qbuf, "update txid.epoch set epoch = %llu, last_value = %llu",
+ (unsigned long long)new_epoch,
+ (unsigned long long)new_value);
+
+ /*
+ * The update may fail in case of SERIALIZABLE transaction.
+ * Try to catch the error and hide it.
+ */
+ BeginInternalSubTransaction(NULL);
+ PG_TRY();
+ {
+ /* do the update */
+ res = SPI_connect();
+ if (res < 0)
+ elog(ERROR, "cannot connect to SPI");
+ res = SPI_execute(qbuf, false, 0);
+ SPI_finish();
+
+ ReleaseCurrentSubTransaction();
+ }
+ PG_CATCH();
+ {
+ /* we expect rollback to clean up inner SPI call */
+ RollbackAndReleaseCurrentSubTransaction();
+ FlushErrorState();
+ res = -1; /* remember failure */
+ }
+ PG_END_TRY();
+
+ /* restore old state */
+ MemoryContextSwitchTo(oldcontext);
+ CurrentResourceOwner = oldowner;
+
+ if (res < 0)
+ return;
+
+ /*
+ * Seems the update was successful, update internal state too.
+ *
+ * There is a chance that the TX will be rollbacked, but then
+ * another backend will do the update, or this one at next
+ * checkpoint.
+ */
+ epoch_state.epoch = new_epoch;
+ epoch_state.last_value = new_value;
+}
+
+static void check_epoch(int update_prio)
+{
+ TransactionId xid = GetTopTransactionId();
+ TransactionId recheck, tx_next;
+ int ok = 1;
+
+ /* should not happen, but just in case */
+ if (xid == InvalidTransactionId)
+ return;
+
+ /* new backend */
+ if (epoch_state.last_value == 0)
+ load_epoch();
+
+ /* try to avoid concurrent access */
+ if (update_prio)
+ recheck = 50000 + 100 * (MyProcPid & 0x1FF);
+ else
+ recheck = 300000 + 1000 * (MyProcPid & 0x1FF);
+
+ /* read table */
+ tx_next = (TransactionId)epoch_state.last_value + recheck;
+ if (TransactionIdFollows(xid, tx_next))
+ ok = load_epoch();
+
+ /*
+ * check if save is needed. last_value may be updated above.
+ */
+ tx_next = (TransactionId)epoch_state.last_value + recheck;
+ if (!ok || TransactionIdFollows(xid, tx_next))
+ save_epoch();
+}
+
+void txid_load_epoch(TxidEpoch *state, int try_write)
+{
+ check_epoch(try_write);
+
+ state->epoch = epoch_state.epoch;
+ state->last_value = epoch_state.last_value;
+}
+
+
+#endif
--- /dev/null
+-- init
+\set ECHO none
+-- i/o
+select '12:13:'::txid_snapshot;
+ txid_snapshot
+---------------
+ 12:13:
+(1 row)
+
+select '12:13:1,2'::txid_snapshot;
+ERROR: illegal txid_snapshot input format
+-- errors
+select '31:12:'::txid_snapshot;
+ERROR: illegal txid_snapshot input format
+select '0:1:'::txid_snapshot;
+ERROR: illegal txid_snapshot input format
+select '12:13:0'::txid_snapshot;
+ERROR: illegal txid_snapshot input format
+select '12:13:2,1'::txid_snapshot;
+ERROR: illegal txid_snapshot input format
+create table snapshot_test (
+ nr integer,
+ snap txid_snapshot
+);
+insert into snapshot_test values (1, '12:13:');
+insert into snapshot_test values (2, '12:20:13,15,18');
+insert into snapshot_test values (3, '100001:100009:100005,100007,100008');
+select snap from snapshot_test order by nr;
+ snap
+------------------------------------
+ 12:13:
+ 12:20:13,15,18
+ 100001:100009:100005,100007,100008
+(3 rows)
+
+select get_snapshot_xmin(snap),
+ get_snapshot_xmax(snap),
+ get_snapshot_active(snap)
+from snapshot_test order by nr;
+ get_snapshot_xmin | get_snapshot_xmax | get_snapshot_active
+-------------------+-------------------+---------------------
+ 12 | 20 | 13
+ 12 | 20 | 15
+ 12 | 20 | 18
+ 100001 | 100009 | 100005
+ 100001 | 100009 | 100007
+ 100001 | 100009 | 100008
+(6 rows)
+
+select id, txid_in_snapshot(id, snap),
+ txid_not_in_snapshot(id, snap)
+from snapshot_test, generate_series(11, 21) id
+where nr = 2;
+ id | txid_in_snapshot | txid_not_in_snapshot
+----+------------------+----------------------
+ 11 | t | f
+ 12 | t | f
+ 13 | f | t
+ 14 | t | f
+ 15 | f | t
+ 16 | t | f
+ 17 | t | f
+ 18 | f | t
+ 19 | t | f
+ 20 | f | t
+ 21 | f | t
+(11 rows)
+
+-- test current values also
+select get_current_txid() >= get_snapshot_xmin(get_current_snapshot());
+ ?column?
+----------
+ t
+(1 row)
+
+select get_current_txid() < get_snapshot_xmax(get_current_snapshot());
+ ?column?
+----------
+ t
+(1 row)
+
+select txid_in_snapshot(get_current_txid(), get_current_snapshot()),
+ txid_not_in_snapshot(get_current_txid(), get_current_snapshot());
+ txid_in_snapshot | txid_not_in_snapshot
+------------------+----------------------
+ t | f
+(1 row)
+
--- /dev/null
+-- init
+\set ECHO none
+\i txid.sql
+\set ECHO all
+
+-- i/o
+select '12:13:'::txid_snapshot;
+select '12:13:1,2'::txid_snapshot;
+
+-- errors
+select '31:12:'::txid_snapshot;
+select '0:1:'::txid_snapshot;
+select '12:13:0'::txid_snapshot;
+select '12:13:2,1'::txid_snapshot;
+
+create table snapshot_test (
+ nr integer,
+ snap txid_snapshot
+);
+
+insert into snapshot_test values (1, '12:13:');
+insert into snapshot_test values (2, '12:20:13,15,18');
+insert into snapshot_test values (3, '100001:100009:100005,100007,100008');
+
+select snap from snapshot_test order by nr;
+
+select get_snapshot_xmin(snap),
+ get_snapshot_xmax(snap),
+ get_snapshot_active(snap)
+from snapshot_test order by nr;
+
+select id, txid_in_snapshot(id, snap),
+ txid_not_in_snapshot(id, snap)
+from snapshot_test, generate_series(11, 21) id
+where nr = 2;
+
+-- test current values also
+select get_current_txid() >= get_snapshot_xmin(get_current_snapshot());
+select get_current_txid() < get_snapshot_xmax(get_current_snapshot());
+
+select txid_in_snapshot(get_current_txid(), get_current_snapshot()),
+ txid_not_in_snapshot(get_current_txid(), get_current_snapshot());
+
--- /dev/null
+/*-------------------------------------------------------------------------
+ * txid.c
+ *
+ * Safe handling of transaction ID's.
+ *
+ * Copyright (c) 2003-2004, PostgreSQL Global Development Group
+ * Author: Jan Wieck, Afilias USA INC.
+ *
+ * 64-bit output: Marko Kreen, Skype Technologies
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <limits.h>
+
+#include "access/xact.h"
+#include "funcapi.h"
+
+#include "txid.h"
+
+#ifdef INT64_IS_BUSTED
+#error txid needs working int64
+#endif
+
+#ifdef PG_MODULE_MAGIC
+PG_MODULE_MAGIC;
+#endif
+
+/*
+ * public functions
+ */
+
+PG_FUNCTION_INFO_V1(txid_current);
+PG_FUNCTION_INFO_V1(txid_snapshot_in);
+PG_FUNCTION_INFO_V1(txid_snapshot_out);
+PG_FUNCTION_INFO_V1(txid_in_snapshot);
+PG_FUNCTION_INFO_V1(txid_not_in_snapshot);
+PG_FUNCTION_INFO_V1(txid_current_snapshot);
+PG_FUNCTION_INFO_V1(txid_snapshot_xmin);
+PG_FUNCTION_INFO_V1(txid_snapshot_xmax);
+PG_FUNCTION_INFO_V1(txid_snapshot_active);
+
+/*
+ * utility functions
+ */
+
+static int _cmp_txid(const void *aa, const void *bb)
+{
+ const uint64 *a = aa;
+ const uint64 *b = bb;
+ if (*a < *b)
+ return -1;
+ if (*a > *b)
+ return 1;
+ return 0;
+}
+
+static void sort_snapshot(TxidSnapshot *snap)
+{
+ qsort(snap->xip, snap->nxip, sizeof(txid), _cmp_txid);
+}
+
+static TxidSnapshot *
+parse_snapshot(const char *str)
+{
+ int a_size;
+ txid *xip;
+
+ int a_used = 0;
+ txid xmin;
+ txid xmax;
+ txid last_val = 0, val;
+ TxidSnapshot *snap;
+ int size;
+
+ char *endp;
+
+ a_size = 1024;
+ xip = (txid *) palloc(sizeof(txid) * a_size);
+
+ xmin = (txid) strtoull(str, &endp, 0);
+ if (*endp != ':')
+ elog(ERROR, "illegal txid_snapshot input format");
+ str = endp + 1;
+
+ xmax = (txid) strtoull(str, &endp, 0);
+ if (*endp != ':')
+ elog(ERROR, "illegal txid_snapshot input format");
+ str = endp + 1;
+
+ /* it should look sane */
+ if (xmin >= xmax || xmin > MAX_INT64 || xmax > MAX_INT64
+ || xmin == 0 || xmax == 0)
+ elog(ERROR, "illegal txid_snapshot input format");
+
+ while (*str != '\0')
+ {
+ if (a_used >= a_size)
+ {
+ a_size *= 2;
+ xip = (txid *) repalloc(xip, sizeof(txid) * a_size);
+ }
+
+ /* read next value */
+ if (*str == '\'')
+ {
+ str++;
+ val = (txid) strtoull(str, &endp, 0);
+ if (*endp != '\'')
+ elog(ERROR, "illegal txid_snapshot input format");
+ str = endp + 1;
+ }
+ else
+ {
+ val = (txid) strtoull(str, &endp, 0);
+ str = endp;
+ }
+
+ /* require the input to be in order */
+ if (val < xmin || val <= last_val || val >= xmax)
+ elog(ERROR, "illegal txid_snapshot input format");
+
+ xip[a_used++] = val;
+ last_val = val;
+
+ if (*str == ',')
+ str++;
+ else
+ {
+ if (*str != '\0')
+ elog(ERROR, "illegal txid_snapshot input format");
+ }
+ }
+
+ size = offsetof(TxidSnapshot, xip) + sizeof(txid) * a_used;
+ snap = (TxidSnapshot *) palloc(size);
+ snap->varsz = size;
+ snap->xmin = xmin;
+ snap->xmax = xmax;
+ snap->nxip = a_used;
+ if (a_used > 0)
+ memcpy(&(snap->xip[0]), xip, sizeof(txid) * a_used);
+ pfree(xip);
+
+ return snap;
+}
+
+/*
+ * Public functions
+ */
+
+/*
+ * txid_current - Return the current transaction ID as txid
+ */
+Datum
+txid_current(PG_FUNCTION_ARGS)
+{
+ txid val;
+ TxidEpoch state;
+
+ txid_load_epoch(&state, 0);
+
+ val = txid_convert_xid(GetTopTransactionId(), &state);
+
+ PG_RETURN_INT64(val);
+}
+
+/*
+ * txid_current_snapshot - return current snapshot
+ */
+Datum
+txid_current_snapshot(PG_FUNCTION_ARGS)
+{
+ TxidSnapshot *snap;
+ unsigned num, i, size;
+ TxidEpoch state;
+
+ if (SerializableSnapshot == NULL)
+ elog(ERROR, "get_current_snapshot: SerializableSnapshot == NULL");
+
+ txid_load_epoch(&state, 1);
+
+ num = SerializableSnapshot->xcnt;
+ size = offsetof(TxidSnapshot, xip) + sizeof(txid) * num;
+ snap = palloc(size);
+ snap->varsz = size;
+ snap->xmin = txid_convert_xid(SerializableSnapshot->xmin, &state);
+ snap->xmax = txid_convert_xid(SerializableSnapshot->xmax, &state);
+ snap->nxip = num;
+ for (i = 0; i < num; i++)
+ snap->xip[i] = txid_convert_xid(SerializableSnapshot->xip[i], &state);
+
+ /* we want then guaranteed ascending order */
+ sort_snapshot(snap);
+
+ PG_RETURN_POINTER(snap);
+}
+
+/*
+ * txid_snapshot_in - input function for type txid_snapshot
+ */
+Datum
+txid_snapshot_in(PG_FUNCTION_ARGS)
+{
+ TxidSnapshot *snap;
+ char *str = PG_GETARG_CSTRING(0);
+
+ snap = parse_snapshot(str);
+ PG_RETURN_POINTER(snap);
+}
+
+/*
+ * txid_snapshot_out - output function for type txid_snapshot
+ */
+Datum
+txid_snapshot_out(PG_FUNCTION_ARGS)
+{
+ TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0);
+
+ char *str = palloc(60 + snap->nxip * 30);
+ char *cp = str;
+ int i;
+
+ snprintf(str, 60, "%llu:%llu:",
+ (unsigned long long)snap->xmin,
+ (unsigned long long)snap->xmax);
+ cp = str + strlen(str);
+
+ for (i = 0; i < snap->nxip; i++)
+ {
+ snprintf(cp, 30, "%llu%s",
+ (unsigned long long)snap->xip[i],
+ (i < snap->nxip - 1) ? "," : "");
+ cp += strlen(cp);
+ }
+
+ PG_RETURN_CSTRING(str);
+}
+
+
+/*
+ * txid_in_snapshot - is txid visible in snapshot ?
+ */
+Datum
+txid_in_snapshot(PG_FUNCTION_ARGS)
+{
+ txid value = PG_GETARG_INT64(0);
+ TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1);
+ int i;
+ int res = true;
+
+ if (value < snap->xmin)
+ res = true;
+ else if (value >= snap->xmax)
+ res = false;
+ else
+ {
+ for (i = 0; i < snap->nxip; i++)
+ if (value == snap->xip[i])
+ {
+ res = false;
+ break;
+ }
+ }
+ PG_FREE_IF_COPY(snap, 1);
+ PG_RETURN_BOOL(res);
+}
+
+
+/*
+ * txid_not_in_snapshot - is txid invisible in snapshot ?
+ */
+Datum
+txid_not_in_snapshot(PG_FUNCTION_ARGS)
+{
+ txid value = PG_GETARG_INT64(0);
+ TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(1);
+ int i;
+ int res = false;
+
+ if (value < snap->xmin)
+ res = false;
+ else if (value >= snap->xmax)
+ res = true;
+ else
+ {
+ for (i = 0; i < snap->nxip; i++)
+ if (value == snap->xip[i])
+ {
+ res = true;
+ break;
+ }
+ }
+ PG_FREE_IF_COPY(snap, 1);
+ PG_RETURN_BOOL(res);
+}
+
+/*
+ * txid_snapshot_xmin - return snapshot's xmin
+ */
+Datum
+txid_snapshot_xmin(PG_FUNCTION_ARGS)
+{
+ TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0);
+ txid res = snap->xmin;
+ PG_FREE_IF_COPY(snap, 0);
+ PG_RETURN_INT64(res);
+}
+
+/*
+ * txid_snapshot_xmin - return snapshot's xmax
+ */
+Datum
+txid_snapshot_xmax(PG_FUNCTION_ARGS)
+{
+ TxidSnapshot *snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0);
+ txid res = snap->xmax;
+ PG_FREE_IF_COPY(snap, 0);
+ PG_RETURN_INT64(res);
+}
+
+/* remember state between function calls */
+struct snap_state {
+ int pos;
+ TxidSnapshot *snap;
+};
+
+/*
+ * txid_snapshot_active - returns uncommitted TXID's in snapshot.
+ */
+Datum
+txid_snapshot_active(PG_FUNCTION_ARGS)
+{
+ FuncCallContext *fctx;
+ struct snap_state *state;
+
+ if (SRF_IS_FIRSTCALL()) {
+ TxidSnapshot *snap;
+ int statelen;
+
+ snap = (TxidSnapshot *) PG_GETARG_VARLENA_P(0);
+
+ fctx = SRF_FIRSTCALL_INIT();
+ statelen = sizeof(*state) + snap->varsz;
+ state = MemoryContextAlloc(fctx->multi_call_memory_ctx, statelen);
+ state->pos = 0;
+ state->snap = (TxidSnapshot *)((char *)state + sizeof(*state));
+ memcpy(state->snap, snap, snap->varsz);
+ fctx->user_fctx = state;
+
+ PG_FREE_IF_COPY(snap, 0);
+ }
+ fctx = SRF_PERCALL_SETUP();
+ state = fctx->user_fctx;
+ if (state->pos < state->snap->nxip) {
+ Datum res = Int64GetDatum(state->snap->xip[state->pos]);
+ state->pos++;
+ SRF_RETURN_NEXT(fctx, res);
+ } else {
+ SRF_RETURN_DONE(fctx);
+ }
+}
+
--- /dev/null
+#ifndef _TXID_H_
+#define _TXID_H_
+
+#define MAX_INT64 0x7FFFFFFFFFFFFFFFLL
+
+/* Use unsigned variant internally */
+typedef uint64 txid;
+
+typedef struct
+{
+ int32 varsz;
+ uint32 nxip;
+ txid xmin;
+ txid xmax;
+ txid xip[1];
+} TxidSnapshot;
+
+
+typedef struct {
+ uint64 last_value;
+ uint64 epoch;
+} TxidEpoch;
+
+/* internal functions */
+void txid_load_epoch(TxidEpoch *state, int try_write);
+txid txid_convert_xid(TransactionId xid, TxidEpoch *state);
+
+/* public functions */
+Datum txid_current(PG_FUNCTION_ARGS);
+Datum txid_current_snapshot(PG_FUNCTION_ARGS);
+
+Datum txid_snapshot_in(PG_FUNCTION_ARGS);
+Datum txid_snapshot_out(PG_FUNCTION_ARGS);
+
+Datum txid_in_snapshot(PG_FUNCTION_ARGS);
+Datum txid_not_in_snapshot(PG_FUNCTION_ARGS);
+Datum txid_snapshot_xmin(PG_FUNCTION_ARGS);
+Datum txid_snapshot_xmax(PG_FUNCTION_ARGS);
+Datum txid_snapshot_active(PG_FUNCTION_ARGS);
+
+
+#endif /* _TXID_H_ */
+
--- /dev/null
+-- ----------
+-- txid.sql
+--
+-- SQL script for loading the transaction ID compatible datatype
+--
+-- Copyright (c) 2003-2004, PostgreSQL Global Development Group
+-- Author: Jan Wieck, Afilias USA INC.
+--
+-- ----------
+
+--
+-- now the epoch storage
+--
+
+CREATE SCHEMA txid;
+
+-- remember txid settings
+-- use bigint so we can do arithmetic with it
+create table txid.epoch (
+ epoch bigint,
+ last_value bigint
+);
+
+-- make sure there exist exactly one row
+insert into txid.epoch values (0, 1);
+
+
+-- then protect it
+create function txid.epoch_guard()
+returns trigger as $$
+begin
+ if TG_OP = 'UPDATE' then
+ -- epoch: allow only small increase
+ if NEW.epoch > OLD.epoch and NEW.epoch < (OLD.epoch + 3) then
+ return NEW;
+ end if;
+ -- last_value: allow only increase
+ if NEW.epoch = OLD.epoch and NEW.last_value > OLD.last_value then
+ return NEW;
+ end if;
+ end if;
+ raise exception 'bad operation on txid.epoch';
+end;
+$$ language plpgsql;
+
+-- the trigger
+create trigger epoch_guard_trigger
+before insert or update or delete on txid.epoch
+for each row execute procedure txid.epoch_guard();
+
--- /dev/null
+-- ----------
+-- txid.sql
+--
+-- SQL script for loading the transaction ID compatible datatype
+--
+-- Copyright (c) 2003-2004, PostgreSQL Global Development Group
+-- Author: Jan Wieck, Afilias USA INC.
+--
+-- ----------
+
+set client_min_messages = 'warning';
+
+CREATE DOMAIN txid AS bigint CHECK (value > 0);
+
+--
+-- A special transaction snapshot data type for faster visibility checks
+--
+CREATE OR REPLACE FUNCTION txid_snapshot_in(cstring)
+ RETURNS txid_snapshot
+ AS 'MODULE_PATHNAME' LANGUAGE C
+ IMMUTABLE STRICT;
+CREATE OR REPLACE FUNCTION txid_snapshot_out(txid_snapshot)
+ RETURNS cstring
+ AS 'MODULE_PATHNAME' LANGUAGE C
+ IMMUTABLE STRICT;
+
+--
+-- The data type itself
+--
+CREATE TYPE txid_snapshot (
+ INPUT = txid_snapshot_in,
+ OUTPUT = txid_snapshot_out,
+ INTERNALLENGTH = variable,
+ STORAGE = extended,
+ ALIGNMENT = double
+);
+
+CREATE OR REPLACE FUNCTION get_current_txid()
+ RETURNS bigint
+ AS 'MODULE_PATHNAME', 'txid_current' LANGUAGE C
+ SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION get_current_snapshot()
+ RETURNS txid_snapshot
+ AS 'MODULE_PATHNAME', 'txid_current_snapshot' LANGUAGE C
+ SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION get_snapshot_xmin(txid_snapshot)
+ RETURNS bigint
+ AS 'MODULE_PATHNAME', 'txid_snapshot_xmin' LANGUAGE C
+ IMMUTABLE STRICT;
+
+CREATE OR REPLACE FUNCTION get_snapshot_xmax(txid_snapshot)
+ RETURNS bigint
+ AS 'MODULE_PATHNAME', 'txid_snapshot_xmax' LANGUAGE C
+ IMMUTABLE STRICT;
+
+CREATE OR REPLACE FUNCTION get_snapshot_active(txid_snapshot)
+ RETURNS setof bigint
+ AS 'MODULE_PATHNAME', 'txid_snapshot_active' LANGUAGE C
+ IMMUTABLE STRICT;
+
+
+--
+-- Special comparision functions used by the remote worker
+-- for sync chunk selection
+--
+CREATE OR REPLACE FUNCTION txid_in_snapshot(bigint, txid_snapshot)
+ RETURNS boolean
+ AS 'MODULE_PATHNAME', 'txid_in_snapshot' LANGUAGE C
+ IMMUTABLE STRICT;
+
+CREATE OR REPLACE FUNCTION txid_not_in_snapshot(bigint, txid_snapshot)
+ RETURNS boolean
+ AS 'MODULE_PATHNAME', 'txid_not_in_snapshot' LANGUAGE C
+ IMMUTABLE STRICT;
+
--- /dev/null
+
+DROP DOMAIN txid;
+DROP TYPE txid_snapshot cascade;
+DROP SCHEMA txid CASCADE;
+DROP FUNCTION get_current_txid();
+DROP FUNCTION get_snapshot_xmin();
+DROP FUNCTION get_snapshot_xmax();
+DROP FUNCTION get_snapshot_active();
+
+
--- /dev/null
+
+PYTHONPATH=../../python:$PYTHONPATH
+PATH=../../python:../../scripts:$PATH
+export PYTHONPATH PATH
+
+
--- /dev/null
+
+[londiste]
+job_name = fread_test
+
+method = file_read
+
+provider_db = dbname=provider
+subscriber_db = dbname=file_subscriber
+
+# it will be used as sql ident so no dots/spaces
+pgq_queue_name = londiste_test_replic
+
+pidfile = sys/pid.%(job_name)s
+logfile = sys/log.%(job_name)s
+
+file_src = ./file_logs
+
--- /dev/null
+
+[londiste]
+job_name = fwrite_test
+
+method = file_write
+
+provider_db = dbname=provider
+subscriber_db = dbname=subscriber
+
+# it will be used as sql ident so no dots/spaces
+pgq_queue_name = londiste_replic
+
+pidfile = sys/pid.%(job_name)s
+logfile = sys/log.%(job_name)s
+
+file_dst = ./file_logs
+
--- /dev/null
+[pgqadm]
+
+job_name = link_ticker
+
+db = dbname=subscriber
+
+# how often to run maintenance [minutes]
+maint_delay_min = 1
+
+# how often to check fot activity [secs]
+loop_delay = 0.1
+
+logfile = sys/log.%(job_name)s
+pidfile = sys/pid.%(job_name)s
+
+use_skylog = 0
+
--- /dev/null
+
+[londiste]
+job_name = replic
+
+provider_db = dbname=provider
+subscriber_db = dbname=subscriber
+
+# it will be used as sql ident so no dots/spaces
+pgq_queue_name = londiste_replic
+
+pidfile = sys/pid.%(job_name)s
+logfile = sys/log.%(job_name)s
+
+loop_delay = 0.3
+
--- /dev/null
+
+[londiste]
+job_name = replic_test
+
+provider_db = dbname=provider
+subscriber_db = dbname=subscriber
+
+# it will be used as sql ident so no dots/spaces
+pgq_queue_name = londiste_test_replic
+
+pidfile = sys/pid.%(job_name)s
+logfile = sys/log.%(job_name)s
+
+# should be in right sequence
+table_list = data1, data2
+
--- /dev/null
+[pgqadm]
+
+job_name = ticker
+
+db = dbname=provider
+
+# how often to run maintenance [minutes]
+maint_delay_min = 1
+
+# how often to check fot activity [secs]
+loop_delay = 0.1
+
+logfile = sys/log.%(job_name)s
+pidfile = sys/pid.%(job_name)s
+
+use_skylog = 0
+
--- /dev/null
+
+set client_min_messages = 'warning';
+
+create table data1 (
+ id serial primary key,
+ data text
+);
+
+create unique index idx_data1_uq on data1 (data);
+
+create index idx_data1_rand on data1 (id, data);
+
+create table data2 (
+ id serial primary key,
+ data text,
+ -- old_id integer references data1,
+ constraint uq_data2 unique (data)
+);
+
+create index idx_data2_rand on data2 (id, data);
+
+
+create sequence test_seq;
+select setval('test_seq', 50);
--- /dev/null
+
+PYTHONPATH=../../python:$PYTHONPATH
+PATH=../../python:../../scripts:$PATH
+export PYTHONPATH PATH
+
+#. /opt/apps/pgsql-dev/env
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+contrib=/usr/share/postgresql/8.1/contrib
+contrib=/opt/apps/pgsql-dev/share/contrib
+contrib=/opt/pgsql/share/contrib
+
+db=provider
+
+
+mkdir -p file_logs sys
+./stop.sh
+sleep 1
+
+rm -rf file_logs sys
+mkdir -p file_logs sys
+
+echo "creating database: $db"
+dropdb $db
+sleep 1
+createdb $db
+pgqadm.py conf/ticker.ini install
+psql -q $db -f data.sql
+
+db=subscriber
+echo "creating database: $db"
+dropdb $db
+sleep 1
+createdb $db
+pgqadm.py conf/linkticker.ini install
+psql -q $db -f data.sql
+
+db=file_subscriber
+echo "creating database: $db"
+dropdb $db
+sleep 1
+createdb $db
+createlang plpgsql $db
+createlang plpythonu $db
+psql -q $db -f data.sql
+
+echo "done, testing"
+
+#pgqmgr.py -d conf/ticker.ini ticker
+#./run-tests.sh
+
--- /dev/null
+#! /bin/sh
+
+. ./env.sh
+
+script=londiste.py
+
+set -e
+
+$script conf/replic.ini provider install
+
+psql -c "update pgq.queue set queue_ticker_idle_period = '3', queue_ticker_max_lag = '2'" provider
+
+pgqadm.py -d conf/ticker.ini ticker
+
+$script conf/replic.ini subscriber install
+
+$script conf/replic.ini subscriber register
+$script conf/replic.ini subscriber unregister
+
+$script -v -d conf/replic.ini replay
+$script -v -d conf/fwrite.ini replay
+
+sleep 2
+
+$script conf/replic.ini provider add data1
+$script conf/replic.ini subscriber add data1
+
+sleep 2
+
+$script conf/replic.ini provider add data2
+$script conf/replic.ini subscriber add data2
+
+sleep 2
+
+$script conf/replic.ini provider tables
+$script conf/replic.ini provider remove data2
+
+sleep 2
+
+$script conf/replic.ini provider add data2
+
+$script conf/replic.ini provider add-seq data1_id_seq
+$script conf/replic.ini provider add-seq test_seq
+$script conf/replic.ini subscriber add-seq data1_id_seq
+$script conf/replic.ini subscriber add-seq test_seq
+
+sleep 2
+
+$script conf/replic.ini subscriber tables
+$script conf/replic.ini subscriber missing
+
+$script conf/replic.ini subscriber remove data2
+sleep 2
+$script conf/replic.ini subscriber add data2
+sleep 2
+
+./testing.py conf/tester.ini
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+./testing.py -s conf/tester.ini
+londiste.py -s conf/fwrite.ini
+londiste.py -s conf/replic.ini
+
+sleep 1
+
+pgqadm.py -s conf/ticker.ini
+pgqadm.py -s conf/linkticker.ini
+
--- /dev/null
+#! /usr/bin/env python
+
+"""Londiste tester.
+"""
+
+import sys, os, skytools
+
+
+class Tester(skytools.DBScript):
+ test_pos = 0
+ nr = 1
+ def __init__(self, args):
+ skytools.DBScript.__init__(self, 'londiste', args)
+ self.log.info('start testing')
+
+ def reload(self):
+ skytools.DBScript.reload(self)
+ self.loop_delay = 0.1
+
+ def work(self):
+
+ src_db = self.get_database('provider_db')
+ dst_db = self.get_database('subscriber_db')
+ src_curs = src_db.cursor()
+ dst_curs = dst_db.cursor()
+ src_curs.execute("insert into data1 (data) values ('foo%d')" % self.nr)
+ src_curs.execute("insert into data2 (data) values ('foo%d')" % self.nr)
+ src_db.commit()
+ self.nr += 1
+
+ if self.bad_state(dst_db, dst_curs):
+ return
+
+ if self.test_pos == 0:
+ self.resync_table(dst_db, dst_curs)
+ self.test_pos += 1
+ elif self.test_pos == 1:
+ self.run_compare()
+ self.test_pos += 1
+
+ def bad_state(self, db, curs):
+ q = "select * from londiste.subscriber_table"
+ curs.execute(q)
+ db.commit()
+ ok = 0
+ bad = 0
+ cnt = 0
+ for row in curs.dictfetchall():
+ cnt += 1
+ if row['merge_state'] == 'ok':
+ ok += 1
+ else:
+ bad += 1
+
+ if cnt < 2:
+ return 1
+ if bad > 0:
+ return 1
+
+ if ok > 0:
+ return 0
+
+ return 1
+
+ def resync_table(self, db, curs):
+ self.log.info('trying to remove table')
+ curs.execute("update londiste.subscriber_table"\
+ " set merge_state = null"
+ " where table_name='public.data1'")
+ db.commit()
+
+ def run_compare(self):
+ args = ["londiste.py", "conf/replic.ini", "compare"]
+ err = os.spawnvp(os.P_WAIT, "londiste.py", args)
+ self.log.info("Compare result=%d" % err)
+
+if __name__ == '__main__':
+ script = Tester(sys.argv[1:])
+ script.start()
+
--- /dev/null
+[cube_dispatcher]
+job_name = cube_test
+
+src_db = dbname=scriptsrc
+dst_db = dbname=scriptdst
+
+pgq_queue_name = data.middle
+
+logfile = sys/%(job_name)s.log
+pidfile = sys/%(job_name)s.pid
+
+# how many rows are kept: keep_latest, keep_all
+mode = keep_latest
+
+part_template =
+ create table _DEST_TABLE (like _PARENT);
+ alter table only _DEST_TABLE add primary key (_PKEY);
+
--- /dev/null
+[queue_mover]
+job_name = queue_mover_test
+
+src_db = dbname=scriptsrc
+dst_db = dbname=scriptsrc
+
+pgq_queue_name = data.src
+
+dst_queue_name = data.middle
+
+logfile = sys/%(job_name)s.log
+pidfile = sys/%(job_name)s.pid
+
--- /dev/null
+[table_dispatcher]
+job_name = table_test
+
+src_db = dbname=scriptsrc
+dst_db = dbname=scriptdst
+
+pgq_queue_name = data.middle
+
+logfile = sys/%(job_name)s.log
+pidfile = sys/%(job_name)s.pid
+
+# where to put data. when partitioning, will be used as base name
+dest_table = data2
+
+# date field with will be used for partitioning
+# special value: _EVTIME - event creation time
+part_column = start_date
+
+#fields = *
+#fields = id_pstn_cdr, foo, bar
+#fields = id_pstn_cdr:my_id, foo, bar:baz
+
+
+# template used for creating partition tables
+# _DEST_TABLE
+part_template =
+ create table _DEST_TABLE () inherits (data2);
+
+
--- /dev/null
+[pgqadm]
+
+job_name = ticker
+
+db = dbname=scriptsrc
+
+# how often to run maintenance [minutes]
+maint_delay_min = 1
+
+# how often to check fot activity [secs]
+loop_delay = 0.1
+
+# if there's no events, how often to tick (to show liveness) [secs]
+idle_interval = 1
+
+# if there's this much events available, tick
+max_events = 10
+
+# if there is not many events, then don't let them stay more than [secs]
+max_lag = 0.5
+
+logfile = sys/log.%(job_name)s
+pidfile = sys/pid.%(job_name)s
+
+use_skylog = 0
+
--- /dev/null
+
+set client_min_messages = 'warning';
+
+create table data1 (
+ id serial primary key,
+ data text
+);
+
+create unique index idx_data1_uq on data1 (data);
+
+create index idx_data1_rand on data1 (id, data);
+
+create table data2 (
+ id serial primary key,
+ data text,
+ -- old_id integer references data1,
+ constraint uq_data2 unique (data)
+);
+
+create index idx_data2_rand on data2 (id, data);
+
+
--- /dev/null
+
+PYTHONPATH=../../python:$PYTHONPATH
+PATH=../../python:../../scripts:$PATH
+export PYTHONPATH PATH
+
+#. /opt/pgsql/env
+
--- /dev/null
+#! /bin/sh
+
+. ./env.sh
+
+contrib=/usr/share/postgresql/8.1/contrib
+contrib=/opt/pgsql/share/contrib
+
+mkdir -p sys
+./stop.sh
+sleep 1
+
+rm -f sys/*
+
+
+db=scriptsrc
+echo "creating database: $db"
+dropdb $db
+sleep 1
+createdb $db
+
+pgqadm.py conf/ticker.ini install
+
+#createlang plpgsql $db
+#createlang plpythonu $db
+#psql -q $db -f $contrib/txid.sql
+#psql -q $db -f $contrib/pgq.sql
+psql -q $db -f $contrib/pgq_ext.sql
+psql -q $db -f $contrib/logutriga.sql
+psql -q $db -f data.sql
+psql -q $db -f install.sql
+
+db=scriptdst
+echo "creating database: $db"
+dropdb $db
+sleep 1
+createdb $db
+createlang plpgsql $db
+psql -q $db -f data.sql
+psql -q $db -f $contrib/pgq_ext.sql
+
+echo "done, testing"
+
+#pgqmgr.py -d conf/ticker.ini ticker
+#./run-tests.sh
+
--- /dev/null
+
+select pgq.create_queue('data.src');
+select pgq.create_queue('data.middle');
+
+create trigger test_logger after insert or update or delete
+on data1 for each row execute procedure pgq.logutriga('data.src');
+
--- /dev/null
+#! /bin/sh
+
+. ./env.sh
+
+pgqadm.py -d conf/ticker.ini ticker
+queue_mover.py -d conf/mover.ini
+cube_dispatcher.py -d conf/cube.ini
+table_dispatcher.py -d conf/table.ini
+
+sleep 1
+psql scriptsrc <<EOF
+insert into data1 (data) values ('data1.1');
+insert into data1 (data) values ('data1.2');
+EOF
+
--- /dev/null
+#! /bin/sh
+
+. ./env.sh
+
+cube_dispatcher.py -s conf/cube.ini
+table_dispatcher.py -s conf/table.ini
+queue_mover.py -s conf/mover.ini
+
+sleep 1
+
+pgqadm.py -s conf/ticker.ini
+
+#killall python
+
--- /dev/null
+#! /usr/bin/env python
+
+import sys, os, skytools
+
+import skytools.skylog
+
+class LogTest(skytools.DBScript):
+ def work(self):
+ self.log.error('test error')
+ self.log.warning('test warning')
+ self.log.info('test info')
+ self.log.debug('test debug')
+
+if __name__ == '__main__':
+ script = LogTest('log_test', sys.argv[1:])
+ script.start()
+
--- /dev/null
+#! /bin/sh
+
+. ../env.sh
+
+exec ./logtest.py test.ini "$@"
+
--- /dev/null
+; notes:
+; - 'args' is mandatory in [handler_*] sections
+; - in lists there must not be spaces
+
+;
+; top-level config
+;
+
+; list of all loggers
+[loggers]
+keys=root
+; root logger sees everything. there can be per-job configs by
+; specifing loggers with job_name of the script
+
+; list of all handlers
+[handlers]
+keys=stderr,logdb,logsrv,logfile
+
+; list of all formatters
+[formatters]
+keys=short,long,none
+
+;
+; map specific loggers to specifig handlers
+;
+[logger_root]
+level=DEBUG
+handlers=stderr,logdb,logsrv,logfile
+;,logfile
+;logdb,logsrv,logfile
+
+;
+; configure formatters
+;
+[formatter_short]
+format=%(asctime)s %(levelname)s %(message)s
+datefmt=%H:%M
+
+[formatter_long]
+format=%(asctime)s %(process)s %(levelname)s %(message)s
+
+[formatter_none]
+format=%(message)s
+
+;
+; configure handlers
+;
+
+; file. args: stream
+[handler_stderr]
+class=StreamHandler
+args=(sys.stderr,)
+formatter=short
+
+; log into db. args: conn_string
+[handler_logdb]
+class=skylog.LogDBHandler
+args=("host=127.0.0.1 port=5432 user=marko dbname=logdb",)
+formatter=none
+level=INFO
+
+; JSON messages over UDP. args: host, port
+[handler_logsrv]
+class=skylog.UdpLogServerHandler
+args=('127.0.0.1', 6666)
+formatter=none
+
+; rotating logfile. args: filename, maxsize, maxcount
+[handler_logfile]
+class=skylog.EasyRotatingFileHandler
+args=('~/log/%(job_name)s.log', 100*1024*1024, 3)
+formatter=long
+
--- /dev/null
+[log_test]
+
+loop_delay = 5
+
+logfile = xtest.log
+
--- /dev/null
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the PostgreSQL Administrator's Guide, chapter "Client
+# Authentication" for a complete description. A short synopsis
+# follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access. Records take one of these forms:
+#
+# local DATABASE USER METHOD [OPTION]
+# host DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain socket,
+# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an
+# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", a database name, or
+# a comma-separated list thereof.
+#
+# USER can be "all", a user name, a group name prefixed with "+", or
+# a comma-separated list thereof. In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names from
+# a separate file.
+#
+# CIDR-ADDRESS specifies the set of hosts the record matches.
+# It is made up of an IP address and a CIDR mask that is an integer
+# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies
+# the number of significant bits in the mask. Alternatively, you can write
+# an IP address and netmask in separate columns to specify the set of hosts.
+#
+# METHOD can be "trust", "reject", "md5", "crypt", "password",
+# "krb5", "ident", or "pam". Note that "password" sends passwords
+# in clear text; "md5" is preferred since it sends encrypted passwords.
+#
+# OPTION is the ident map or the name of the PAM service, depending on METHOD.
+#
+# Database and user names containing spaces, commas, quotes and other special
+# characters must be quoted. Quoting one of the keywords "all", "sameuser" or
+# "samerole" makes the name lose its special character, and just match a
+# database or username with that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect. You can use
+# "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records. In that case you will also need to make PostgreSQL listen
+# on a non-local interface via the listen_addresses configuration parameter,
+# or via the -i or -h command line switches.
+#
+
+# CAUTION: Configuring the system for local "trust" authentication allows
+# any local user to connect as any PostgreSQL user, including the database
+# superuser. If you do not trust all your local users, use another
+# authentication method.
+
+
+# TYPE DATABASE USER CIDR-ADDRESS METHOD
+
+# "local" is for Unix domain socket connections only
+local all all trust
+# IPv4 local connections:
+host all all 127.0.0.1/32 trust
+# IPv6 local connections:
+host all all ::1/128 trust
--- /dev/null
+# PostgreSQL Ident Authentication Maps
+# ====================================
+#
+# Refer to the PostgreSQL Administrator's Guide, chapter "Client
+# Authentication" for a complete description. A short synopsis
+# follows.
+#
+# This file controls PostgreSQL ident-based authentication. It maps
+# ident user names (typically Unix user names) to their corresponding
+# PostgreSQL user names. Records are of the form:
+#
+# MAPNAME IDENT-USERNAME PG-USERNAME
+#
+# (The uppercase quantities must be replaced by actual values.)
+#
+# MAPNAME is the (otherwise freely chosen) map name that was used in
+# pg_hba.conf. IDENT-USERNAME is the detected user name of the
+# client. PG-USERNAME is the requested PostgreSQL user name. The
+# existence of a record specifies that IDENT-USERNAME may connect as
+# PG-USERNAME. Multiple maps may be specified in this file and used
+# by pg_hba.conf.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect. You can use
+# "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# No map names are defined in the default configuration. If all ident
+# user names and PostgreSQL user names are the same, you don't need
+# this file. Instead, use the special map name "sameuser" in
+# pg_hba.conf.
+
+# MAPNAME IDENT-USERNAME PG-USERNAME
--- /dev/null
+# - Connection Settings -
+#port = 5432
+port = 7200
+unix_socket_directory = '/tmp'
+
+#archive_command = '' # command to use to archive a logfile
+archive_command = '/usr/bin/walmgr.py /opt/src/mgrtest/wal-master.ini xarchive %p %f'
+ # segment
+
+
+# These settings are initialized by initdb -- they might be changed
+lc_messages = 'C' # locale for system error message
+ # strings
+lc_monetary = 'C' # locale for monetary formatting
+lc_numeric = 'C' # locale for number formatting
+lc_time = 'C' # locale for time formatting
+
--- /dev/null
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the PostgreSQL Administrator's Guide, chapter "Client
+# Authentication" for a complete description. A short synopsis
+# follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access. Records take one of these forms:
+#
+# local DATABASE USER METHOD [OPTION]
+# host DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain socket,
+# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an
+# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", a database name, or
+# a comma-separated list thereof.
+#
+# USER can be "all", a user name, a group name prefixed with "+", or
+# a comma-separated list thereof. In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names from
+# a separate file.
+#
+# CIDR-ADDRESS specifies the set of hosts the record matches.
+# It is made up of an IP address and a CIDR mask that is an integer
+# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies
+# the number of significant bits in the mask. Alternatively, you can write
+# an IP address and netmask in separate columns to specify the set of hosts.
+#
+# METHOD can be "trust", "reject", "md5", "crypt", "password",
+# "krb5", "ident", or "pam". Note that "password" sends passwords
+# in clear text; "md5" is preferred since it sends encrypted passwords.
+#
+# OPTION is the ident map or the name of the PAM service, depending on METHOD.
+#
+# Database and user names containing spaces, commas, quotes and other special
+# characters must be quoted. Quoting one of the keywords "all", "sameuser" or
+# "samerole" makes the name lose its special character, and just match a
+# database or username with that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect. You can use
+# "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records. In that case you will also need to make PostgreSQL listen
+# on a non-local interface via the listen_addresses configuration parameter,
+# or via the -i or -h command line switches.
+#
+
+# CAUTION: Configuring the system for local "trust" authentication allows
+# any local user to connect as any PostgreSQL user, including the database
+# superuser. If you do not trust all your local users, use another
+# authentication method.
+
+
+# TYPE DATABASE USER CIDR-ADDRESS METHOD
+
+# "local" is for Unix domain socket connections only
+local all all trust
+# IPv4 local connections:
+host all all 127.0.0.1/32 trust
+# IPv6 local connections:
+host all all ::1/128 trust
--- /dev/null
+# PostgreSQL Ident Authentication Maps
+# ====================================
+#
+# Refer to the PostgreSQL Administrator's Guide, chapter "Client
+# Authentication" for a complete description. A short synopsis
+# follows.
+#
+# This file controls PostgreSQL ident-based authentication. It maps
+# ident user names (typically Unix user names) to their corresponding
+# PostgreSQL user names. Records are of the form:
+#
+# MAPNAME IDENT-USERNAME PG-USERNAME
+#
+# (The uppercase quantities must be replaced by actual values.)
+#
+# MAPNAME is the (otherwise freely chosen) map name that was used in
+# pg_hba.conf. IDENT-USERNAME is the detected user name of the
+# client. PG-USERNAME is the requested PostgreSQL user name. The
+# existence of a record specifies that IDENT-USERNAME may connect as
+# PG-USERNAME. Multiple maps may be specified in this file and used
+# by pg_hba.conf.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect. You can use
+# "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# No map names are defined in the default configuration. If all ident
+# user names and PostgreSQL user names are the same, you don't need
+# this file. Instead, use the special map name "sameuser" in
+# pg_hba.conf.
+
+# MAPNAME IDENT-USERNAME PG-USERNAME
--- /dev/null
+# -----------------------------
+# PostgreSQL configuration file
+# -----------------------------
+#
+# This file consists of lines of the form:
+#
+# name = value
+#
+# (The '=' is optional.) White space may be used. Comments are introduced
+# with '#' anywhere on a line. The complete list of option names and
+# allowed values can be found in the PostgreSQL documentation. The
+# commented-out settings shown in this file represent the default values.
+#
+# Please note that re-commenting a setting is NOT sufficient to revert it
+# to the default value, unless you restart the postmaster.
+#
+# Any option can also be given as a command line switch to the
+# postmaster, e.g. 'postmaster -c log_connections=on'. Some options
+# can be changed at run-time with the 'SET' SQL command.
+#
+# This file is read on postmaster startup and when the postmaster
+# receives a SIGHUP. If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect, or use
+# "pg_ctl reload". Some settings, such as listen_addresses, require
+# a postmaster shutdown and restart to take effect.
+
+
+#---------------------------------------------------------------------------
+# FILE LOCATIONS
+#---------------------------------------------------------------------------
+
+# The default values of these variables are driven from the -D command line
+# switch or PGDATA environment variable, represented here as ConfigDir.
+
+#data_directory = 'ConfigDir' # use data in another directory
+#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file
+#ident_file = 'ConfigDir/pg_ident.conf' # IDENT configuration file
+
+# If external_pid_file is not explicitly set, no extra pid file is written.
+#external_pid_file = '(none)' # write an extra pid file
+
+
+#---------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#---------------------------------------------------------------------------
+
+# - Connection Settings -
+
+#listen_addresses = 'localhost' # what IP address(es) to listen on;
+ # comma-separated list of addresses;
+ # defaults to 'localhost', '*' = all
+#port = 5432
+port = 7201
+max_connections = 100
+# note: increasing max_connections costs ~400 bytes of shared memory per
+# connection slot, plus lock space (see max_locks_per_transaction). You
+# might also need to raise shared_buffers to support more connections.
+#superuser_reserved_connections = 2
+unix_socket_directory = '/opt/src/mgrtest'
+#unix_socket_directory = ''
+#unix_socket_group = ''
+#unix_socket_permissions = 0777 # octal
+#bonjour_name = '' # defaults to the computer name
+
+# - Security & Authentication -
+
+#authentication_timeout = 60 # 1-600, in seconds
+#ssl = off
+#password_encryption = on
+#db_user_namespace = off
+
+# Kerberos
+#krb_server_keyfile = ''
+#krb_srvname = 'postgres'
+#krb_server_hostname = '' # empty string matches any keytab entry
+#krb_caseins_users = off
+
+# - TCP Keepalives -
+# see 'man 7 tcp' for details
+
+#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds;
+ # 0 selects the system default
+#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds;
+ # 0 selects the system default
+#tcp_keepalives_count = 0 # TCP_KEEPCNT;
+ # 0 selects the system default
+
+
+#---------------------------------------------------------------------------
+# RESOURCE USAGE (except WAL)
+#---------------------------------------------------------------------------
+
+# - Memory -
+
+shared_buffers = 1000 # min 16 or max_connections*2, 8KB each
+#temp_buffers = 1000 # min 100, 8KB each
+#max_prepared_transactions = 5 # can be 0 or more
+# note: increasing max_prepared_transactions costs ~600 bytes of shared memory
+# per transaction slot, plus lock space (see max_locks_per_transaction).
+#work_mem = 1024 # min 64, size in KB
+#maintenance_work_mem = 16384 # min 1024, size in KB
+#max_stack_depth = 2048 # min 100, size in KB
+
+# - Free Space Map -
+
+#max_fsm_pages = 20000 # min max_fsm_relations*16, 6 bytes each
+#max_fsm_relations = 1000 # min 100, ~70 bytes each
+
+# - Kernel Resource Usage -
+
+#max_files_per_process = 1000 # min 25
+#preload_libraries = ''
+
+# - Cost-Based Vacuum Delay -
+
+#vacuum_cost_delay = 0 # 0-1000 milliseconds
+#vacuum_cost_page_hit = 1 # 0-10000 credits
+#vacuum_cost_page_miss = 10 # 0-10000 credits
+#vacuum_cost_page_dirty = 20 # 0-10000 credits
+#vacuum_cost_limit = 200 # 0-10000 credits
+
+# - Background writer -
+
+#bgwriter_delay = 200 # 10-10000 milliseconds between rounds
+#bgwriter_lru_percent = 1.0 # 0-100% of LRU buffers scanned/round
+#bgwriter_lru_maxpages = 5 # 0-1000 buffers max written/round
+#bgwriter_all_percent = 0.333 # 0-100% of all buffers scanned/round
+#bgwriter_all_maxpages = 5 # 0-1000 buffers max written/round
+
+
+#---------------------------------------------------------------------------
+# WRITE AHEAD LOG
+#---------------------------------------------------------------------------
+
+# - Settings -
+
+#fsync = on # turns forced synchronization on or off
+#wal_sync_method = fsync # the default is the first option
+ # supported by the operating system:
+ # open_datasync
+ # fdatasync
+ # fsync
+ # fsync_writethrough
+ # open_sync
+#full_page_writes = on # recover from partial page writes
+#wal_buffers = 8 # min 4, 8KB each
+#commit_delay = 0 # range 0-100000, in microseconds
+#commit_siblings = 5 # range 1-1000
+
+# - Checkpoints -
+
+#checkpoint_segments = 3 # in logfile segments, min 1, 16MB each
+#checkpoint_timeout = 300 # range 30-3600, in seconds
+#checkpoint_warning = 30 # in seconds, 0 is off
+
+# - Archiving -
+
+#archive_command = '' # command to use to archive a logfile
+archive_command = '/usr/bin/walmgr.py /opt/src/mgrtest/wal-master.ini xarchive %p %f'
+ # segment
+
+
+#---------------------------------------------------------------------------
+# QUERY TUNING
+#---------------------------------------------------------------------------
+
+# - Planner Method Configuration -
+
+#enable_bitmapscan = on
+#enable_hashagg = on
+#enable_hashjoin = on
+#enable_indexscan = on
+#enable_mergejoin = on
+#enable_nestloop = on
+#enable_seqscan = on
+#enable_sort = on
+#enable_tidscan = on
+
+# - Planner Cost Constants -
+
+#effective_cache_size = 1000 # typically 8KB each
+#random_page_cost = 4 # units are one sequential page fetch
+ # cost
+#cpu_tuple_cost = 0.01 # (same)
+#cpu_index_tuple_cost = 0.001 # (same)
+#cpu_operator_cost = 0.0025 # (same)
+
+# - Genetic Query Optimizer -
+
+#geqo = on
+#geqo_threshold = 12
+#geqo_effort = 5 # range 1-10
+#geqo_pool_size = 0 # selects default based on effort
+#geqo_generations = 0 # selects default based on effort
+#geqo_selection_bias = 2.0 # range 1.5-2.0
+
+# - Other Planner Options -
+
+#default_statistics_target = 10 # range 1-1000
+#constraint_exclusion = off
+#from_collapse_limit = 8
+#join_collapse_limit = 8 # 1 disables collapsing of explicit
+ # JOINs
+
+
+#---------------------------------------------------------------------------
+# ERROR REPORTING AND LOGGING
+#---------------------------------------------------------------------------
+
+# - Where to Log -
+
+#log_destination = 'stderr' # Valid values are combinations of
+ # stderr, syslog and eventlog,
+ # depending on platform.
+
+# This is used when logging to stderr:
+#redirect_stderr = off # Enable capturing of stderr into log
+ # files
+
+# These are only used if redirect_stderr is on:
+#log_directory = 'pg_log' # Directory where log files are written
+ # Can be absolute or relative to PGDATA
+#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # Log file name pattern.
+ # Can include strftime() escapes
+#log_truncate_on_rotation = off # If on, any existing log file of the same
+ # name as the new log file will be
+ # truncated rather than appended to. But
+ # such truncation only occurs on
+ # time-driven rotation, not on restarts
+ # or size-driven rotation. Default is
+ # off, meaning append to existing files
+ # in all cases.
+#log_rotation_age = 1440 # Automatic rotation of logfiles will
+ # happen after so many minutes. 0 to
+ # disable.
+#log_rotation_size = 10240 # Automatic rotation of logfiles will
+ # happen after so many kilobytes of log
+ # output. 0 to disable.
+
+# These are relevant when logging to syslog:
+#syslog_facility = 'LOCAL0'
+#syslog_ident = 'postgres'
+
+
+# - When to Log -
+
+#client_min_messages = notice # Values, in order of decreasing detail:
+ # debug5
+ # debug4
+ # debug3
+ # debug2
+ # debug1
+ # log
+ # notice
+ # warning
+ # error
+
+#log_min_messages = notice # Values, in order of decreasing detail:
+ # debug5
+ # debug4
+ # debug3
+ # debug2
+ # debug1
+ # info
+ # notice
+ # warning
+ # error
+ # log
+ # fatal
+ # panic
+
+#log_error_verbosity = default # terse, default, or verbose messages
+
+#log_min_error_statement = panic # Values in order of increasing severity:
+ # debug5
+ # debug4
+ # debug3
+ # debug2
+ # debug1
+ # info
+ # notice
+ # warning
+ # error
+ # panic(off)
+
+#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements
+ # and their durations, in milliseconds.
+
+#silent_mode = off # DO NOT USE without syslog or
+ # redirect_stderr
+
+# - What to Log -
+
+#debug_print_parse = off
+#debug_print_rewritten = off
+#debug_print_plan = off
+#debug_pretty_print = off
+#log_connections = off
+#log_disconnections = off
+#log_duration = off
+#log_line_prefix = '' # Special values:
+ # %u = user name
+ # %d = database name
+ # %r = remote host and port
+ # %h = remote host
+ # %p = PID
+ # %t = timestamp (no milliseconds)
+ # %m = timestamp with milliseconds
+ # %i = command tag
+ # %c = session id
+ # %l = session line number
+ # %s = session start timestamp
+ # %x = transaction id
+ # %q = stop here in non-session
+ # processes
+ # %% = '%'
+ # e.g. '<%u%%%d> '
+#log_statement = 'none' # none, mod, ddl, all
+#log_hostname = off
+
+
+#---------------------------------------------------------------------------
+# RUNTIME STATISTICS
+#---------------------------------------------------------------------------
+
+# - Statistics Monitoring -
+
+#log_parser_stats = off
+#log_planner_stats = off
+#log_executor_stats = off
+#log_statement_stats = off
+
+# - Query/Index Statistics Collector -
+
+#stats_start_collector = on
+#stats_command_string = off
+#stats_block_level = off
+#stats_row_level = off
+#stats_reset_on_server_start = off
+
+
+#---------------------------------------------------------------------------
+# AUTOVACUUM PARAMETERS
+#---------------------------------------------------------------------------
+
+#autovacuum = off # enable autovacuum subprocess?
+#autovacuum_naptime = 60 # time between autovacuum runs, in secs
+#autovacuum_vacuum_threshold = 1000 # min # of tuple updates before
+ # vacuum
+#autovacuum_analyze_threshold = 500 # min # of tuple updates before
+ # analyze
+#autovacuum_vacuum_scale_factor = 0.4 # fraction of rel size before
+ # vacuum
+#autovacuum_analyze_scale_factor = 0.2 # fraction of rel size before
+ # analyze
+#autovacuum_vacuum_cost_delay = -1 # default vacuum cost delay for
+ # autovac, -1 means use
+ # vacuum_cost_delay
+#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for
+ # autovac, -1 means use
+ # vacuum_cost_limit
+
+
+#---------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#---------------------------------------------------------------------------
+
+# - Statement Behavior -
+
+#search_path = '$user,public' # schema names
+#default_tablespace = '' # a tablespace name, '' uses
+ # the default
+#check_function_bodies = on
+#default_transaction_isolation = 'read committed'
+#default_transaction_read_only = off
+#statement_timeout = 0 # 0 is disabled, in milliseconds
+
+# - Locale and Formatting -
+
+#datestyle = 'iso, mdy'
+#timezone = unknown # actually, defaults to TZ
+ # environment setting
+#australian_timezones = off
+#extra_float_digits = 0 # min -15, max 2
+#client_encoding = sql_ascii # actually, defaults to database
+ # encoding
+
+# These settings are initialized by initdb -- they might be changed
+lc_messages = 'C' # locale for system error message
+ # strings
+lc_monetary = 'C' # locale for monetary formatting
+lc_numeric = 'C' # locale for number formatting
+lc_time = 'C' # locale for time formatting
+
+# - Other Defaults -
+
+#explain_pretty_print = on
+#dynamic_library_path = '$libdir'
+
+
+#---------------------------------------------------------------------------
+# LOCK MANAGEMENT
+#---------------------------------------------------------------------------
+
+#deadlock_timeout = 1000 # in milliseconds
+#max_locks_per_transaction = 64 # min 10
+# note: each lock table slot uses ~220 bytes of shared memory, and there are
+# max_locks_per_transaction * (max_connections + max_prepared_transactions)
+# lock table slots.
+
+
+#---------------------------------------------------------------------------
+# VERSION/PLATFORM COMPATIBILITY
+#---------------------------------------------------------------------------
+
+# - Previous Postgres Versions -
+
+#add_missing_from = off
+#backslash_quote = safe_encoding # on, off, or safe_encoding
+#default_with_oids = off
+#escape_string_warning = off
+#regex_flavor = advanced # advanced, extended, or basic
+#sql_inheritance = on
+
+# - Other Platforms & Clients -
+
+#transform_null_equals = off
+
+
+#---------------------------------------------------------------------------
+# CUSTOMIZED OPTIONS
+#---------------------------------------------------------------------------
+
+#custom_variable_classes = '' # list of custom variable class names
--- /dev/null
+#! /bin/sh
+
+set -e
+
+tmp=/tmp/waltest
+src=$PWD
+walmgr=$src/../../python/walmgr.py
+
+test -f $tmp/data.master/postmaster.pid \
+&& kill `head -1 $tmp/data.master/postmaster.pid` || true
+test -f $tmp/data.slave/postmaster.pid \
+&& kill `head -1 $tmp/data.slave/postmaster.pid` || true
+
+rm -rf $tmp
+mkdir -p $tmp
+cd $tmp
+
+LANG=C
+PATH=/usr/lib/postgresql/8.1/bin:$PATH
+export PATH LANG
+
+mkdir log slave
+
+#
+# Prepare configs
+#
+
+### wal.master.ini ###
+cat > wal.master.ini <<EOF
+[wal-master]
+logfile = $tmp/log/wal-master.log
+master_db = dbname=template1 port=7200 host=127.0.0.1
+master_data = $tmp/data.master
+master_config = %(master_data)s/postgresql.conf
+slave = localhost:$tmp/slave
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+# syncdaemon update frequency
+loop_delay = 10.0
+EOF
+
+### wal.slave.ini ###
+cat > wal.slave.ini <<EOF
+[wal-slave]
+logfile = $tmp/log/wal-slave.log
+slave_data = $tmp/data.slave
+slave_stop_cmd = $tmp/rc.slave stop
+slave_start_cmd = $tmp/rc.slave start
+slave = $tmp/slave
+completed_wals = %(slave)s/logs.complete
+partial_wals = %(slave)s/logs.partial
+full_backup = %(slave)s/data.master
+EOF
+
+### rc.slave ###
+cat > rc.slave <<EOF
+#! /bin/sh
+cd $tmp
+test -f $tmp/data.slave/postgresql.conf \
+|| cp $src/conf.slave/* $tmp/data.slave
+pg_ctl -l $tmp/log/pg.slave.log -D $tmp/data.slave "\$1"
+EOF
+chmod +x rc.slave
+
+#
+# Initialize master db
+#
+echo "### Running initdb for master ###"
+initdb data.master > log/initdb.log 2>&1
+cp $src/conf.master/* data.master/
+pg_ctl -D data.master -l log/pg.master.log start
+sleep 4
+
+echo '####' $walmgr $tmp/wal.master.ini setup
+$walmgr wal.master.ini setup
+echo '####' $walmgr $tmp/wal.master.ini backup
+$walmgr wal.master.ini backup
+
+echo '####' $walmgr $tmp/wal.slave.ini restore
+$walmgr $tmp/wal.slave.ini restore
+sleep 10
+echo '####' $walmgr $tmp/wal.slave.ini boot
+$walmgr $tmp/wal.slave.ini boot
+
projects / skytools.git / commitdiff