TODO.html pgq-sql.html pgq-nodupes.html \
$(SCRIPT_HTMLS) faq.html set.notes.html skytools3.html devnotes.html
-SCRIPT_TXTS = londiste3.txt walmgr3.txt qadmin.txt scriptmgr.txt skytools_upgrade.txt
+SCRIPT_TXTS = londiste3.txt walmgr3.txt qadmin.txt scriptmgr.txt skytools_upgrade.txt \
+ queue_mover.txt queue_splitter.txt
SCRIPT_HTMLS = $(SCRIPT_TXTS:.txt=.html)
MAN5 =
-MAN1_SFX = scriptmgr.1 skytools_upgrade.1 walmgr3.1 londiste3.1
-MAN1 = qadmin.1 pgqd.1
+MAN1_SFX = scriptmgr.1 skytools_upgrade.1 queue_mover.1 queue_splitter.1
+MAN1 = qadmin.1 pgqd.1 walmgr3.1 londiste3.1
FQHTML = $(addprefix html/, $(HTMLS))
FQMAN1 = $(addprefix man/, $(MAN1))
--- /dev/null
+
+= queue_mover(1) =
+
+== NAME ==
+
+queue_mover - PgQ consumer that copies data from one queue to another.
+
+== SYNOPSIS ==
+
+ queue_mover.py [switches] config.ini
+
+== DESCRIPTION ==
+
+queue_mover is PgQ consumer that transports events from source queue into
+target queue. One use case is when events are produced in several databases
+then queue_mover is used to consolidate these events into single queue
+that can then be processed by consumers who need to handle theses events.
+For example in case of patitioned databases it's convenient to move events
+from each partition into one central queue database and then process them there.
+That way configuration and dependancies of partiton databases are
+simpler and more robust. Another use case is to move events from OLTP
+database to batch processing server.
+
+Transactionality: events will be inserted as one transaction on target side.
+That means only batch_id needs to be tracked on target side.
+
+== QUICK-START ==
+
+Basic PgQ setup and usage can be summarized by the following
+steps:
+
+ 1. PgQ must be installed both in source and target databases.
+ See pgqadm man page for details.
+
+ 2. Target database must also have pgq_ext schema installed.
+ It is used to keep sync between two databases.
+
+ 3. Create a queue_mover configuration file, say qmover_sourceq_to_targetdb.ini
+
+ 4. create source and target queues
+
+ $ pgqadm.py sourcedb_ticker.ini create <srcqueue>
+ $ pgqadm.py targetdb_ticker.ini create <dstqueue>
+
+ 5. launch queue mover in daemon mode
+
+ $ queue_mover.py -d qmover_sourceq_to_targetdb.ini
+
+ 6. start producing and consuming events
+
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== queue_mover parameters ===
+
+src_db::
+ Source database.
+
+dst_db::
+ Target database.
+
+dst_queue_name::
+ Target queue name.
+
+=== Example config file ===
+
+ [queue_mover]
+ job_name = eventlog_to_target_mover
+ src_db = dbname=sourcedb
+ dst_db = dbname=targetdb
+ pgq_queue_name = eventlog
+ dst_queue_name = copy_of_eventlog
+ pidfile = log/%(job_name)s.pid
+ logfile = pid/%(job_name)s.log
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+== BUGS ==
+
+Event ID is not kept on target side. If needed is can be kept,
+then event_id seq at target side need to be increased by hand to
+inform ticker about new events.
+
--- /dev/null
+= queue_splitter(1) =
+
+== NAME ==
+
+queue_splitter - PgQ consumer that transports events from one queue into several target queues
+
+== SYNOPSIS ==
+
+ queue_splitter.py [switches] config.ini
+
+== DESCRIPTION ==
+
+queue_spliter is PgQ consumer that transports events from source queue into
+several target queues. `ev_extra1` field in each event shows into which
+target queue it must go. (`pgq.logutriga()` puts there the table name.)
+
+One use case is to move events from OLTP database to batch processing server.
+By using queue spliter it is possible to move all kinds of events for batch
+processing with one consumer thus keeping OLTP database less crowded.
+
+== QUICK-START ==
+
+Basic queue_splitter setup and usage can be summarized by the following
+steps:
+
+ 1. pgq must be installed both in source and target databases.
+ See pgqadm man page for details. Target database must also
+ have pgq_ext schema installed.
+
+ 2. edit a queue_splitter configuration file, say queue_splitter_sourcedb_sourceq_targetdb.ini
+
+ 3. create source and target queues
+
+ $ pgqadm.py ticker.ini create <queue>
+
+ 4. launch queue splitter in daemon mode
+
+ $ queue_splitter.py queue_splitter_sourcedb_sourceq_targetdb.ini -d
+
+ 5. start producing and consuming events
+
+== CONFIG ==
+
+include::common.config.txt[]
+
+=== queue_splitter parameters ===
+
+src_db::
+ Source database.
+
+dst_db::
+ Target database.
+
+=== Example config file ===
+
+ [queue_splitter]
+ job_name = queue_spliter_sourcedb_sourceq_targetdb
+
+ src_db = dbname=sourcedb
+ dst_db = dbname=targetdb
+
+ pgq_queue_name = sourceq
+
+ logfile = ~/log/%(job_name)s.log
+ pidfile = ~/pid/%(job_name)s.pid
+
+== COMMAND LINE SWITCHES ==
+
+include::common.switches.txt[]
+
+== USECASE ==
+
+How to to process events created in secondary database
+with several queues but have only one queue in primary
+database. This also shows how to insert events into
+queues with regular SQL easily.
+
+ CREATE SCHEMA queue;
+ CREATE TABLE queue.event1 (
+ -- this should correspond to event internal structure
+ -- here you can put checks that correct data is put into queue
+ id int4,
+ name text,
+ -- not needed, but good to have:
+ primary key (id)
+ );
+ -- put data into queue in urlencoded format, skip actual insert
+ CREATE TRIGGER redirect_queue1_trg BEFORE INSERT ON queue.event1
+ FOR EACH ROW EXECUTE PROCEDURE pgq.logutriga('singlequeue', 'SKIP');
+ -- repeat the above for event2
+
+ -- now the data can be inserted:
+ INSERT INTO queue.event1 (id, name) VALUES (1, 'user');
+
+If the queue_splitter is put on "singlequeue", it spreads the event
+on target to queues named "queue.event1", "queue.event2", etc.
+This keeps PgQ load on primary database minimal both CPU-wise
+and maintenance-wise.
+
projects / skytools.git / commitdiff