From 88e25dc1f4d9269348d1f4a2351369402b42479c Mon Sep 17 00:00:00 2001 From: Marko Kreen Date: Tue, 21 Aug 2007 13:57:11 +0000 Subject: [PATCH] asciidoc manpages - londiste.1, londiste.5 By Dimitri Fontaine. --- doc/Makefile | 32 ++++- doc/londiste.cmdline.txt | 281 +++++++++++++++++++++++++++++++++++++++ doc/londiste.config.txt | 66 +++++++++ doc/londiste.txt | 64 --------- 4 files changed, 377 insertions(+), 66 deletions(-) create mode 100644 doc/londiste.cmdline.txt create mode 100644 doc/londiste.config.txt delete mode 100644 doc/londiste.txt diff --git a/doc/Makefile b/doc/Makefile index ee791998..55e194c8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,4 +1,6 @@ +include ../config.mak + wiki = https://developer.skype.com/SkypeGarage/DbProjects/SkyTools web = mkz@shell.pgfoundry.org:/home/pgfoundry.org/groups/skytools/htdocs/ @@ -7,7 +9,16 @@ EPYDOC = epydoc-py25 EPYARGS = --no-private --url="http://pgfoundry.org/projects/skytools/" \ --name="Skytools" --html --no-private -all: +all: man + +man: londiste.1 londiste.5 +html: londiste.cmdline.html londiste.config.html + +install: man + mkdir -p $(mandir)/man1 + mkdir -p $(mandir)/man5 + install -m 644 londiste.1 $(mandir)/man1 + install -m 644 londiste.5 $(mandir)/man1 upload: devupload.sh overview.txt $(wiki) @@ -39,8 +50,25 @@ apiupload: apidoc rsync -rtlz ../sql/pgq/docs/html/* $(web)/pgq/ clean: - rm -rf api + rm -rf api *.1 *.5 *.html *.xml distclean: rm -rf ../sql/pgq/docs/pgq api +londiste.1: londiste.cmdline.xml + xmlto man $< + +londiste.5: londiste.config.xml + xmlto man $< + +%.xml: %.txt + asciidoc -d manpage -b docbook $< + +%.html: %.txt + asciidoc $< + +manupload: londiste.1 londiste.1.html + rsync -avz londiste.1 londiste.1.html londiste.1.txt pgf:web/skytools/htdocs/man/ + + + diff --git a/doc/londiste.cmdline.txt b/doc/londiste.cmdline.txt new file mode 100644 index 00000000..c3c258e0 --- /dev/null +++ b/doc/londiste.cmdline.txt @@ -0,0 +1,281 @@ += londiste(1) = + + +== NAME == + +londiste - PostgreSQL replication engine written in python + +== SYNOPSIS == + + londiste.py [option] config.ini command [arguments] + +== DESCRIPTION == + +Londiste is the PostgreSQL replication engine portion of the SkyTools suite, +by Skype. This suite includes packages implementing specific replication +tasks and/or solutions in layers, building upon each other. + +PgQ is a generic queue implementation based on ideas from Slony-I's +snapshot based event batching. Londiste uses PgQ as its transport +mechanism to implement a robust and easy to use replication solution. + +Londiste is an asynchronous master-slave(s) replication +system. Asynchronous means that a transaction commited on the master is +not guaranteed to have made it to any slave at the master's commit time; and +master-slave means that data changes on slaves are not reported back to +the master, it's the other way around only. + +The replication is trigger based, and you choose a set of tables to +replicate from the provider to the subscriber(s). Any data changes +occuring on the provider (in a replicated table) will fire the +londiste trigger, which fills a queue of events for any subscriber(s) to +care about. + +A replay process consumes the queue in batches, and applies all given +changes to any subscriber(s). The initial replication step involves using the +PostgreSQL's COPY command for efficient data loading. + +== QUICK-START == + +Basic londiste setup and usage can be summarized by the following +steps: + + 1. create the subscriber database, with tables to replicate + + 2. edit a londiste configuration file, say conf.ini, and a PgQ ticker + configuration file, say ticker.ini + + 3. install londiste on the provider and subscriber nodes. This step + requires admin privileges on both provider and subscriber sides, + and both install commands can be run remotely: + + $ londiste.py conf.ini provider install + $ londiste.py conf.ini subscriber install + + 4. launch the PgQ ticker on the provider machine: + + $ pgqadm.py -d ticker.ini ticker + + 5. launch the londiste replay process: + + $ londiste.py -d conf.ini replay + + 6. add tables to replicate from the provider database: + + $ londiste.py conf.ini provider add table1 table2 ... + + 7. add tables to replicate to the subscriber database: + + $ londiste.py conf.ini subscriber add table1 table2 ... + +To replicate to more than one subscriber database just repeat each of the +described subscriber steps for each subscriber. + +Londiste will then consider updating data as needed. See the londiste +reference for more information about syncronisation state details: +https://developer.skype.com/SkypeGarage/DbProjects/SkyTools/LondisteReference[] + +== COMMANDS == + +The londiste command is parsed globally, and has both options and +subcommands. Some options are reserved to a subset of the commands, +and others should be used without any command at all. + +== GENERAL OPTIONS == + +This section presents options available to all and any londiste +command. + + -h, --help: + show this help message and exit + + -q, --quiet: + make program silent + + -v, --verbose: + make program more verbose + + +== PROVIDER COMMANDS == + + londiste.py config.ini provider + +Where command is one of: + +=== provider install === + +Installs code into provider and subscriber database and creates +queue. Equivalent to doing following by hand: + + CREATE LANGUAGE plpgsql; + CREATE LANGUAGE plpython; + \i .../contrib/txid.sql + \i .../contrib/logtriga.sql + \i .../contrib/pgq.sql + \i .../contrib/londiste.sql + select pgq.create_queue(queue name); + +Notes: + + * If the PostgreSQL modules are not installed on the same machine as + the Python scripts are, the commands need to be done by hand. + + * The schema/tables are installed under user Londiste is configured + to run. If you prefer to run Londiste under non-admin user, they + should also be installed by hand. + +=== provider add ... === + +Registers table(s) on the provider database and adds the londiste trigger to +the table(s) which will send events to the queue. Table names can be schema +qualified with the schema name defaulting to public if not supplied. + +=== provider remove
... === + +Unregisters table(s) on the provider side and removes the londiste triggers +from the table(s). The table removal event is also sent to the queue, so all +subscribers unregister the table(s) on their end as well. Table names can be +schema qualified with the schema name defaulting to public if not supplied. + +=== provider tables === + +Shows registered tables on provider side. + +=== provider seqs === + +Shows registered sequences on provider side. + +== SUBSCRIBER COMMANDS == + + londiste.py config.ini subscriber + +Where command is one of: + +=== subscriber install === + +Installs code into subscriber database. Equivalent to doing following +by hand: + + CREATE LANGUAGE plpgsql; + \i .../contrib/londiste.sql + +This will be done under the Postgres Londiste user, if the tables should +be owned by someone else, it needs to be done by hand. + +=== subscriber add
... === + +Registers table(s) on subscriber side. Table names can be schema qualified +with the schema name defaulting to `public` if not supplied. + +Switches (optional): + + --excect-sync: + Table is tagged as in-sync so initial COPY is skipped. + --skip-truncate: + When doing initial COPY, don't remove old data. + --force: + Ignore table structure differences. + +=== subscriber remove
... === + +Unregisters table(s) from subscriber. No events will be applied to +the table anymore. Actual table will not be touched. Table names can be +schema qualified with the schema name defaulting to public if not supplied. + +=== subscriber resync
... === + +Tags table(s) as "not synced". Later the replay process will notice this +and launch copy process(es) to sync the table(s) again. + +=== subscriber tables === + +Shows registered tables on the subscriber side, and the current state of +each table. Possible state values are: + +NEW:: + the table has not yet been considered by londiste. + +in-copy:: + Full-table copy is in progress. + +catching-up:: + Table is copied, missing events are replayed on to it. + +wanna-sync::: + The "copy" process catched up, wants to hand the table over to + "replay". + +do-sync::: + "replay" process is ready to accept it. + +ok:: + table is in sync. + + +== REPLICATION COMMANDS == + +=== replay === + +The actual replication process. Should be run as daemon with -d +switch, because it needs to be always running. + +It's main task is to get batches of events from PgQ and apply +them to subscriber database. + + +Switches: + + -d, --daemon: + go background + + -r, --reload: + reload config (send SIGHUP) + + -s, --stop: + stop program safely (send SIGINT) + + -k, --kill: + kill program immidiately (send SIGTERM) + +== UTILITY COMMAND == + +=== repair
... === + +Attempts to achieve a state where the table(s) is/are in sync, compares +them, and writes out SQL statements that would fix differences. + +Syncing happens by locking provider tables against updates and then +waiting until the replay process has applied all pending changes to +subscriber database. As this is dangerous operation, it has a hardwired +limit of 10 seconds for locking. If the replay process does not catch up +in that time, the locks are released and the repair operation is cancelled. + +Comparing happens by dumping out the table contents of both sides, +sorting them and then comparing line-by-line. As this is a CPU and +memory-hungry operation, good practice is to run the repair command on a +third machine to avoid consuming resources on either the provider or the +subscriber. + +=== compare
... === + +Syncs tables like repair, but just runs SELECT count(*) on both +sides to get a little bit cheaper, but also less precise, way of +checking if the tables are in sync. + +== CONFIGURATION == + +Londiste and PgQ both use INI configuration files, your distribution of +skytools include examples. You often just have to edit the database +connection strings, namely db in PgQ ticker.ini and provider_db and +subscriber_db in londiste conf.ini as well as logfile and pidfile to adapt to +you system paths. + +See `londiste(5)`. + + +== SEE ALSO == + +`londiste(5)` + +https://developer.skype.com/SkypeGarage/DbProjects/SkyTools/[] + diff --git a/doc/londiste.config.txt b/doc/londiste.config.txt new file mode 100644 index 00000000..92fc3d44 --- /dev/null +++ b/doc/londiste.config.txt @@ -0,0 +1,66 @@ += londiste(5) = + +== NAME == + +londiste - PostgreSQL replication engine written in python + +== SYNOPSIS == + + [londiste] + job_name = asd + +== DESCRIPTION == + +The londiste configuration file follow the famous .INI syntax. It +contains only one section named londiste. + +Most defaults values are reasonable ones. That means you can only edit +provider_db, subscriber_db and pgq_queue_name and be done with +londiste configuration. + +== OPTIONS == + +You can configure the following options into the londiste section. + +job_name:: + Each running londiste replay process has a unique job name. + +provider_db:: + Provider database connection string (DSN). + +subscriber_db:: + Subscriber database connection string (DSN). + +pgq_queue_name:: + Name of the PgQ to use. Must match the PgQ ticker configuration. + +logfile:: + Where to log londiste activity. + +pidfile:: + Where to store the pid of the main londiste process, the replay one. + +mirror_queue:: + TO DOCUMENT. + +== EXAMPLE == + + [londiste] + job_name = test_to_subcriber + + provider_db = dbname=provider port=6000 host=127.0.0.1 + subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 + + # it will be used as sql ident so no dots/spaces + pgq_queue_name = londiste.replika + + logfile = /tmp/%(job_name)s.log + pidfile = /tmp/%(job_name)s.pid + + # both events and ticks will be copied there + #mirror_queue = replika_mirror + + +== SEE ALSO == + +londiste(1) diff --git a/doc/londiste.txt b/doc/londiste.txt deleted file mode 100644 index bdb26a52..00000000 --- a/doc/londiste.txt +++ /dev/null @@ -1,64 +0,0 @@ - - -== Config file == - -{{{ -[londiste] -job_name = test_to_subcriber - -# source database, where the queue resides -provider_db = dbname=provider port=6000 host=127.0.0.1 - -# destination database -subscriber_db = dbname=subscriber port=6000 host=127.0.0.1 - -# the queue where to listen on -pgq_queue_name = londiste.replika - -# where to log -logfile = ~/log/%(job_name)s.log - -# pidfile is used for avoiding duplicate processes -pidfile = ~/pid/%(job_name)s.pid - -}}} - -== Command line overview == - -{{{ -$ londiste.py --help -usage: londiste.py [options] INI CMD [subcmd args] - -commands: - provider install installs modules, creates queue - provider add TBL ... add table to queue - provider remove TBL ... remove table from queue - provider tables show all tables linked to queue - subscriber install installs schema - subscriber register attaches subscriber to queue (also done by replay) - subscriber unregister detach subscriber from queue - subscriber add TBL ... add table to subscriber - subscriber remove TBL ... remove table from subscriber - subscriber resync TBL ... do full copy again - subscriber tables list tables subscriber has attached to - subscriber missing list tables subscriber has not yet attached to - subscriber check compare table structure on both sides - subscriber fkeys print out fkey drop/create commands - replay replay events to subscriber - copy full copy of table, internal cmd - compare [TBL ...] compare table contents on both sides - repair [TBL ...] repair data on subscriber - -options: - -h, --help show this help message and exit - -q, --quiet make program silent - -v, --verbose make program verbose - -d, --daemon go background - --expect-sync no copy needed (for add command) - --force ignore some warnings - - control running process: - -r, --reload reload config (send SIGHUP) - -s, --stop stop program safely (send SIGINT) - -k, --kill kill program immidiately (send SIGTERM) -}}} -- 2.39.5