Adding new wd_cli utility
authorMuhammad Usama <m.usama@gmail.com>
Mon, 23 Sep 2019 20:30:30 +0000 (01:30 +0500)
committerMuhammad Usama <m.usama@gmail.com>
Mon, 23 Sep 2019 20:30:30 +0000 (01:30 +0500)
wd_cli makes it easier to integrate the external health check systems with the
Pgpool-II. It provides a very convenient and easy to use interface to interact with
watchdog and perform health check related node operations.

For example:
If some external health-check system monitoring the health of the watchdog
cluster detects the remote node failure. It can use wd_cli utility to inform
Pgpool-II about that node failure

    $wd_cli  --inform DEAD -N 'Remote watchdog name'  -p 9001 -m 'node 1 failed'

Similarly when the node becomes reachable again

    $wd_cli  --inform ALIVE -N 'Remote watchdog name'  -p 9001 -m 'node 1 found'

This utility can also be used to get the live information of all nodes.

    $wd_cli -f simple_conf/watchdog/pgpool_wd1.conf --info -v --all

    Total Watchdog nodes configured for lifecheck:    2
    *****************
    Node ID:           0
    Node Status code   4
    Node Status:       MASTER
    Node Name:         localhost:9991 Linux localhost.localdomain
    Node Host:         localhost
    Node WD Port:      9001
    Node Pgpool Port:  9991

    Node ID:           1
    Node Status code   7
    Node Status:       STANDBY
    Node Name:         localhost:9992 Linux localhost.localdomain
    Node Host:         localhost
    Node WD Port:      9002
    Node Pgpool Port:  9992

wd_cli provides many options for node search criteria.
See wd_cli --help for complete list of options.

The commit also re-arrange the code related to socket communication and
watchdog IPC commands and separates the internal and external watchdog
commands to make the code structure more modular.

The regression tests and documentation updates are not part of this commit
and will be taken care of separately.

36 files changed:
configure
configure.ac
src/Makefile.am
src/Makefile.in
src/include/utils/fe_ports.h
src/include/utils/json_writer.h
src/include/utils/pool_stream.h
src/include/utils/socket_stream.h [new file with mode: 0644]
src/include/watchdog/wd_commands.h [new file with mode: 0644]
src/include/watchdog/wd_internal_commands.h [moved from src/include/watchdog/wd_ipc_commands.h with 66% similarity]
src/include/watchdog/wd_ipc_conn.h [new file with mode: 0644]
src/include/watchdog/wd_ipc_defines.h
src/include/watchdog/wd_json_data.h
src/include/watchdog/wd_lifecheck.h
src/main/pgpool_main.c
src/pcp_con/pcp_worker.c
src/pcp_con/recovery.c
src/protocol/child.c
src/protocol/pool_connection_pool.c
src/protocol/pool_process_query.c
src/tools/Makefile.am
src/tools/Makefile.in
src/tools/watchdog/Makefile.am [new file with mode: 0644]
src/tools/watchdog/Makefile.in [new file with mode: 0644]
src/tools/watchdog/wd_cli.c [new file with mode: 0644]
src/utils/pool_stream.c
src/utils/psprintf.c
src/utils/socket_stream.c [new file with mode: 0644]
src/watchdog/Makefile.am
src/watchdog/Makefile.in
src/watchdog/watchdog.c
src/watchdog/wd_commands.c
src/watchdog/wd_internal_commands.c [new file with mode: 0644]
src/watchdog/wd_ipc_conn.c [new file with mode: 0644]
src/watchdog/wd_json_data.c
src/watchdog/wd_lifecheck.c

index 99d088de6eb04fd76e86783be1c6b5cfd3b11cc8..395d8bc36535dd3fbb9f914b4839a33789c7c183 100755 (executable)
--- a/configure
+++ b/configure
@@ -15258,7 +15258,7 @@ $as_echo "enable cassert = $enable_cassert" >&6; }
 ac_config_headers="$ac_config_headers src/include/config.h"
 
 
-ac_config_files="$ac_config_files Makefile doc/Makefile doc/src/Makefile doc/src/sgml/Makefile doc.ja/Makefile doc.ja/src/Makefile doc.ja/src/sgml/Makefile src/Makefile src/include/Makefile src/parser/Makefile src/libs/Makefile src/libs/pcp/Makefile src/tools/Makefile src/tools/pgmd5/Makefile src/tools/pgenc/Makefile src/tools/pcp/Makefile src/tools/pgproto/Makefile src/watchdog/Makefile"
+ac_config_files="$ac_config_files Makefile doc/Makefile doc/src/Makefile doc/src/sgml/Makefile doc.ja/Makefile doc.ja/src/Makefile doc.ja/src/sgml/Makefile src/Makefile src/include/Makefile src/parser/Makefile src/libs/Makefile src/libs/pcp/Makefile src/tools/Makefile src/tools/pgmd5/Makefile src/tools/pgenc/Makefile src/tools/pcp/Makefile src/tools/pgproto/Makefile src/tools/watchdog/Makefile src/watchdog/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -16291,6 +16291,7 @@ do
     "src/tools/pgenc/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/pgenc/Makefile" ;;
     "src/tools/pcp/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/pcp/Makefile" ;;
     "src/tools/pgproto/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/pgproto/Makefile" ;;
+    "src/tools/watchdog/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/watchdog/Makefile" ;;
     "src/watchdog/Makefile") CONFIG_FILES="$CONFIG_FILES src/watchdog/Makefile" ;;
 
   *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
index 958fbc1ae7b57fc31f0436555e6f871d69b4e827..c686b789e4e249f966a5d505317086b8c4669bfb 100644 (file)
@@ -449,4 +449,4 @@ AC_MSG_RESULT([enable cassert = $enable_cassert])
 
 AM_CONFIG_HEADER(src/include/config.h)
 
-AC_OUTPUT([Makefile doc/Makefile  doc/src/Makefile doc/src/sgml/Makefile doc.ja/Makefile  doc.ja/src/Makefile doc.ja/src/sgml/Makefile src/Makefile src/include/Makefile src/parser/Makefile src/libs/Makefile src/libs/pcp/Makefile src/tools/Makefile src/tools/pgmd5/Makefile src/tools/pgenc/Makefile src/tools/pcp/Makefile src/tools/pgproto/Makefile src/watchdog/Makefile])
+AC_OUTPUT([Makefile doc/Makefile  doc/src/Makefile doc/src/sgml/Makefile doc.ja/Makefile  doc.ja/src/Makefile doc.ja/src/sgml/Makefile src/Makefile src/include/Makefile src/parser/Makefile src/libs/Makefile src/libs/pcp/Makefile src/tools/Makefile src/tools/pgmd5/Makefile src/tools/pgenc/Makefile src/tools/pcp/Makefile src/tools/pgproto/Makefile src/tools/watchdog/Makefile src/watchdog/Makefile])
index 9ee8f9061490041a1a18d5708f45db53693eab43..34e95b4a16624f0207d43d87742ad7f0b140610e 100644 (file)
@@ -45,6 +45,7 @@ pgpool_SOURCES = main/main.c \
        utils/pool_process_reporting.c \
        utils/pool_ssl.c \
        utils/pool_stream.c \
+       utils/socket_stream.c \
        utils/getopt_long.c \
        utils/mmgr/mcxt.c \
        utils/mmgr/aset.c \
index 9e71357d38d39553d6d60940511f5ba5baae5fb3..57f886476753c045eae7d8a598674e8ca627f92a 100644 (file)
@@ -129,13 +129,14 @@ am_pgpool_OBJECTS = main/main.$(OBJEXT) main/pool_globals.$(OBJEXT) \
        utils/pool_relcache.$(OBJEXT) \
        utils/pool_process_reporting.$(OBJEXT) \
        utils/pool_ssl.$(OBJEXT) utils/pool_stream.$(OBJEXT) \
-       utils/getopt_long.$(OBJEXT) utils/mmgr/mcxt.$(OBJEXT) \
-       utils/mmgr/aset.$(OBJEXT) utils/error/elog.$(OBJEXT) \
-       utils/error/assert.$(OBJEXT) utils/pcp/pcp_stream.$(OBJEXT) \
-       utils/regex_array.$(OBJEXT) utils/json_writer.$(OBJEXT) \
-       utils/json.$(OBJEXT) utils/scram-common.$(OBJEXT) \
-       utils/base64.$(OBJEXT) utils/sha2.$(OBJEXT) \
-       utils/ssl_utils.$(OBJEXT) utils/statistics.$(OBJEXT)
+       utils/socket_stream.$(OBJEXT) utils/getopt_long.$(OBJEXT) \
+       utils/mmgr/mcxt.$(OBJEXT) utils/mmgr/aset.$(OBJEXT) \
+       utils/error/elog.$(OBJEXT) utils/error/assert.$(OBJEXT) \
+       utils/pcp/pcp_stream.$(OBJEXT) utils/regex_array.$(OBJEXT) \
+       utils/json_writer.$(OBJEXT) utils/json.$(OBJEXT) \
+       utils/scram-common.$(OBJEXT) utils/base64.$(OBJEXT) \
+       utils/sha2.$(OBJEXT) utils/ssl_utils.$(OBJEXT) \
+       utils/statistics.$(OBJEXT)
 pgpool_OBJECTS = $(am_pgpool_OBJECTS)
 pgpool_DEPENDENCIES = parser/libsql-parser.a parser/nodes.o \
        watchdog/lib-watchdog.a
@@ -470,6 +471,7 @@ pgpool_SOURCES = main/main.c \
        utils/pool_process_reporting.c \
        utils/pool_ssl.c \
        utils/pool_stream.c \
+       utils/socket_stream.c \
        utils/getopt_long.c \
        utils/mmgr/mcxt.c \
        utils/mmgr/aset.c \
@@ -739,6 +741,7 @@ utils/pool_relcache.$(OBJEXT): utils/$(am__dirstamp)
 utils/pool_process_reporting.$(OBJEXT): utils/$(am__dirstamp)
 utils/pool_ssl.$(OBJEXT): utils/$(am__dirstamp)
 utils/pool_stream.$(OBJEXT): utils/$(am__dirstamp)
+utils/socket_stream.$(OBJEXT): utils/$(am__dirstamp)
 utils/getopt_long.$(OBJEXT): utils/$(am__dirstamp)
 utils/mmgr/$(am__dirstamp):
        @$(MKDIR_P) utils/mmgr
index 0561e993fd31eb16f22697510b9302dce08711d7..8a2ad201e7b196ae603dcf32f56f7ef6513af3d6 100644 (file)
@@ -31,7 +31,7 @@
 #include <stdlib.h>
 #ifndef FE_PORTS
 #define FE_PORTS
-
+#include "parser/pg_config_manual.h"
 #include "pool_type.h"
 
 
@@ -119,6 +119,13 @@ extern void errfinish(int dummy,...);
                                                                         * errors followed by readyForQuery
                                                                         * message */
 
+#define exprLocation(x)  errcode_ign(0)
+#define _(x) (x)
+#define gettext(x) (x)
+#define dgettext(d,x) (x)
+#define ngettext(s,p,n) ((n) == 1 ? (s) : (p))
+#define dngettext(d,s,p,n) ((n) == 1 ? (s) : (p))
+
 #define ereport(elevel, rest)  \
 do { \
        const int elevel_ = (elevel); \
@@ -137,4 +144,8 @@ typedef enum
        PGERROR_VERBOSE                         /* all the facts, ma'am */
 }                      PGErrorVerbosity;
 
+/* sprintf into a palloc'd buffer --- these are in psprintf.c */
+extern char *psprintf(const char *fmt,...) pg_attribute_printf(1, 2);
+extern size_t pvsnprintf(char *buf, size_t len, const char *fmt, va_list args) pg_attribute_printf(3, 0);
+
 #endif                                                 /* FE_PORTS */
index 87bca0d0dd681fb6a9f6808b60a510336668ac13..bc9c78b077aff21b4541120fd97eb288136c6f88 100644 (file)
@@ -22,7 +22,7 @@
 
 #ifndef pgpool_xcode_json_writer_h
 #define pgpool_xcode_json_writer_h
-
+#include "parser/stringinfo.h"
 #define MAX_STACK_DEPTH 10
 typedef enum JWElementType
 {
index 706a1d8d132e6ec0d4a41f90d08acfa7925b99c9..8ddba327b40e574208b97728ae46d1624389991d 100644 (file)
@@ -26,6 +26,8 @@
 #ifndef POOL_STREAM_H
 #define POOL_STREAM_H
 
+#include "utils/socket_stream.h"
+
 #define READBUFSZ 1024
 #define WRITEBUFSZ 8192
 
@@ -61,11 +63,6 @@ extern int   pool_unread(POOL_CONNECTION * cp, void *data, int len);
 extern int     pool_push(POOL_CONNECTION * cp, void *data, int len);
 extern void pool_pop(POOL_CONNECTION * cp, int *len);
 extern int     pool_stacklen(POOL_CONNECTION * cp);
-extern void pool_set_nonblock(int fd);
-extern void pool_unset_nonblock(int fd);
-
-extern int     socket_read(int sock, void *buf, size_t len, int timeout);
-extern int     socket_write(int fd, void *buf, size_t len);
 
 extern void pool_set_db_node_id(POOL_CONNECTION * con, int db_node_id);
 
diff --git a/src/include/utils/socket_stream.h b/src/include/utils/socket_stream.h
new file mode 100644 (file)
index 0000000..eac9fa9
--- /dev/null
@@ -0,0 +1,36 @@
+/* -*-pgsql-c-*- */
+/*
+ *
+ * $Header$
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * pool_steam.h.: pool_stream.c related header file
+ *
+ */
+
+#ifndef SOCKET_STREAM_H
+#define SOCKET_STREAM_H
+
+extern void socket_set_nonblock(int fd);
+extern void socket_unset_nonblock(int fd);
+
+extern int     socket_read(int sock, void *buf, size_t len, int timeout);
+extern int     socket_write(int fd, void *buf, size_t len);
+
+
+#endif                                                 /* SOCKET_STREAM_H */
diff --git a/src/include/watchdog/wd_commands.h b/src/include/watchdog/wd_commands.h
new file mode 100644 (file)
index 0000000..cbab402
--- /dev/null
@@ -0,0 +1,66 @@
+
+/* -*-pgsql-c-*- */
+/*
+ *
+ * $Header$
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ *
+ */
+
+#ifndef WD_COMMANDS_H
+#define WD_COMMANDS_H
+
+#include "watchdog/wd_ipc_defines.h"
+#include "watchdog/wd_ipc_conn.h"
+
+typedef struct WDNodeInfo
+{
+       int                     state;
+       char            nodeName[WD_MAX_HOST_NAMELEN];
+       char            hostName[WD_MAX_HOST_NAMELEN];  /* host name */
+       char            stateName[WD_MAX_HOST_NAMELEN]; /* watchdog state name */
+       int                     wd_port;                /* watchdog port */
+       int                     pgpool_port;    /* pgpool port */
+       int                     wd_priority;    /* node priority */
+       char            delegate_ip[WD_MAX_HOST_NAMELEN];       /* delegate IP */
+       int                     id;
+}                      WDNodeInfo;
+
+typedef struct WDGenericData
+{
+       WDValueDataType valueType;
+       union data
+       {
+               char       *stringVal;
+               int                     intVal;
+               bool            boolVal;
+               long            longVal;
+       }                       data;
+}                      WDGenericData;
+
+
+
+extern WDGenericData * get_wd_runtime_variable_value(char *wd_authkey, char *varName);
+extern WD_STATES get_watchdog_local_node_state(char *wd_authkey);
+extern int     get_watchdog_quorum_state(char *wd_authkey);
+extern char *wd_get_watchdog_nodes_json(char *wd_authkey, int nodeID);
+extern void set_wd_command_timeout(int sec);
+extern char* get_request_json(char *key, char *value, char *authKey);
+extern WDNodeInfo *parse_watchdog_node_info_from_wd_node_json(json_value * source);
+#endif                                                 /* WD_COMMANDS_H */
similarity index 66%
rename from src/include/watchdog/wd_ipc_commands.h
rename to src/include/watchdog/wd_internal_commands.h
index 78c78c6cd0fc3b99b6e75074a6fa98ede3706a26..e224018306700936e39ec56ec96d7b32fbb9f5ae 100644 (file)
@@ -7,7 +7,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2016     PgPool Global Development Group
+ * Copyright (c) 2003-2019     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
  *
  */
 
-#ifndef WD_IPC_COMMANDS_H
-#define WD_IPC_COMMANDS_H
+#ifndef WD_INTERNAL_COMMANDS_H
+#define WD_INTERNAL_COMMANDS_H
 
 #include "watchdog/wd_ipc_defines.h"
 #include "watchdog/wd_json_data.h"
-
-typedef enum WdCommandResult
-{
-       CLUSTER_IN_TRANSATIONING,
-       COMMAND_OK,
-       COMMAND_FAILED,
-       COMMAND_TIMEOUT
-}                      WdCommandResult;
-
-
-typedef struct WDIPCCmdResult
-{
-       char            type;
-       int                     length;
-       char       *data;
-}                      WDIPCCmdResult;
-
-typedef struct WDGenericData
-{
-       WDValueDataType valueType;
-       union data
-       {
-               char       *stringVal;
-               int                     intVal;
-               bool            boolVal;
-               long            longVal;
-       }                       data;
-}                      WDGenericData;
-
-
-extern void wd_ipc_initialize_data(void);
-extern char *get_watchdog_ipc_address(void);
-extern unsigned int *get_ipc_shared_key(void);
-extern void set_watchdog_process_needs_cleanup(void);
-extern void reset_watchdog_process_needs_cleanup(void);
-extern bool get_watchdog_process_needs_cleanup(void);
-extern void set_watchdog_node_escalated(void);
-extern void reset_watchdog_node_escalated(void);
-extern bool get_watchdog_node_escalation_state(void);
+#include "watchdog/wd_ipc_conn.h"
+#include "watchdog/wd_commands.h"
 
 extern WdCommandResult wd_start_recovery(void);
 extern WdCommandResult wd_end_recovery(void);
@@ -75,19 +38,24 @@ extern WDFailoverCMDResults wd_degenerate_backend_set(int *node_id_set, int coun
 extern WDFailoverCMDResults wd_promote_backend(int node_id, unsigned char flags);
 
 extern WDPGBackendStatus * get_pg_backend_status_from_master_wd_node(void);
-extern WDGenericData * get_wd_runtime_variable_value(char *varName);
-extern WD_STATES get_watchdog_local_node_state(void);
-extern int     get_watchdog_quorum_state(void);
-
-extern char *wd_get_watchdog_nodes(int nodeID);
 
-extern WDIPCCmdResult * issue_command_to_watchdog(char type, int timeout_sec, char *data, int data_len, bool blocking);
+extern WD_STATES wd_internal_get_watchdog_local_node_state(void);
+extern int     wd_internal_get_watchdog_quorum_state(void);
+extern char *wd_internal_get_watchdog_nodes_json(int nodeID);
 
+extern void wd_ipc_initialize_data(void);
 
 /* functions for failover commands interlocking */
 extern WDFailoverCMDResults wd_failover_end(void);
 extern WDFailoverCMDResults wd_failover_start(void);
 
+extern unsigned int *get_ipc_shared_key(void);
+extern void set_watchdog_process_needs_cleanup(void);
+extern void reset_watchdog_process_needs_cleanup(void);
+extern bool get_watchdog_process_needs_cleanup(void);
+extern void set_watchdog_node_escalated(void);
+extern void reset_watchdog_node_escalated(void);
+extern bool get_watchdog_node_escalation_state(void);
 
 
-#endif                                                 /* WD_IPC_COMMANDS_H */
+#endif                                                 /* WD_INTERNAL_COMMANDS_H */
diff --git a/src/include/watchdog/wd_ipc_conn.h b/src/include/watchdog/wd_ipc_conn.h
new file mode 100644 (file)
index 0000000..aa7ad1d
--- /dev/null
@@ -0,0 +1,57 @@
+
+/* -*-pgsql-c-*- */
+/*
+ *
+ * $Header$
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ *
+ */
+
+#ifndef WD_IPC_CONN_H
+#define WD_IPC_CONN_H
+
+#include "watchdog/wd_ipc_defines.h"
+#include "watchdog/wd_json_data.h"
+
+typedef enum WdCommandResult
+{
+       CLUSTER_IN_TRANSATIONING,
+       COMMAND_OK,
+       COMMAND_FAILED,
+       COMMAND_TIMEOUT
+}                      WdCommandResult;
+
+
+typedef struct WDIPCCmdResult
+{
+       char            type;
+       int                     length;
+       char       *data;
+}                      WDIPCCmdResult;
+
+
+extern void wd_ipc_conn_initialize(void);
+extern void wd_set_ipc_address(char *socket_dir, int port);
+extern char *get_watchdog_ipc_address(void);
+
+extern WDIPCCmdResult * issue_command_to_watchdog(char type, int timeout_sec, char *data, int data_len, bool blocking);
+
+extern void FreeCmdResult(WDIPCCmdResult * res);
+
+#endif                                                 /* WD_IPC_CONN_H */
index 1e19e8e5b833a99511c51bf3d0c6746e77c42061..0bda9c1bb97ca7009dee359bcc4dc4463fad10af 100644 (file)
@@ -6,7 +6,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2017     PgPool Global Development Group
+ * Copyright (c) 2003-2019     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
index 1e22ee531c1b05fe2f3bacfb92a096c165b38eae..0080a217e26f26aaed7b3d7bf9bf3b42200e607d 100644 (file)
 #ifndef WD_JSON_DATA_H
 #define WD_JSON_DATA_H
 
-typedef struct WDNodeInfo
-{
-       int                     state;
-       char            nodeName[WD_MAX_HOST_NAMELEN];
-       char            hostName[WD_MAX_HOST_NAMELEN];  /* host name */
-       char            stateName[WD_MAX_HOST_NAMELEN]; /* watchdog state name */
-       int                     wd_port;                /* watchdog port */
-       int                     pgpool_port;    /* pgpool port */
-       int                     wd_priority;    /* node priority */
-       char            delegate_ip[WD_MAX_HOST_NAMELEN];       /* delegate IP */
-       int                     id;
-}                      WDNodeInfo;
-
 /*
  * The structure to hold the parsed PG backend node status data fetched
  * from the master watchdog node
@@ -61,7 +48,6 @@ extern char *get_pool_config_json(void);
 extern char *get_lifecheck_node_status_change_json(int nodeID, int nodeStatus, char *message, char *authKey);
 extern bool parse_node_status_json(char *json_data, int data_len, int *nodeID, int *nodeStatus, char **message);
 
-extern WDNodeInfo * get_WDNodeInfo_from_wd_node_json(json_value * source);
 
 extern bool parse_beacon_message_json(char *json_data, int data_len, int *state,
                                                  long *seconds_since_node_startup,
index d612ab38e34e7d67fb91495900b63f17f91d91c9..2f0680a574076187fa0bd8b2078f2c3604c77305 100644 (file)
@@ -39,7 +39,9 @@ typedef enum NodeState
 typedef struct LifeCheckNode
 {
        NodeStates      nodeState;
-       int ID;
+       int             ID;
+       WD_STATES       wdState;
+       char            stateName[128];
        char            hostName[128];
        char            nodeName[128];
        int                     wdPort;
index 1a03927bced31f65178304ea2117af95850acecd..b24f513aa1d97562f01adb14aa39f3cde91cf92c 100644 (file)
@@ -57,7 +57,7 @@
 #include "auth/pool_passwd.h"
 #include "auth/pool_hba.h"
 #include "query_cache/pool_memqcache.h"
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_internal_commands.h"
 #include "watchdog/wd_lifecheck.h"
 
 #include "watchdog/watchdog.h"
@@ -1552,7 +1552,7 @@ sigusr1_interupt_processor(void)
                                (errmsg("Pgpool-II parent process received watchdog quorum change signal from watchdog")));
 
                user1SignalSlot->signalFlags[SIG_WATCHDOG_QUORUM_CHANGED] = false;
-               if (get_watchdog_quorum_state() >= 0)
+               if (wd_internal_get_watchdog_quorum_state() >= 0)
                {
                        ereport(LOG,
                                        (errmsg("watchdog cluster now holds the quorum"),
@@ -1576,7 +1576,7 @@ sigusr1_interupt_processor(void)
                                (errmsg("Pgpool-II parent process received sync backend signal from watchdog")));
 
                user1SignalSlot->signalFlags[SIG_BACKEND_SYNC_REQUIRED] = false;
-               if (get_watchdog_local_node_state() == WD_STANDBY)
+               if (wd_internal_get_watchdog_local_node_state() == WD_STANDBY)
                {
                        ereport(LOG,
                                        (errmsg("master watchdog has performed failover"),
@@ -1591,7 +1591,7 @@ sigusr1_interupt_processor(void)
                                (errmsg("Pgpool-II parent process received watchdog state change signal from watchdog")));
 
                user1SignalSlot->signalFlags[SIG_WATCHDOG_STATE_CHANGED] = false;
-               if (get_watchdog_local_node_state() == WD_STANDBY)
+               if (wd_internal_get_watchdog_local_node_state() == WD_STANDBY)
                {
                        ereport(LOG,
                                        (errmsg("we have joined the watchdog cluster as STANDBY node"),
@@ -4076,7 +4076,7 @@ update_backend_quarantine_status(void)
         * Reset the quarantine flag from each backend and set it to con_wait
         */
        int                     i;
-       WD_STATES       wd_state = get_watchdog_local_node_state();
+       WD_STATES       wd_state = wd_internal_get_watchdog_local_node_state();
 
        for (i = 0; i < NUM_BACKENDS; i++)
        {
index 4ce2c215a19561651245308b1c4fed873c9b3008..d33a3f4b44065fabc6bd3293825b63d1438f819f 100644 (file)
@@ -47,7 +47,7 @@
 #include "context/pool_process_context.h"
 #include "utils/pool_process_reporting.h"
 #include "watchdog/wd_json_data.h"
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_internal_commands.h"
 #include "utils/elog.h"
 
 #define MAX_FILE_LINE_LEN    512
@@ -782,7 +782,7 @@ inform_watchdog_info(PCP_CONNECTION * frontend, char *buf)
 
        wd_index = atoi(buf);
 
-       json_data = wd_get_watchdog_nodes(wd_index);
+       json_data = wd_internal_get_watchdog_nodes_json(wd_index);
        if (json_data == NULL)
                ereport(ERROR,
                                (errmsg("PCP: informing watchdog info failed"),
index a7cc4c7abab8213ab7e04998f6f53eb13d0a0fe2..7bb98de8ef57e712f67496ef0c5b75afaac88c19 100644 (file)
@@ -33,7 +33,7 @@
 
 #include "libpq-fe.h"
 
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_internal_commands.h"
 
 #define WAIT_RETRY_COUNT (pool_config->recovery_timeout / 3)
 
index 6c1d72c223f14410fa6d51860da50264bc7d454d..aa76d5c56a81176160e512165f2164a3aaa65ab5 100644 (file)
@@ -163,7 +163,7 @@ do_child(int *fds)
 #ifdef NONE_BLOCK
        /* set listen fds to none-blocking */
        for (walk = fds; *walk != -1; walk++)
-               pool_set_nonblock(*walk);
+               socket_set_nonblock(*walk);
 #endif
        for (walk = fds; *walk != -1; walk++)
        {
@@ -355,7 +355,7 @@ do_child(int *fds)
                child_frontend = get_connection(front_end_fd, &saddr);
 
                /* set frontend fd to blocking */
-               pool_unset_nonblock(child_frontend->fd);
+               socket_unset_nonblock(child_frontend->fd);
 
                /* reset busy flag */
                idle = 0;
@@ -1518,9 +1518,9 @@ discard_persistent_db_connection(POOL_CONNECTION_POOL_SLOT * cp)
         * This could happen in copy command (remember the famous "lost
         * synchronization with server, resetting connection" message)
         */
-       pool_set_nonblock(cp->con->fd);
+       socket_set_nonblock(cp->con->fd);
        pool_flush_it(cp->con);
-       pool_unset_nonblock(cp->con->fd);
+       socket_unset_nonblock(cp->con->fd);
 
        pool_close(cp->con);
        free_persisten_db_connection_memory(cp);
@@ -1859,7 +1859,7 @@ wait_for_new_connections(int *fds, struct timeval *timeout, SockAddr *saddr)
                                tmback = {0, 0};
 
        for (walk = fds; *walk != -1; walk++)
-               pool_set_nonblock(*walk);
+               socket_set_nonblock(*walk);
 
        if (SERIALIZE_ACCEPT)
                set_ps_display("wait for accept lock", false);
@@ -2042,7 +2042,7 @@ retry_accept:
        /*
         * Make sure that the socket is non blocking.
         */
-       pool_unset_nonblock(afd);
+       socket_unset_nonblock(afd);
 
 #ifdef ACCEPT_PERFORMANCE
        gettimeofday(&now2, 0);
@@ -2439,9 +2439,9 @@ set_pg_frontend_blocking(bool blocking)
        if (child_frontend->socket_state != POOL_SOCKET_VALID)
                return -1;
        if (blocking)
-               pool_unset_nonblock(child_frontend->fd);
+               socket_unset_nonblock(child_frontend->fd);
        else
-               pool_set_nonblock(child_frontend->fd);
+               socket_set_nonblock(child_frontend->fd);
        return 0;
 }
 
index 2c3ba11f33f4bd99ed33127f8d9e8c2f61d1d8ce..325fd1bcb27423b0b75a3347054f14448b8f9940 100644 (file)
@@ -568,7 +568,7 @@ connect_with_timeout(int fd, struct addrinfo *walk, char *host, int port, bool r
        int                     error;
        socklen_t       socklen;
 
-       pool_set_nonblock(fd);
+       socket_set_nonblock(fd);
 
        for (;;)
        {
@@ -733,7 +733,7 @@ connect_with_timeout(int fd, struct addrinfo *walk, char *host, int port, bool r
                break;
        }
 
-       pool_unset_nonblock(fd);
+       socket_unset_nonblock(fd);
        return true;
 }
 
index 2760ef16a592cd3ed31c6df6c0f9bc7d0c2c2c64..a36a972448373fbd5a4cab527bb7eb87c892d288 100644 (file)
@@ -673,9 +673,9 @@ pool_send_frontend_exits(POOL_CONNECTION_POOL * backend)
                         * famous "lost synchronization with server, resetting connection"
                         * message)
                         */
-                       pool_set_nonblock(CONNECTION(backend, i)->fd);
+                       socket_set_nonblock(CONNECTION(backend, i)->fd);
                        pool_flush_it(CONNECTION(backend, i));
-                       pool_unset_nonblock(CONNECTION(backend, i)->fd);
+                       socket_unset_nonblock(CONNECTION(backend, i)->fd);
                }
        }
 }
@@ -1282,7 +1282,7 @@ pool_send_severity_message(POOL_CONNECTION * frontend, int protoMajor,
 */
 #define MAXDATA        (MAXMSGBUF+1)*7+1
 
-       pool_set_nonblock(frontend->fd);
+       socket_set_nonblock(frontend->fd);
 
        if (protoMajor == PROTO_MAJOR_V2)
        {
@@ -1363,7 +1363,7 @@ pool_send_severity_message(POOL_CONNECTION * frontend, int protoMajor,
                ereport(ERROR,
                                (errmsg("send_error_message: unknown protocol major %d", protoMajor)));
 
-       pool_unset_nonblock(frontend->fd);
+       socket_unset_nonblock(frontend->fd);
 }
 
 void
index 55d31cf6d6c504f9193ccfa5198ab749f1630756..2ecc723f7ffe471b637c4fc063fa254eaa7ef8b1 100644 (file)
@@ -1,12 +1,9 @@
-SUBDIRS = pcp pgmd5 pgenc pgproto
+SUBDIRS = pcp pgmd5 pgenc pgproto watchdog
 
-dist_bin_SCRIPTS =  pgpool_setup watchdog_setup
+dist_bin_SCRIPTS =  pgpool_setup
 
 pgpool_setup:
        ln -s ../test/pgpool_setup .
 
-watchdog_setup:
-       ln -s ../test/watchdog_setup .
-
 clean-local:
        -rm -f $(dist_bin_SCRIPTS)
index 7e42c9d077c31206aa4ce89d5066a2958354a769..f521c9295c50e93a20607aa230b60fa78495efc1 100644 (file)
@@ -344,8 +344,8 @@ target_alias = @target_alias@
 top_build_prefix = @top_build_prefix@
 top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
-SUBDIRS = pcp pgmd5 pgenc pgproto
-dist_bin_SCRIPTS = pgpool_setup watchdog_setup
+SUBDIRS = pcp pgmd5 pgenc pgproto watchdog
+dist_bin_SCRIPTS = pgpool_setup
 all: all-recursive
 
 .SUFFIXES:
@@ -700,9 +700,6 @@ uninstall-am: uninstall-dist_binSCRIPTS
 pgpool_setup:
        ln -s ../test/pgpool_setup .
 
-watchdog_setup:
-       ln -s ../test/watchdog_setup .
-
 clean-local:
        -rm -f $(dist_bin_SCRIPTS)
 
diff --git a/src/tools/watchdog/Makefile.am b/src/tools/watchdog/Makefile.am
new file mode 100644 (file)
index 0000000..e3ab79b
--- /dev/null
@@ -0,0 +1,74 @@
+parser_incdir = $(top_srcdir)/src/include/parser
+AM_CPPFLAGS = -D_GNU_SOURCE -DPOOL_PRIVATE -I $(parser_incdir) -I @PGSQL_INCLUDE_DIR@ 
+bin_PROGRAMS = wd_cli
+
+dist_bin_SCRIPTS = watchdog_setup
+
+
+dist_wd_cli_SOURCES = wd_cli.c \
+               ../fe_port.c
+nodist_wd_cli_SOURCES = ssl_utils.c \
+               wd_ipc_conn.c \
+               wd_commands.c \
+               json_writer.c \
+               json.c \
+               pool_config_variables.c \
+               pool_config.c \
+               fe_memutils.c \
+               stringinfo.h \
+               stringinfo.c \
+               strlcpy.c \
+               socket_stream.c \
+               regex_array.c \
+               psprintf.c \
+               md5.c \
+               pool_globals.c
+
+DEFS = @DEFS@ \
+    -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\" -DPOOL_TOOLS
+
+watchdog_setup:
+       ln -s ../../test/watchdog_setup .
+psprintf.c: ../../../src/utils/psprintf.c
+       rm -f $@ && ln -s $< .
+md5.c: ../../../src/auth/md5.c
+       rm -f $@ && ln -s $< .
+md5.h: ../../../src/include/auth/md5.h
+       rm -f $@ && ln -s $< .
+socket_stream.c: ../../../src/utils/socket_stream.c
+       rm -f $@ && ln -s $< .
+strlcpy.c: ../../../src/utils/strlcpy.c
+       rm -f $@ && ln -s $< .
+regex_array.c: ../../../src/utils/regex_array.c
+       rm -f $@ && ln -s $< .
+wd_ipc_conn.c: ../../../src/watchdog/wd_ipc_conn.c
+       rm -f $@ && ln -s $< .
+wd_commands.c: ../../../src/watchdog/wd_commands.c
+       rm -f $@ && ln -s $< .
+wd_utils.c: ../../../src/watchdog/wd_utils.c
+       rm -f $@ && ln -s $< .
+ssl_utils.c: ../../../src/utils/ssl_utils.c
+       rm -f $@ && ln -s $< .
+base64.h: ../../../src/include/utils/base64.h
+       rm -f $@ && ln -s $< .
+ssl_utils.h: ../../../src/include/utils/ssl_utils.h
+       rm -f $@ && ln -s $< .
+json_writer.c: ../../../src/utils/json_writer.c
+       rm -f $@ && ln -s $< .
+json.c: ../../../src/utils/json.c
+       rm -f $@ && ln -s $< .
+stringinfo.c: ../../../src/parser/stringinfo.c
+       rm -f $@ && ln -s $< .
+stringinfo.h: ../../../src/include/parser/stringinfo.h
+       rm -f $@ && ln -s $< .
+pool_config_variables.c: ../../../src/config/pool_config_variables.c
+       rm -f $@ && ln -s $< .
+pool_config.c: ../../../src/config/pool_config.c
+       rm -f $@ && ln -s $< .
+fe_memutils.c: ../../../src/tools/fe_memutils.c
+       rm -f $@ && ln -s $< .
+pool_globals.c: ../../../src/main/pool_globals.c
+       rm -f $@ && ln -s $< .
+
+clean-local:
+       -rm -f $(nodist_wd_cli_SOURCES) $(dist_bin_SCRIPTS)
diff --git a/src/tools/watchdog/Makefile.in b/src/tools/watchdog/Makefile.in
new file mode 100644 (file)
index 0000000..6a1f66e
--- /dev/null
@@ -0,0 +1,771 @@
+# Makefile.in generated by automake 1.13.4 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2013 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = test -n '$(MAKEFILE_LIST)' && test -n '$(MAKELEVEL)'
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \  ]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs  ]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+bin_PROGRAMS = wd_cli$(EXEEXT)
+subdir = src/tools/watchdog
+DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
+       $(top_srcdir)/mkinstalldirs $(dist_bin_SCRIPTS)
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/docbook.m4 \
+       $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
+       $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+       $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/c-compiler.m4 \
+       $(top_srcdir)/c-library.m4 $(top_srcdir)/general.m4 \
+       $(top_srcdir)/ac_func_accept_argtypes.m4 \
+       $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+       $(ACLOCAL_M4)
+mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
+CONFIG_HEADER = $(top_builddir)/src/include/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)"
+PROGRAMS = $(bin_PROGRAMS)
+am__dirstamp = $(am__leading_dot)dirstamp
+dist_wd_cli_OBJECTS = wd_cli.$(OBJEXT) ../fe_port.$(OBJEXT)
+nodist_wd_cli_OBJECTS = ssl_utils.$(OBJEXT) wd_ipc_conn.$(OBJEXT) \
+       wd_commands.$(OBJEXT) json_writer.$(OBJEXT) json.$(OBJEXT) \
+       pool_config_variables.$(OBJEXT) pool_config.$(OBJEXT) \
+       fe_memutils.$(OBJEXT) stringinfo.$(OBJEXT) strlcpy.$(OBJEXT) \
+       socket_stream.$(OBJEXT) regex_array.$(OBJEXT) \
+       psprintf.$(OBJEXT) md5.$(OBJEXT) pool_globals.$(OBJEXT)
+wd_cli_OBJECTS = $(dist_wd_cli_OBJECTS) $(nodist_wd_cli_OBJECTS)
+wd_cli_LDADD = $(LDADD)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+SCRIPTS = $(dist_bin_SCRIPTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)/src/include
+depcomp =
+am__depfiles_maybe =
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+       $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+       $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+       $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+       $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+       $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+       $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(dist_wd_cli_SOURCES) $(nodist_wd_cli_SOURCES)
+DIST_SOURCES = $(dist_wd_cli_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CATALOG = @CATALOG@
+CC = @CC@
+CFLAGS = @CFLAGS@
+COLLATEINDEX = @COLLATEINDEX@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@ \
+    -DDEFAULT_CONFIGDIR=\"$(sysconfdir)\" -DPOOL_TOOLS
+
+DLLTOOL = @DLLTOOL@
+DOCBOOKSTYLE = @DOCBOOKSTYLE@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+JADE = @JADE@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LEX = @LEX@
+LEXLIB = @LEXLIB@
+LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+LYNX = @LYNX@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MEMCACHED_DIR = @MEMCACHED_DIR@
+MEMCACHED_INCLUDE_OPT = @MEMCACHED_INCLUDE_OPT@
+MEMCACHED_LINK_OPT = @MEMCACHED_LINK_OPT@
+MEMCACHED_RPATH_OPT = @MEMCACHED_RPATH_OPT@
+MKDIR_P = @MKDIR_P@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NSGMLS = @NSGMLS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OSX = @OSX@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PERL = @PERL@
+PGCONFIG = @PGCONFIG@
+PGSQL_INCLUDE_DIR = @PGSQL_INCLUDE_DIR@
+PGSQL_LIB_DIR = @PGSQL_LIB_DIR@
+RANLIB = @RANLIB@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+STYLE = @STYLE@
+SUNIFDEF = @SUNIFDEF@
+VERSION = @VERSION@
+XMLLINT = @XMLLINT@
+XSLTPROC = @XSLTPROC@
+XSLTPROC_HTML_FLAGS = @XSLTPROC_HTML_FLAGS@
+YACC = @YACC@
+YFLAGS = @YFLAGS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__leading_dot = @am__leading_dot@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+have_docbook = @have_docbook@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+parser_incdir = $(top_srcdir)/src/include/parser
+AM_CPPFLAGS = -D_GNU_SOURCE -DPOOL_PRIVATE -I $(parser_incdir) -I @PGSQL_INCLUDE_DIR@ 
+dist_bin_SCRIPTS = watchdog_setup
+dist_wd_cli_SOURCES = wd_cli.c \
+               ../fe_port.c
+
+nodist_wd_cli_SOURCES = ssl_utils.c \
+               wd_ipc_conn.c \
+               wd_commands.c \
+               json_writer.c \
+               json.c \
+               pool_config_variables.c \
+               pool_config.c \
+               fe_memutils.c \
+               stringinfo.h \
+               stringinfo.c \
+               strlcpy.c \
+               socket_stream.c \
+               regex_array.c \
+               psprintf.c \
+               md5.c \
+               pool_globals.c
+
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+       @for dep in $?; do \
+         case '$(am__configure_deps)' in \
+           *$$dep*) \
+             ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+               && { if test -f $@; then exit 0; else break; fi; }; \
+             exit 1;; \
+         esac; \
+       done; \
+       echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign --ignore-deps src/tools/watchdog/Makefile'; \
+       $(am__cd) $(top_srcdir) && \
+         $(AUTOMAKE) --foreign --ignore-deps src/tools/watchdog/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+       @case '$?' in \
+         *config.status*) \
+           cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+         *) \
+           echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+           cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+       esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+       cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+       cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+       cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-binPROGRAMS: $(bin_PROGRAMS)
+       @$(NORMAL_INSTALL)
+       @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+       if test -n "$$list"; then \
+         echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \
+         $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \
+       fi; \
+       for p in $$list; do echo "$$p $$p"; done | \
+       sed 's/$(EXEEXT)$$//' | \
+       while read p p1; do if test -f $$p \
+        || test -f $$p1 \
+         ; then echo "$$p"; echo "$$p"; else :; fi; \
+       done | \
+       sed -e 'p;s,.*/,,;n;h' \
+           -e 's|.*|.|' \
+           -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
+       sed 'N;N;N;s,\n, ,g' | \
+       $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
+         { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+           if ($$2 == $$4) files[d] = files[d] " " $$1; \
+           else { print "f", $$3 "/" $$4, $$1; } } \
+         END { for (d in files) print "f", d, files[d] }' | \
+       while read type dir files; do \
+           if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+           test -z "$$files" || { \
+           echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \
+           $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \
+           } \
+       ; done
+
+uninstall-binPROGRAMS:
+       @$(NORMAL_UNINSTALL)
+       @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+       files=`for p in $$list; do echo "$$p"; done | \
+         sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
+             -e 's/$$/$(EXEEXT)/' \
+       `; \
+       test -n "$$list" || exit 0; \
+       echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \
+       cd "$(DESTDIR)$(bindir)" && rm -f $$files
+
+clean-binPROGRAMS:
+       @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \
+       echo " rm -f" $$list; \
+       rm -f $$list || exit $$?; \
+       test -n "$(EXEEXT)" || exit 0; \
+       list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+       echo " rm -f" $$list; \
+       rm -f $$list
+../$(am__dirstamp):
+       @$(MKDIR_P) ..
+       @: > ../$(am__dirstamp)
+../fe_port.$(OBJEXT): ../$(am__dirstamp)
+
+wd_cli$(EXEEXT): $(wd_cli_OBJECTS) $(wd_cli_DEPENDENCIES) $(EXTRA_wd_cli_DEPENDENCIES) 
+       @rm -f wd_cli$(EXEEXT)
+       $(AM_V_CCLD)$(LINK) $(wd_cli_OBJECTS) $(wd_cli_LDADD) $(LIBS)
+install-dist_binSCRIPTS: $(dist_bin_SCRIPTS)
+       @$(NORMAL_INSTALL)
+       @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || list=; \
+       if test -n "$$list"; then \
+         echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \
+         $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \
+       fi; \
+       for p in $$list; do \
+         if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+         if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \
+       done | \
+       sed -e 'p;s,.*/,,;n' \
+           -e 'h;s|.*|.|' \
+           -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \
+       $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \
+         { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+           if ($$2 == $$4) { files[d] = files[d] " " $$1; \
+             if (++n[d] == $(am__install_max)) { \
+               print "f", d, files[d]; n[d] = 0; files[d] = "" } } \
+           else { print "f", d "/" $$4, $$1 } } \
+         END { for (d in files) print "f", d, files[d] }' | \
+       while read type dir files; do \
+            if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+            test -z "$$files" || { \
+              echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \
+              $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \
+            } \
+       ; done
+
+uninstall-dist_binSCRIPTS:
+       @$(NORMAL_UNINSTALL)
+       @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \
+       files=`for p in $$list; do echo "$$p"; done | \
+              sed -e 's,.*/,,;$(transform)'`; \
+       dir='$(DESTDIR)$(bindir)'; $(am__uninstall_files_from_dir)
+
+mostlyclean-compile:
+       -rm -f *.$(OBJEXT)
+       -rm -f ../*.$(OBJEXT)
+
+distclean-compile:
+       -rm -f *.tab.c
+
+.c.o:
+       $(AM_V_CC)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+       $(AM_V_CC)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+       $(AM_V_CC)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+       -rm -f *.lo
+
+clean-libtool:
+       -rm -rf .libs _libs
+
+ID: $(am__tagged_files)
+       $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+       set x; \
+       here=`pwd`; \
+       $(am__define_uniq_tagged_files); \
+       shift; \
+       if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+         test -n "$$unique" || unique=$$empty_fix; \
+         if test $$# -gt 0; then \
+           $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+             "$$@" $$unique; \
+         else \
+           $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+             $$unique; \
+         fi; \
+       fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+       $(am__define_uniq_tagged_files); \
+       test -z "$(CTAGS_ARGS)$$unique" \
+         || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+            $$unique
+
+GTAGS:
+       here=`$(am__cd) $(top_builddir) && pwd` \
+         && $(am__cd) $(top_srcdir) \
+         && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+       list='$(am__tagged_files)'; \
+       case "$(srcdir)" in \
+         [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+         *) sdir=$(subdir)/$(srcdir) ;; \
+       esac; \
+       for i in $$list; do \
+         if test -f "$$i"; then \
+           echo "$(subdir)/$$i"; \
+         else \
+           echo "$$sdir/$$i"; \
+         fi; \
+       done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+       -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(DISTFILES)
+       @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+       topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+       list='$(DISTFILES)'; \
+         dist_files=`for file in $$list; do echo $$file; done | \
+         sed -e "s|^$$srcdirstrip/||;t" \
+             -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+       case $$dist_files in \
+         */*) $(MKDIR_P) `echo "$$dist_files" | \
+                          sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+                          sort -u` ;; \
+       esac; \
+       for file in $$dist_files; do \
+         if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+         if test -d $$d/$$file; then \
+           dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+           if test -d "$(distdir)/$$file"; then \
+             find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+           fi; \
+           if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+             cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+             find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+           fi; \
+           cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+         else \
+           test -f "$(distdir)/$$file" \
+           || cp -p $$d/$$file "$(distdir)/$$file" \
+           || exit 1; \
+         fi; \
+       done
+check-am: all-am
+check: check-am
+all-am: Makefile $(PROGRAMS) $(SCRIPTS)
+installdirs:
+       for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)"; do \
+         test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+       done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+       @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+       if test -z '$(STRIP)'; then \
+         $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+           install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+             install; \
+       else \
+         $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+           install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+           "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+       fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+       -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+       -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+       -rm -f ../$(am__dirstamp)
+
+maintainer-clean-generic:
+       @echo "This command is intended for maintainers to use"
+       @echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-binPROGRAMS clean-generic clean-libtool clean-local \
+       mostlyclean-am
+
+distclean: distclean-am
+       -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+       distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am:
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-binPROGRAMS install-dist_binSCRIPTS
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+       -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+       mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-binPROGRAMS uninstall-dist_binSCRIPTS
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am check check-am clean \
+       clean-binPROGRAMS clean-generic clean-libtool clean-local \
+       cscopelist-am ctags ctags-am distclean distclean-compile \
+       distclean-generic distclean-libtool distclean-tags distdir dvi \
+       dvi-am html html-am info info-am install install-am \
+       install-binPROGRAMS install-data install-data-am \
+       install-dist_binSCRIPTS install-dvi install-dvi-am \
+       install-exec install-exec-am install-html install-html-am \
+       install-info install-info-am install-man install-pdf \
+       install-pdf-am install-ps install-ps-am install-strip \
+       installcheck installcheck-am installdirs maintainer-clean \
+       maintainer-clean-generic mostlyclean mostlyclean-compile \
+       mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+       tags tags-am uninstall uninstall-am uninstall-binPROGRAMS \
+       uninstall-dist_binSCRIPTS
+
+
+watchdog_setup:
+       ln -s ../../test/watchdog_setup .
+psprintf.c: ../../../src/utils/psprintf.c
+       rm -f $@ && ln -s $< .
+md5.c: ../../../src/auth/md5.c
+       rm -f $@ && ln -s $< .
+md5.h: ../../../src/include/auth/md5.h
+       rm -f $@ && ln -s $< .
+socket_stream.c: ../../../src/utils/socket_stream.c
+       rm -f $@ && ln -s $< .
+strlcpy.c: ../../../src/utils/strlcpy.c
+       rm -f $@ && ln -s $< .
+regex_array.c: ../../../src/utils/regex_array.c
+       rm -f $@ && ln -s $< .
+wd_ipc_conn.c: ../../../src/watchdog/wd_ipc_conn.c
+       rm -f $@ && ln -s $< .
+wd_commands.c: ../../../src/watchdog/wd_commands.c
+       rm -f $@ && ln -s $< .
+wd_utils.c: ../../../src/watchdog/wd_utils.c
+       rm -f $@ && ln -s $< .
+ssl_utils.c: ../../../src/utils/ssl_utils.c
+       rm -f $@ && ln -s $< .
+base64.h: ../../../src/include/utils/base64.h
+       rm -f $@ && ln -s $< .
+ssl_utils.h: ../../../src/include/utils/ssl_utils.h
+       rm -f $@ && ln -s $< .
+json_writer.c: ../../../src/utils/json_writer.c
+       rm -f $@ && ln -s $< .
+json.c: ../../../src/utils/json.c
+       rm -f $@ && ln -s $< .
+stringinfo.c: ../../../src/parser/stringinfo.c
+       rm -f $@ && ln -s $< .
+stringinfo.h: ../../../src/include/parser/stringinfo.h
+       rm -f $@ && ln -s $< .
+pool_config_variables.c: ../../../src/config/pool_config_variables.c
+       rm -f $@ && ln -s $< .
+pool_config.c: ../../../src/config/pool_config.c
+       rm -f $@ && ln -s $< .
+fe_memutils.c: ../../../src/tools/fe_memutils.c
+       rm -f $@ && ln -s $< .
+pool_globals.c: ../../../src/main/pool_globals.c
+       rm -f $@ && ln -s $< .
+
+clean-local:
+       -rm -f $(nodist_wd_cli_SOURCES) $(dist_bin_SCRIPTS)
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/tools/watchdog/wd_cli.c b/src/tools/watchdog/wd_cli.c
new file mode 100644 (file)
index 0000000..0824e01
--- /dev/null
@@ -0,0 +1,749 @@
+/*
+ * $Header$
+ *
+ * Provides CLI interface to interact with Pgpool-II watchdog
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ */
+#include <stdio.h>
+#include <errno.h>
+#include <ctype.h>
+#include <time.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <netdb.h>
+#include <sys/wait.h>
+
+#include "pool.h"
+#include "pool_config.h"
+#include "version.h"
+#include "parser/stringinfo.h"
+#include "utils/json.h"
+#include "utils/json_writer.h"
+#include "utils/fe_ports.h"
+
+#include "watchdog/wd_ipc_conn.h"
+#include "watchdog/wd_lifecheck.h"
+#include "watchdog/wd_commands.h"
+
+#ifdef HAVE_GETOPT_H
+#include <getopt.h>
+#else
+#include "utils/getopt_long.h"
+#endif
+
+
+#define LIFECHECK_GETNODE_WAIT_SEC_COUNT 5     /* max number of seconds the
+                                                                                        * lifecheck process should waits
+                                                                                        * before giving up while fetching
+                                                                                        * the configured watchdog node
+                                                                                        * infromation from watchdog
+                                                                                        * process through IPC channel */
+
+const char *progname = NULL;
+LifeCheckCluster *gslifeCheckCluster = NULL;
+
+
+static void usage(void);
+static bool validate_number(char* ptr);
+const char *get_progname(const char *argv0);
+static void print_lifecheck_cluster(bool include_nodes, bool verbose);
+static void print_node_info(LifeCheckNode* lifeCheckNode, bool verbose);
+
+static bool fetch_watchdog_nodes_data(char *authkey, bool debug);
+static bool inform_node_is_alive(LifeCheckNode * node, char *message, char* authkey);
+static bool inform_node_is_dead(LifeCheckNode * node, char *message, char* authkey);
+
+static void load_watchdog_nodes_from_json(char *json_data, int len);
+static char *get_node_status_change_json(int nodeID, int nodeStatus, char *message, char *authKey);
+
+static void print_node_info(LifeCheckNode* lifeCheckNode, bool verbose);
+static LifeCheckNode* get_node_by_options(char *node_name, char* node_host, int node_port, int node_id);
+
+int
+main(int argc, char **argv)
+{
+       LifeCheckNode* lifeCheckNode;
+       char*           conf_file_path = NULL;
+       char*           node_host = NULL;
+       char*           node_name = NULL;
+       char*           wd_authkey = NULL;
+       char*           socket_dir = NULL;
+       char*           message = NULL;
+       int                     node_wd_port = -1;
+       int                     node_id = -1;
+       int                     port = -1;
+       int                     ch;
+       int                     optindex;
+       bool            debug = false;
+       bool            info_req = false;
+       bool            inform_status = false;
+       bool            verbose = false;
+       bool            all_nodes = false;
+       bool            status_ALIVE = false;
+       bool            status_DEAD = false;
+       
+       /* here we put all the allowed long options for all utilities */
+       static struct option long_options[] = {
+               {"help", no_argument, NULL, '?'},
+               {"all", no_argument, NULL, 'a'},
+               {"debug", no_argument, NULL, 'd'},
+               {"config-file", required_argument, NULL, 'f'},
+               {"node-host", required_argument, NULL, 'H'},
+               {"info", no_argument, NULL, 'i'},
+               {"inform", required_argument, NULL, 'I'},
+               {"auth-key", required_argument, NULL, 'k'},
+               {"message", required_argument, NULL, 'm'},
+               {"node-id", required_argument, NULL, 'n'},
+               {"node-name", required_argument, NULL, 'N'},
+               {"node-port", required_argument, NULL, 'P'},
+               {"ipc-port", required_argument, NULL, 'p'},
+               {"socket-dir", required_argument, NULL, 's'},
+               {"version", no_argument, NULL, 'V'},
+               {"verbose", no_argument, NULL, 'v'},
+               {NULL, 0, NULL, 0}
+       };
+
+       /* Identify the utility app */
+       progname = get_progname(argv[0]);
+
+       if (argc > 1)
+       {
+               if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
+               {
+                       usage();
+                       exit(0);
+               }
+               else if (strcmp(argv[1], "-V") == 0
+                                || strcmp(argv[1], "--version") == 0)
+               {
+                       fprintf(stderr, "%s (%s) %s\n", progname, PACKAGE, VERSION);
+                       exit(0);
+               }
+       }
+
+       while ((ch = getopt_long(argc, argv, "?aAdDf:H:iI:k:m:n:N:p:P:s:Vv", long_options, &optindex)) != -1)
+       {
+               switch (ch)
+               {
+                       case 'd':
+                               debug = true;
+                               break;
+
+                       case 'a':
+                               all_nodes = true;
+                               break;
+
+                       case 'i': /* Info Request */
+                       {
+                               info_req = true;
+                               if (inform_status)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid option, 'info' and 'inform' are mutually exclusive options\n");
+                                       exit(EXIT_FAILURE);
+                               }
+                       }
+                               break;
+
+                       case 'I':
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               inform_status = true;
+                               if (info_req)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid option, 'info' and 'inform' are mutually exclusive options\n");
+                                       exit(EXIT_FAILURE);
+                               }
+                               if (strcasecmp("DEAD",optarg) == 0)
+                                       status_DEAD = true;
+                               else if (strcasecmp("ALIVE",optarg) == 0)
+                                       status_ALIVE = true;
+                               else
+                               {
+                                       fprintf(stderr, "ERROR: Invalid node status \"%s\", Allowd options are DEAD or ALIVE''\n",optarg);
+                                       exit(EXIT_FAILURE);
+                               }
+                               break;
+
+                       case 'n':
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               if (validate_number(optarg) == false)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid value %s, node-id can only contain numeric values\n",optarg);
+                                       exit(EXIT_FAILURE);
+                               }
+                               node_id = atoi(optarg);
+                               break;
+
+                       case 'H':
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               node_host = pstrdup(optarg);
+                               break;
+
+                       case 'N':
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               node_name = pstrdup(optarg);
+                               break;
+
+                       case 'P':
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               if (validate_number(optarg) == false)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid value %s, node-port can only contain numeric values\n",optarg);
+                                       exit(EXIT_FAILURE);
+                               }
+                               node_wd_port = atoi(optarg);
+                               break;
+
+                       case 's':                       /* socket dir */
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               socket_dir = pstrdup(socket_dir);
+                               break;
+
+                       case 'm':                       /* message */
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               message = pstrdup(optarg);
+                               break;
+
+                       case 'f':                       /* specify configuration file */
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               conf_file_path = pstrdup(optarg);
+                               break;
+
+                       case 'k':                       /* specify authkey */
+                               if (!optarg)
+                               {
+                                       usage();
+                                       exit(EXIT_FAILURE);
+                               }
+                               wd_authkey = pstrdup(optarg);
+                               break;
+
+                       case 'v':
+                               verbose = true;
+                               break;
+
+                       case 'V':
+                               fprintf(stderr, "%s for %s version %s (%s),\n", progname, PACKAGE, VERSION, PGPOOLVERSION);
+                               exit(0);
+                               break;
+
+                       case 'p':
+                               if (validate_number(optarg) == false)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid value %s, port can only contain numeric values\n",optarg);
+                                       exit(EXIT_FAILURE);
+                               }
+                               port = atoi(optarg);
+                               if (port <= 1024 || port > 65535)
+                               {
+                                       fprintf(stderr, "ERROR: Invalid port number \"%s\", must be between 1024 and 65535\n", optarg);
+                                       exit(0);
+                               }
+                               break;
+
+                       case '?':
+                       default:
+                               
+                               /*
+                                * getopt_long whould already have emitted a complaint
+                                */
+                               fprintf(stderr, "Try \"%s --help\" for more information.\n\n", progname);
+                               exit(EXIT_FAILURE);
+               }
+       }
+
+       if (!info_req && !inform_status)
+       {
+               fprintf(stderr, "ERROR: Missing operation. Try %s --help\" for more information.\n\n", progname);
+               exit(EXIT_FAILURE);
+       }
+
+       if (inform_status && all_nodes)
+       {
+               if (all_nodes)
+               {
+                       fprintf(stderr, "ERROR: Invalid option \"-a --all\" for inform status operation. Try %s --help\" for more information.\n\n", progname);
+                       exit(EXIT_FAILURE);
+               }
+               if (node_name == NULL && node_host == NULL && node_wd_port < 0 && node_id < 0)
+               {
+                       fprintf(stderr, "ERROR: Missing node search criteria. Try %s --help\" for more information.\n\n", progname);
+                       exit(EXIT_FAILURE);
+               }
+       }
+
+       if (conf_file_path)
+       {
+               if (pool_init_config())
+               {
+                       fprintf(stderr, "pool_init_config() failed\n\n");
+                       exit(EXIT_FAILURE);
+               }
+               if (pool_get_config(conf_file_path, CFGCXT_INIT) == false)
+               {
+                       fprintf(stderr, "ERROR: Unable to get configuration. Exiting...\n\n");
+                       exit(EXIT_FAILURE);
+               }
+
+               if (debug)
+                       printf("DEBUG: From config %s:%d\n",pool_config->wd_ipc_socket_dir, pool_config->wd_port);
+
+               pfree(conf_file_path);
+               /* only use values from pg_config that are not provided explicitely*/
+               if (wd_authkey == NULL)
+                       wd_authkey = pstrdup(pool_config->wd_authkey);
+               if (port < 0)
+                       port = pool_config->wd_port;
+               if (socket_dir == NULL)
+                       socket_dir = pstrdup(pool_config->wd_ipc_socket_dir);
+       }
+
+       if (port < 0)
+               port = 9898;
+
+       if (socket_dir == NULL)
+               socket_dir = pstrdup("/tmp");
+
+       if(debug)
+       {
+               fprintf(stderr, "DEBUG: setting IPC address to %s:%d\n",socket_dir,port);
+       }
+
+       wd_set_ipc_address(socket_dir,port);
+       wd_ipc_conn_initialize();
+
+       if(debug)
+       {
+               char c_node_id[10],c_wd_port[10];
+               snprintf(c_node_id, sizeof(c_node_id), "%d",node_id);
+               snprintf(c_wd_port, sizeof(c_wd_port), "%d",node_wd_port);
+
+               fprintf(stderr, "DEBUG: OPERATION:%s ALL NODE CRITERIA = %s\n",
+                               info_req?"\"INFO REQUEST\"":"\"INFORM NODE STATUS\"",
+                               all_nodes?"TRUE":"FALSE"
+                               );
+               fprintf(stderr, "DEBUG: Search criteria:[ID=%s AND Name=%s AND Host=%s AND WDPort=%s]\n",
+                               (node_id < 0)?"ANY":c_node_id,
+                               node_name?node_name:"ANY",
+                               node_host?node_host:"ANY",
+                               (node_wd_port < 0)?"ANY":c_wd_port);
+       }
+
+       fetch_watchdog_nodes_data(wd_authkey, debug);
+
+       if (info_req)
+       {
+               if (all_nodes)
+               {
+                       print_lifecheck_cluster(true, verbose);
+                       exit(0);
+               }
+               if (node_name == NULL && node_host == NULL && node_wd_port < 0 && node_id < 0)
+               {
+                       fprintf(stderr, "WARNING: Missing node search criteria. applying --all option\n");
+                       print_lifecheck_cluster(true, verbose);
+                       exit(0);
+               }
+       }
+
+       lifeCheckNode = get_node_by_options(node_name, node_host, node_wd_port, node_id);
+       if (!lifeCheckNode)
+       {
+               char c_node_id[10],c_wd_port[10];
+               fprintf(stderr, "ERROR: unable to find the node with the requested criteria\n");
+               snprintf(c_node_id, sizeof(c_node_id), "%d",node_id);
+               snprintf(c_wd_port, sizeof(c_wd_port), "%d",node_wd_port);
+               fprintf(stderr, "Criteria:[ID=%s AND Name=%s AND Host=%s AND WDPort=%s]\n",
+                               (node_id < 0)?"ANY":c_node_id,
+                               node_name?node_name:"ANY",
+                               node_host?node_host:"ANY",
+                               (node_wd_port < 0)?"ANY":c_wd_port);
+               exit(EXIT_FAILURE);
+
+       }
+       if (info_req)
+       {
+               print_lifecheck_cluster(false, verbose);
+               print_node_info(lifeCheckNode, verbose);
+               exit (0);
+       }
+
+       if (status_DEAD)
+       {
+               if (inform_node_is_dead(lifeCheckNode, message, wd_authkey))
+               {
+                       fprintf(stderr,"INFO: informed watchdog about node id %d is dead\n",node_id);
+                       exit(0);
+               }
+               fprintf(stderr,"ERROR: failed to inform watchdog about node id %d is dead\n",node_id);
+               exit(EXIT_FAILURE);
+       }
+       else if (status_ALIVE)
+       {
+               if (inform_node_is_alive(lifeCheckNode, message, wd_authkey))
+               {
+                       fprintf(stderr,"INFO: informed watchdog about node id %d is alive\n",node_id);
+                       exit(0);
+               }
+               fprintf(stderr,"ERROR: failed to inform watchdog about node id %d is alive\n",node_id);
+               exit(EXIT_FAILURE);
+       }
+
+       return 0;
+}
+
+const char *
+get_progname(const char *argv0)
+{
+       return "wd_cli";
+}
+
+
+static bool
+validate_number(char* ptr)
+{
+       while (*ptr)
+       {
+               if (isdigit(*ptr) == 0)
+                       return false;
+               ptr++;
+       }
+       return true;
+}
+
+static bool
+inform_node_status(LifeCheckNode * node, char *message, char* authkey)
+{
+       int                     node_status,
+                               x;
+       char       *json_data;
+       WDIPCCmdResult *res = NULL;
+       char       *new_status;
+
+       if (node->nodeState == NODE_DEAD)
+       {
+               new_status = "NODE DEAD";
+               node_status = WD_LIFECHECK_NODE_STATUS_DEAD;
+       }
+       else if (node->nodeState == NODE_ALIVE)
+       {
+               new_status = "NODE ALIVE";
+               node_status = WD_LIFECHECK_NODE_STATUS_ALIVE;
+       }
+       else
+               return false;
+
+       fprintf(stderr,"INFO: informing the node status change to watchdog");
+       fprintf(stderr,"INFO: node id :%d status = \"%s\" message:\"%s\"", node->ID, new_status, message);
+
+       json_data = get_node_status_change_json(node->ID, node_status, message, authkey);
+       if (json_data == NULL)
+               return false;
+
+       for (x = 0; x < MAX_SEC_WAIT_FOR_CLUSTER_TRANSATION; x++)
+       {
+               res = issue_command_to_watchdog(WD_NODE_STATUS_CHANGE_COMMAND, 0, json_data, strlen(json_data), false);
+               if (res)
+                       break;
+               sleep(1);
+       }
+       pfree(json_data);
+       if (res)
+       {
+               pfree(res);
+               return true;
+       }
+       return false;
+}
+
+static bool
+fetch_watchdog_nodes_data(char *authkey, bool debug)
+{
+       char       *json_data = wd_get_watchdog_nodes_json(authkey, -1);
+
+       if (json_data == NULL)
+       {
+               ereport(ERROR,
+                               (errmsg("get node list command reply contains no data")));
+               return false;
+       }
+
+       if(debug)
+               printf("DEBUG:************\n%s\n************\n",json_data);
+
+       load_watchdog_nodes_from_json(json_data, strlen(json_data));
+       pfree(json_data);
+       return true;
+}
+
+static void
+load_watchdog_nodes_from_json(char *json_data, int len)
+{
+       json_value *root;
+       json_value *value;
+       int                     i,
+                               nodeCount;
+
+       root = json_parse(json_data, len);
+
+       /* The root node must be object */
+       if (root == NULL || root->type != json_object)
+       {
+               json_value_free(root);
+               ereport(ERROR,
+                               (errmsg("unable to parse json data for node list")));
+       }
+
+       if (json_get_int_value_for_key(root, "NodeCount", &nodeCount))
+       {
+               json_value_free(root);
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find NodeCount node from data")));
+       }
+
+       /* find the WatchdogNodes array */
+       value = json_get_value_for_key(root, "WatchdogNodes");
+       if (value == NULL)
+       {
+               json_value_free(root);
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find WatchdogNodes node from data")));
+       }
+       if (value->type != json_array)
+       {
+               json_value_free(root);
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("WatchdogNodes node does not contains Array")));
+       }
+       if (nodeCount != value->u.array.length)
+       {
+               json_value_free(root);
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("WatchdogNodes array contains %d nodes while expecting %d", value->u.array.length, nodeCount)));
+       }
+
+       /* okay we are done, put this in shared memory */
+       gslifeCheckCluster = malloc(sizeof(LifeCheckCluster));
+       gslifeCheckCluster->nodeCount = nodeCount;
+       gslifeCheckCluster->lifeCheckNodes = malloc(sizeof(LifeCheckNode) * gslifeCheckCluster->nodeCount);
+       for (i = 0; i < nodeCount; i++)
+       {
+               WDNodeInfo *nodeInfo = parse_watchdog_node_info_from_wd_node_json(value->u.array.values[i]);
+
+               gslifeCheckCluster->lifeCheckNodes[i].nodeState = NODE_EMPTY;
+               gslifeCheckCluster->lifeCheckNodes[i].wdState = nodeInfo->state;
+               strcpy(gslifeCheckCluster->lifeCheckNodes[i].stateName, nodeInfo->stateName);
+               gslifeCheckCluster->lifeCheckNodes[i].ID = nodeInfo->id;
+               strcpy(gslifeCheckCluster->lifeCheckNodes[i].hostName, nodeInfo->hostName);
+               strcpy(gslifeCheckCluster->lifeCheckNodes[i].nodeName, nodeInfo->nodeName);
+               gslifeCheckCluster->lifeCheckNodes[i].wdPort = nodeInfo->wd_port;
+               gslifeCheckCluster->lifeCheckNodes[i].pgpoolPort = nodeInfo->pgpool_port;
+               gslifeCheckCluster->lifeCheckNodes[i].retry_lives = pool_config->wd_life_point;
+               pfree(nodeInfo);
+       }
+       json_value_free(root);
+}
+
+
+static bool
+inform_node_is_dead(LifeCheckNode * node, char *message, char* authkey)
+{
+       node->nodeState = NODE_DEAD;
+       return inform_node_status(node, message, authkey);
+}
+
+static bool
+inform_node_is_alive(LifeCheckNode * node, char *message, char* authkey)
+{
+       node->nodeState = NODE_ALIVE;
+       return inform_node_status(node, message, authkey);
+}
+
+static LifeCheckNode*
+get_node_by_options(char *node_name, char* node_host, int node_port, int node_id)
+{
+       int i;
+       if (!gslifeCheckCluster)
+               return NULL;
+       for (i = 0; i < gslifeCheckCluster->nodeCount; i++)
+       {
+               if (node_id >= 0 && node_id != gslifeCheckCluster->lifeCheckNodes[i].ID)
+                       continue;
+               if (node_port >= 0 && node_port != gslifeCheckCluster->lifeCheckNodes[i].wdPort)
+                       continue;
+               if (node_name && strcasecmp(gslifeCheckCluster->lifeCheckNodes[i].nodeName, node_name) != 0 )
+                       continue;
+               if (node_host && strcasecmp(gslifeCheckCluster->lifeCheckNodes[i].hostName, node_host) != 0 )
+                       continue;
+
+               return &gslifeCheckCluster->lifeCheckNodes[i];
+       }
+       return NULL;
+}
+
+
+static void
+print_lifecheck_cluster(bool include_nodes, bool verbose)
+{
+       int                     i;
+       if (!gslifeCheckCluster)
+       {
+               fprintf(stdout,"ERROR: node information not found\n");
+               return;
+       }
+       fprintf(stdout,"Total Watchdog nodes configured for lifecheck:    %d\n", gslifeCheckCluster->nodeCount);
+       if (verbose)
+               fprintf(stdout,"*****************\n");
+       if(!include_nodes)
+               return;
+
+       for (i = 0; i < gslifeCheckCluster->nodeCount; i++)
+               print_node_info(&gslifeCheckCluster->lifeCheckNodes[i], verbose);
+}
+
+
+
+static void
+print_node_info(LifeCheckNode* lifeCheckNode, bool verbose)
+{
+       if (verbose)
+       {
+               fprintf(stdout,"Node ID:           %d\n",lifeCheckNode->ID);
+               fprintf(stdout,"Node Status code   %d\n",lifeCheckNode->wdState);
+               fprintf(stdout,"Node Status:       %s\n",lifeCheckNode->stateName);
+               fprintf(stdout,"Node Name:         %s\n",lifeCheckNode->nodeName);
+               fprintf(stdout,"Node Host:         %s\n",lifeCheckNode->hostName);
+               fprintf(stdout,"Node WD Port:      %d\n",lifeCheckNode->wdPort);
+               fprintf(stdout,"Node Pgpool Port:  %d\n\n",lifeCheckNode->pgpoolPort);
+       }
+       else
+       {
+               fprintf(stdout,"%d %d \"%s\"", lifeCheckNode->ID,
+                               lifeCheckNode->nodeState,
+                               lifeCheckNode->stateName),
+               fprintf(stdout,"\"%s\"",lifeCheckNode->nodeName),
+               fprintf(stdout,"\"%s\" %d %d\n",
+                               lifeCheckNode->hostName,
+                               lifeCheckNode->wdPort,
+                               lifeCheckNode->pgpoolPort);
+       }
+}
+
+static char *
+get_node_status_change_json(int nodeID, int nodeStatus, char *message, char *authKey)
+{
+       char       *json_str;
+       JsonNode   *jNode = jw_create_with_object(true);
+       
+       if (authKey != NULL && strlen(authKey) > 0)
+               jw_put_string(jNode, WD_IPC_AUTH_KEY, authKey); /* put the auth key */
+       
+       /* add the node ID */
+       jw_put_int(jNode, "NodeID", nodeID);
+       /* add the node status */
+       jw_put_int(jNode, "NodeStatus", nodeStatus);
+       /* add the node message if any */
+       if (message)
+               jw_put_string(jNode, "Message", message);
+       
+       jw_finish_document(jNode);
+       json_str = pstrdup(jw_get_json_string(jNode));
+       jw_destroy(jNode);
+       return json_str;
+}
+
+static void
+usage(void)
+{
+       
+       fprintf(stderr, "\nWatchdog CLI for ");
+       fprintf(stderr, "%s version %s (%s)\n", PACKAGE, VERSION, PGPOOLVERSION);
+
+       fprintf(stderr, "\nUsage:\n");
+       fprintf(stderr, "  %s [ operation] [ options] [node srarch criteria]\n",progname);
+
+       fprintf(stderr, "\n Operations:\n");
+       fprintf(stderr, "  -i, --info                  Get the node status for nodes based on node search criteria\n");
+       fprintf(stderr, "  -I, --inform=NEW-STATUS     Inform Pgpool-II about new watchdog node status\n");
+       fprintf(stderr, "                              Allowed values are DEAD and ALIVE\n");
+
+       fprintf(stderr, "\n Node search options:\n");
+
+       fprintf(stderr, "  -a, --all                Search all nodes (only available with --info option)\n");
+       fprintf(stderr, "  -n, --node-id=ID         Search watchdog node with node_id\n");
+       fprintf(stderr, "  -N, --node-name=Name     Search watchdog node with name\n");
+       fprintf(stderr, "  -H, --node-name=Host     Search watchdog node with Host\n");
+       fprintf(stderr, "  -P, --node-port=port     Search watchdog node with wd_port\n");
+
+       fprintf(stderr, "\n Options:\n");
+
+       fprintf(stderr, "  -f, --config-file=CONFIG_FILE\n");
+       fprintf(stderr, "                      Set the path to the pgpool.conf configuration file\n");
+       fprintf(stderr, "  -k, --auth-key=KEY\n");
+       fprintf(stderr, "                      watchdog auth key\n");
+       fprintf(stderr, "                      This over rides the pgpool.conf->wd_authkey value\n");
+       fprintf(stderr, "  -s, --socket-dir=WD_IPC_SOCKET_DIRECTORY\n");
+       fprintf(stderr, "                      Path to the WD IPC socket directory\n");
+       fprintf(stderr, "                      This over rides the pgpool.conf->wd_ipc_socket_dir value\n");
+       fprintf(stderr, "  -p, --ipc-port=WD_IPC_PORT\n");
+       fprintf(stderr, "                      Port number of watchdog IPC socket\n");
+       fprintf(stderr, "                      This over rides the pgpool.conf->wd_port value\n");
+       fprintf(stderr, "  -m, --message=message string\n");
+       fprintf(stderr, "                      Message to be passed to Pgpool-II along with new node status\n");
+       fprintf(stderr, "  -v, --verbose       Output verbose messages\n");
+       fprintf(stderr, "  -V, --version       Output Version information\n");
+       fprintf(stderr, "  -d, --debug         Enable debug output\n");
+       fprintf(stderr, "  -h, --help          Print this help\n\n");
+}
index f4134d192accb4073502106b76106b858db7223d..62bf1c3e5f80ed401483857c41c1ef34d025bade 100644 (file)
 #include <errno.h>
 #include <unistd.h>
 
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
 
 #include "pool.h"
 #include "utils/elog.h"
 #include "utils/palloc.h"
 #include "utils/memutils.h"
+#include "utils/socket_stream.h"
 #include "utils/pool_stream.h"
 #include "pool_config.h"
 
@@ -1280,55 +1278,6 @@ pool_stacklen(POOL_CONNECTION * cp)
        return cp->bufsz3;
 }
 
-/*
- * set non-block flag
- */
-void
-pool_set_nonblock(int fd)
-{
-       int                     var;
-
-       /* set fd to none blocking */
-       var = fcntl(fd, F_GETFL, 0);
-       if (var == -1)
-       {
-               ereport(FATAL,
-                               (errmsg("unable to set options on socket"),
-                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
-
-       }
-       if (fcntl(fd, F_SETFL, var | O_NONBLOCK) == -1)
-       {
-               ereport(FATAL,
-                               (errmsg("unable to set options on socket"),
-                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
-       }
-}
-
-/*
- * unset non-block flag
- */
-void
-pool_unset_nonblock(int fd)
-{
-       int                     var;
-
-       /* set fd to none blocking */
-       var = fcntl(fd, F_GETFL, 0);
-       if (var == -1)
-       {
-               ereport(FATAL,
-                               (errmsg("unable to set options on socket"),
-                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
-       }
-       if (fcntl(fd, F_SETFL, var & ~O_NONBLOCK) == -1)
-       {
-               ereport(FATAL,
-                               (errmsg("unable to set options on socket"),
-                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
-       }
-}
-
 #ifdef DEBUG
 /*
  * Debug aid
@@ -1343,91 +1292,6 @@ dump_buffer(char *buf, int len)
        }
 }
 #endif
-int
-socket_write(int fd, void *buf, size_t len)
-{
-       int                     bytes_send = 0;
-
-       do
-       {
-               int                     ret;
-
-               ret = write(fd, buf + bytes_send, (len - bytes_send));
-               if (ret <= 0)
-               {
-                       if (errno == EINTR || errno == EAGAIN)
-                       {
-                               ereport(DEBUG5,
-                                               (errmsg("write on socket failed with error :\"%s\"", strerror(errno)),
-                                                errdetail("retrying...")));
-                               continue;
-                       }
-                       ereport(LOG,
-                                       (errmsg("write on socket failed with error :\"%s\"", strerror(errno))));
-                       return -1;
-               }
-               bytes_send += ret;
-       } while (bytes_send < len);
-       return bytes_send;
-}
-
-int
-socket_read(int fd, void *buf, size_t len, int timeout)
-{
-       int                     ret,
-                               read_len;
-
-       read_len = 0;
-       struct timeval timeoutval;
-       fd_set          readmask;
-       int                     fds;
-
-       while (read_len < len)
-       {
-               FD_ZERO(&readmask);
-               FD_SET(fd, &readmask);
-
-               timeoutval.tv_sec = timeout;
-               timeoutval.tv_usec = 0;
-
-               fds = select(fd + 1, &readmask, NULL, NULL, timeout ? &timeoutval : NULL);
-               if (fds == -1)
-               {
-                       if (errno == EAGAIN || errno == EINTR)
-                               continue;
-
-                       ereport(WARNING,
-                                       (errmsg("select failed with error: \"%s\"", strerror(errno))));
-                       return -1;
-               }
-               else if (fds == 0)
-               {
-                       return -2;
-               }
-               ret = read(fd, buf + read_len, (len - read_len));
-               if (ret < 0)
-               {
-                       if (errno == EINTR || errno == EAGAIN)
-                       {
-                               ereport(DEBUG5,
-                                               (errmsg("read from socket failed with error :\"%s\"", strerror(errno)),
-                                                errdetail("retrying...")));
-                               continue;
-                       }
-                       ereport(LOG,
-                                       (errmsg("read from socket failed with error :\"%s\"", strerror(errno))));
-                       return -1;
-               }
-               if (ret == 0)
-               {
-                       ereport(LOG,
-                                       (errmsg("read from socket failed, remote end closed the connection")));
-                       return 0;
-               }
-               read_len += ret;
-       }
-       return read_len;
-}
 
 /*
  * Set timeout in seconds for pool_check_fd
index 430b044f5b53840196acbe63f79076ccde287378..7dba828e4a0a49baf6c8141904490cb2f3792e4d 100644 (file)
  *-------------------------------------------------------------------------
  */
 
-#ifndef POOL_PRIVATE
 
 #include <errno.h>
 #include <stdarg.h>
 #include "pool.h"
+
+#ifndef POOL_PRIVATE
+
 #include "utils/elog.h"
 #include "utils/palloc.h"
 #include "utils/memutils.h"
diff --git a/src/utils/socket_stream.c b/src/utils/socket_stream.c
new file mode 100644 (file)
index 0000000..199e84a
--- /dev/null
@@ -0,0 +1,195 @@
+/* -*-pgsql-c-*- */
+/*
+* pgpool: a language independent connection pool server for PostgreSQL
+* written by Tatsuo Ishii
+*
+* Copyright (c) 2003-2019      PgPool Global Development Group
+*
+* Permission to use, copy, modify, and distribute this software and
+* its documentation for any purpose and without fee is hereby
+* granted, provided that the above copyright notice appear in all
+* copies and that both that copyright notice and this permission
+* notice appear in supporting documentation, and that the name of the
+* author not be used in advertising or publicity pertaining to
+* distribution of the software without specific, written prior
+* permission. The author makes no representations about the
+* suitability of this software for any purpose.  It is provided "as
+* is" without express or implied warranty.
+*
+* pool_stream.c: stream I/O modules
+*
+*/
+
+#include "config.h"
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include "pool.h"
+#include "utils/socket_stream.h"
+#ifndef POOL_PRIVATE
+#include "utils/elog.h"
+#else
+#include "utils/fe_ports.h"
+#endif
+
+
+/*
+ * set non-block flag
+ */
+void
+socket_set_nonblock(int fd)
+{
+       int                     var;
+
+       /* set fd to none blocking */
+       var = fcntl(fd, F_GETFL, 0);
+       if (var == -1)
+       {
+               ereport(FATAL,
+                               (errmsg("unable to set options on socket"),
+                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
+
+       }
+       if (fcntl(fd, F_SETFL, var | O_NONBLOCK) == -1)
+       {
+               ereport(FATAL,
+                               (errmsg("unable to set options on socket"),
+                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
+       }
+}
+
+/*
+ * unset non-block flag
+ */
+void
+socket_unset_nonblock(int fd)
+{
+       int                     var;
+
+       /* set fd to none blocking */
+       var = fcntl(fd, F_GETFL, 0);
+       if (var == -1)
+       {
+               ereport(FATAL,
+                               (errmsg("unable to set options on socket"),
+                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
+       }
+       if (fcntl(fd, F_SETFL, var & ~O_NONBLOCK) == -1)
+       {
+               ereport(FATAL,
+                               (errmsg("unable to set options on socket"),
+                                errdetail("fcntl system call failed with error \"%s\"", strerror(errno))));
+       }
+}
+
+#ifdef DEBUG
+/*
+ * Debug aid
+ */
+static void
+dump_buffer(char *buf, int len)
+{
+       while (--len)
+       {
+               ereport(DEBUG5,
+                               (errmsg("%02x", *buf++)));
+       }
+}
+#endif
+int
+socket_write(int fd, void *buf, size_t len)
+{
+       int                     bytes_send = 0;
+
+       do
+       {
+               int                     ret;
+
+               ret = write(fd, buf + bytes_send, (len - bytes_send));
+               if (ret <= 0)
+               {
+                       if (errno == EINTR || errno == EAGAIN)
+                       {
+                               ereport(DEBUG5,
+                                               (errmsg("write on socket failed with error :\"%s\"", strerror(errno)),
+                                                errdetail("retrying...")));
+                               continue;
+                       }
+                       ereport(LOG,
+                                       (errmsg("write on socket failed with error :\"%s\"", strerror(errno))));
+                       return -1;
+               }
+               bytes_send += ret;
+       } while (bytes_send < len);
+       return bytes_send;
+}
+
+int
+socket_read(int fd, void *buf, size_t len, int timeout)
+{
+       int                     ret,
+                               read_len;
+
+       read_len = 0;
+       struct timeval timeoutval;
+       fd_set          readmask;
+       int                     fds;
+
+       while (read_len < len)
+       {
+               FD_ZERO(&readmask);
+               FD_SET(fd, &readmask);
+
+               timeoutval.tv_sec = timeout;
+               timeoutval.tv_usec = 0;
+
+               fds = select(fd + 1, &readmask, NULL, NULL, timeout ? &timeoutval : NULL);
+               if (fds == -1)
+               {
+                       if (errno == EAGAIN || errno == EINTR)
+                               continue;
+
+                       ereport(WARNING,
+                                       (errmsg("select failed with error: \"%s\"", strerror(errno))));
+                       return -1;
+               }
+               else if (fds == 0)
+               {
+                       return -2;
+               }
+               ret = read(fd, buf + read_len, (len - read_len));
+               if (ret < 0)
+               {
+                       if (errno == EINTR || errno == EAGAIN)
+                       {
+                               ereport(DEBUG5,
+                                               (errmsg("read from socket failed with error :\"%s\"", strerror(errno)),
+                                                errdetail("retrying...")));
+                               continue;
+                       }
+                       ereport(LOG,
+                                       (errmsg("read from socket failed with error :\"%s\"", strerror(errno))));
+                       return -1;
+               }
+               if (ret == 0)
+               {
+                       ereport(LOG,
+                                       (errmsg("read from socket failed, remote end closed the connection")));
+                       return 0;
+               }
+               read_len += ret;
+       }
+       return read_len;
+}
index 7c9921ed19a0de93d5a1a11e4f2725a9b30192d0..bb4c2204c1ae0ad33dbb8ac2a7b21b896a344c3a 100644 (file)
@@ -8,6 +8,8 @@ lib_watchdog_a_SOURCES = \
        wd_if.c \
        wd_lifecheck.c \
        wd_commands.c \
+       wd_internal_commands.c \
+       wd_ipc_conn.c \
        wd_json_data.c \
        wd_ping.c \
        wd_heartbeat.c \
index a526a709be246504e867eab1bf1e90b003e4fd8b..891d6058b0014bc68bb668269b02bbc472072ded 100644 (file)
@@ -105,6 +105,7 @@ lib_watchdog_a_AR = $(AR) $(ARFLAGS)
 lib_watchdog_a_LIBADD =
 am_lib_watchdog_a_OBJECTS = watchdog.$(OBJEXT) wd_if.$(OBJEXT) \
        wd_lifecheck.$(OBJEXT) wd_commands.$(OBJEXT) \
+       wd_internal_commands.$(OBJEXT) wd_ipc_conn.$(OBJEXT) \
        wd_json_data.$(OBJEXT) wd_ping.$(OBJEXT) \
        wd_heartbeat.$(OBJEXT) wd_utils.$(OBJEXT) \
        wd_escalation.$(OBJEXT)
@@ -317,6 +318,8 @@ lib_watchdog_a_SOURCES = \
        wd_if.c \
        wd_lifecheck.c \
        wd_commands.c \
+       wd_internal_commands.c \
+       wd_ipc_conn.c \
        wd_json_data.c \
        wd_ping.c \
        wd_heartbeat.c \
index f0b0f0d9e919d3249d5904774fec73320578a9a5..c2fdb7c7144bb9bef5554adcd87eb6abc46fa9f7 100644 (file)
@@ -46,7 +46,7 @@
 #include "utils/elog.h"
 #include "utils/json_writer.h"
 #include "utils/json.h"
-#include "utils/pool_stream.h"
+#include "utils/socket_stream.h"
 #include "pool_config.h"
 
 #include <net/if.h>
@@ -55,7 +55,7 @@
 #include "watchdog/watchdog.h"
 #include "watchdog/wd_json_data.h"
 #include "watchdog/wd_ipc_defines.h"
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_internal_commands.h"
 #include "parser/stringinfo.h"
 
 /* These defines enables the consensus building feature
@@ -780,7 +780,7 @@ wd_create_recv_socket(int port)
                                 errdetail("create socket failed with reason: \"%s\"", strerror(errno))));
        }
 
-       pool_set_nonblock(sock);
+       socket_set_nonblock(sock);
 
        if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof(one)) == -1)
        {
@@ -901,7 +901,7 @@ wd_create_client_socket(char *hostname, int port, bool *connected)
        len = sizeof(struct sockaddr_in);
 
        /* set socket to non blocking */
-       pool_set_nonblock(sock);
+       socket_set_nonblock(sock);
 
        if (connect(sock, (struct sockaddr *) &addr, len) < 0)
        {
@@ -911,7 +911,7 @@ wd_create_client_socket(char *hostname, int port, bool *connected)
                }
                if (errno == EISCONN)
                {
-                       pool_unset_nonblock(sock);
+                       socket_unset_nonblock(sock);
                        *connected = true;
                        return sock;
                }
@@ -922,7 +922,7 @@ wd_create_client_socket(char *hostname, int port, bool *connected)
                return -1;
        }
        /* set socket to blocking again */
-       pool_unset_nonblock(sock);
+       socket_unset_nonblock(sock);
        *connected = true;
        return sock;
 }
@@ -3230,7 +3230,7 @@ update_successful_outgoing_cons(fd_set *wmask, int pending_fds_count)
                                                ereport(LOG,
                                                                (errmsg("new outbound connection to %s:%d ", wdNode->hostname, wdNode->wd_port)));
                                                /* set socket to blocking again */
-                                               pool_unset_nonblock(wdNode->client_socket.sock);
+                                               socket_unset_nonblock(wdNode->client_socket.sock);
                                                watchdog_state_machine(WD_EVENT_NEW_OUTBOUND_CONNECTION, wdNode, NULL, NULL);
                                        }
                                }
index 82d91809b5485745cf4cbecd596934ce9465004f..43efab4de3f3622b98c446f142ed3caefbceb5ab 100644 (file)
@@ -1,12 +1,12 @@
 /*
  * $Header$
  *
- * Handles watchdog connection, and protocol communication with pgpool-II
+ * Handles watchdog ipc commands that are also allowed from outside the pgpool-II
  *
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2016     PgPool Global Development Group
+ * Copyright (c) 2003-2019     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
 #include <sys/wait.h>
 
 #include "pool.h"
+#ifndef POOL_PRIVATE
 #include "utils/elog.h"
+#else
+#include "utils/fe_ports.h"
+#endif
+
 #include "utils/json_writer.h"
 #include "utils/json.h"
-#include "utils/pool_stream.h"
-#include "pool_config.h"
-#include "watchdog/wd_json_data.h"
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_commands.h"
 #include "watchdog/wd_ipc_defines.h"
 
 #define WD_DEFAULT_IPC_COMMAND_TIMEOUT 8       /* default number of seconds to
                                                                                         * wait for IPC command results */
-#define WD_INTERLOCK_WAIT_MSEC         500
-#define WD_INTERLOCK_TIMEOUT_SEC       10
-#define WD_INTERLOCK_WAIT_COUNT ((int) ((WD_INTERLOCK_TIMEOUT_SEC * 1000)/WD_INTERLOCK_WAIT_MSEC))
-
-static void FreeCmdResult(WDIPCCmdResult * res);
-static char *get_wd_failover_state_json(bool start);
-
-static int     open_wd_command_sock(bool throw_error);
-static WDFailoverCMDResults wd_get_failover_result_from_data(WDIPCCmdResult * result, unsigned int *wd_failover_id);
-static WDFailoverCMDResults wd_issue_failover_command(char *func_name, int *node_id_set, int count, unsigned char flags);
-/* shared memory variables */
-char      *watchdog_ipc_address = NULL;
-bool      *watchdog_require_cleanup = NULL;    /* shared memory variable set
-                                                                                                * to true when watchdog
-                                                                                                * process terminates
-                                                                                                * abnormally */
-bool      *watchdog_node_escalated = NULL; /* shared memory variable set to
-                                                                                        * true when watchdog process has
-                                                                                        * performed escalation */
-unsigned int *ipc_shared_key = NULL;   /* key lives in shared memory used to
-                                                                                * identify the ipc internal clients */
-void
-wd_ipc_initialize_data(void)
-{
-       if (watchdog_ipc_address == NULL)
-       {
-               char            wd_ipc_sock_addr[255];
-
-               snprintf(wd_ipc_sock_addr, sizeof(wd_ipc_sock_addr), "%s/.s.PGPOOLWD_CMD.%d",
-                                pool_config->wd_ipc_socket_dir,
-                                pool_config->wd_port);
-
-               watchdog_ipc_address = pool_shared_memory_create(strlen(wd_ipc_sock_addr) + 1);
-               strcpy(watchdog_ipc_address, wd_ipc_sock_addr);
-       }
-
-       if (ipc_shared_key == NULL)
-       {
-               ipc_shared_key = pool_shared_memory_create(sizeof(unsigned int));
-               *ipc_shared_key = 0;
-               while (*ipc_shared_key == 0)
-               {
-                       pool_random_salt((char *) ipc_shared_key);
-               }
-       }
 
-       if (watchdog_require_cleanup == NULL)
-       {
-               watchdog_require_cleanup = pool_shared_memory_create(sizeof(bool));
-               *watchdog_require_cleanup = false;
-       }
-
-       if (watchdog_node_escalated == NULL)
-       {
-               watchdog_node_escalated = pool_shared_memory_create(sizeof(bool));
-               *watchdog_node_escalated = false;
-       }
-}
+int wd_command_timeout_sec = WD_DEFAULT_IPC_COMMAND_TIMEOUT;
 
 WD_STATES
-get_watchdog_local_node_state(void)
+get_watchdog_local_node_state(char* wd_authkey)
 {
        WD_STATES       ret = WD_DEAD;
-       WDGenericData *state = get_wd_runtime_variable_value(WD_RUNTIME_VAR_WD_STATE);
+       WDGenericData *state = get_wd_runtime_variable_value(wd_authkey, WD_RUNTIME_VAR_WD_STATE);
 
        if (state == NULL)
        {
@@ -137,10 +83,10 @@ get_watchdog_local_node_state(void)
 }
 
 int
-get_watchdog_quorum_state(void)
+get_watchdog_quorum_state(char* wd_authkey)
 {
        WD_STATES       ret = WD_DEAD;
-       WDGenericData *state = get_wd_runtime_variable_value(WD_RUNTIME_VAR_QUORUM_STATE);
+       WDGenericData *state = get_wd_runtime_variable_value(wd_authkey, WD_RUNTIME_VAR_QUORUM_STATE);
 
        if (state == NULL)
        {
@@ -162,215 +108,19 @@ get_watchdog_quorum_state(void)
        return ret;
 }
 
-char *
-get_watchdog_ipc_address(void)
-{
-       return watchdog_ipc_address;
-}
-
-unsigned int *
-get_ipc_shared_key(void)
-{
-       return ipc_shared_key;
-}
-
-void
-set_watchdog_process_needs_cleanup(void)
-{
-       *watchdog_require_cleanup = true;
-}
-
-void
-reset_watchdog_process_needs_cleanup(void)
-{
-       *watchdog_require_cleanup = false;
-}
-
-bool
-get_watchdog_process_needs_cleanup(void)
-{
-       return *watchdog_require_cleanup;
-}
-
-
-void
-set_watchdog_node_escalated(void)
-{
-       *watchdog_node_escalated = true;
-}
-
-void
-reset_watchdog_node_escalated(void)
-{
-       *watchdog_node_escalated = false;
-}
-
-bool
-get_watchdog_node_escalation_state(void)
-{
-       return *watchdog_node_escalated;
-}
-
-/*
- * function issues the command to watchdog process over the watchdog
- * IPC command socket.
- * type:            command type to send. valid command
- *                  types are defined in wd_ipc_defines.h
- * timeout_sec:     number of seconds to wait for the command response
- *                  from watchdog
- * data:            command data
- * data_len:        length of data
- * blocking:        send true if caller wants to wait for the results
- *                  when blocking is false the timeout_sec is ignored
- */
-WDIPCCmdResult *
-issue_command_to_watchdog(char type, int timeout_sec, char *data, int data_len, bool blocking)
-{
-       struct timeval start_time,
-                               tv;
-       int                     sock;
-       WDIPCCmdResult *result = NULL;
-       char            res_type = 'P';
-       int                     res_length,
-                               len;
-
-       gettimeofday(&start_time, NULL);
-
-       /* open the watchdog command socket for IPC */
-       sock = open_wd_command_sock(false);
-       if (sock < 0)
-               return NULL;
-
-       len = htonl(data_len);
-
-       if (socket_write(sock, &type, sizeof(char)) <= 0)
-       {
-               close(sock);
-               return NULL;
-       }
-
-       if (socket_write(sock, &len, sizeof(int)) <= 0)
-       {
-               close(sock);
-               return NULL;
-       }
-       if (data && data_len > 0)
-       {
-               if (socket_write(sock, data, data_len) <= 0)
-               {
-                       close(sock);
-                       return NULL;
-               }
-       }
-
-       if (blocking)
-       {
-               /* if we are asked to wait for results */
-               fd_set          fds;
-               struct timeval *timeout_st = NULL;
-
-               if (timeout_sec > 0)
-               {
-                       tv.tv_sec = timeout_sec;
-                       tv.tv_usec = 0;
-                       timeout_st = &tv;
-               }
-               FD_ZERO(&fds);
-               FD_SET(sock, &fds);
-               for (;;)
-               {
-                       int                     select_res;
-
-                       select_res = select(sock + 1, &fds, NULL, NULL, timeout_st);
-                       if (select_res == 0)
-                       {
-                               close(sock);
-                               result = palloc(sizeof(WDIPCCmdResult));
-                               result->type = WD_IPC_CMD_TIMEOUT;
-                               result->length = 0;
-                               result->data = NULL;
-                               return result;
-                       }
-                       if (select_res < 0)
-                       {
-                               if (errno == EAGAIN || errno == EINTR)
-                                       continue;
-                               ereport(WARNING,
-                                               (errmsg("error reading from IPC command socket for ipc command %c", type),
-                                                errdetail("select system call failed with error \"%s\"", strerror(errno))));
-                               close(sock);
-                               return NULL;
-                       }
-                       if (select_res > 0)
-                       {
-                               /* read the result type char */
-                               if (socket_read(sock, &res_type, 1, 0) <= 0)
-                               {
-                                       ereport(WARNING,
-                                                       (errmsg("error reading from IPC command socket for ipc command %c", type),
-                                                        errdetail("read from socket failed with error \"%s\"", strerror(errno))));
-                                       close(sock);
-                                       return result;
-                               }
-                               /* read the result data length */
-                               if (socket_read(sock, &res_length, sizeof(int), 0) <= 0)
-                               {
-                                       ereport(WARNING,
-                                                       (errmsg("error reading from IPC command socket for ipc command %c", type),
-                                                        errdetail("read from socket failed with error \"%s\"", strerror(errno))));
-                                       close(sock);
-                                       return result;
-                               }
-
-                               result = palloc(sizeof(WDIPCCmdResult));
-                               result->type = res_type;
-                               result->length = ntohl(res_length);
-                               result->data = NULL;
-
-                               if (result->length > 0)
-                               {
-                                       result->data = palloc(result->length);
-                                       if (socket_read(sock, result->data, result->length, 0) <= 0)
-                                       {
-                                               pfree(result->data);
-                                               pfree(result);
-                                               ereport(DEBUG1,
-                                                               (errmsg("error reading from IPC command socket for ipc command %c", type),
-                                                                errdetail("read from socket failed with error \"%s\"", strerror(errno))));
-                                               close(sock);
-                                               return NULL;
-                                       }
-                               }
-                               break;
-                       }
-               }
-       }
-       else
-       {
-               /*
-                * For non blocking mode if we are sucessful in sending the command
-                * that means the command is success
-                */
-               result = palloc0(sizeof(WDIPCCmdResult));
-               result->type = WD_IPC_CMD_RESULT_OK;
-       }
-       close(sock);
-       return result;
-}
 
 /*
  * Function gets the runtime value of watchdog varibale using the
  * watchdog IPC
  */
 WDGenericData *
-get_wd_runtime_variable_value(char *varName)
+get_wd_runtime_variable_value(char* wd_authkey, char *varName)
 {
-       unsigned int *shared_key = get_ipc_shared_key();
-       char       *data = get_simple_request_json(WD_JSON_KEY_VARIABLE_NAME, varName,
-                                                                                          shared_key ? *shared_key : 0, pool_config->wd_authkey);
+       char       *data = get_request_json(WD_JSON_KEY_VARIABLE_NAME, varName,
+                                                                                          wd_authkey);
 
        WDIPCCmdResult *result = issue_command_to_watchdog(WD_GET_RUNTIME_VARIABLE_VALUE,
-                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          wd_command_timeout_sec,
                                                                                                           data, strlen(data), true);
 
        pfree(data);
@@ -516,345 +266,29 @@ get_wd_runtime_variable_value(char *varName)
 
 }
 
-/*
- * function gets the PG backend status of all attached nodes from
- * the master watchdog node.
- */
-WDPGBackendStatus *
-get_pg_backend_status_from_master_wd_node(void)
-{
-       unsigned int *shared_key = get_ipc_shared_key();
-       char       *data = get_data_request_json(WD_DATE_REQ_PG_BACKEND_DATA,
-                                                                                        shared_key ? *shared_key : 0, pool_config->wd_authkey);
-
-       WDIPCCmdResult *result = issue_command_to_watchdog(WD_GET_MASTER_DATA_REQUEST,
-                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
-                                                                                                          data, strlen(data), true);
-
-       pfree(data);
-
-       if (result == NULL)
-       {
-               ereport(WARNING,
-                               (errmsg("get backend node status from master watchdog failed"),
-                                errdetail("issue command to watchdog returned NULL")));
-               return NULL;
-       }
-       if (result->type == WD_IPC_CMD_CLUSTER_IN_TRAN)
-       {
-               ereport(WARNING,
-                               (errmsg("get backend node status from master watchdog failed"),
-                                errdetail("watchdog cluster is not in stable state"),
-                                errhint("try again when the cluster is fully initialized")));
-               FreeCmdResult(result);
-               return NULL;
-       }
-       else if (result->type == WD_IPC_CMD_TIMEOUT)
-       {
-               ereport(WARNING,
-                               (errmsg("get backend node status from master watchdog failed"),
-                                errdetail("ipc command timeout")));
-               FreeCmdResult(result);
-               return NULL;
-       }
-       else if (result->type == WD_IPC_CMD_RESULT_OK)
-       {
-               WDPGBackendStatus *backendStatus = get_pg_backend_node_status_from_json(result->data, result->length);
-
-               /*
-                * Watchdog returns the zero length data when the node itself is a
-                * master watchdog node
-                */
-               if (result->length <= 0)
-               {
-                       backendStatus = palloc0(sizeof(WDPGBackendStatus));
-                       backendStatus->node_count = -1;
-               }
-               else
-               {
-                       backendStatus = get_pg_backend_node_status_from_json(result->data, result->length);
-               }
-               FreeCmdResult(result);
-               return backendStatus;
-       }
-
-       ereport(WARNING,
-                       (errmsg("get backend node status from master watchdog failed")));
-       FreeCmdResult(result);
-       return NULL;
-}
-
-WdCommandResult
-wd_start_recovery(void)
-{
-       char            type;
-       unsigned int *shared_key = get_ipc_shared_key();
-
-       char       *func = get_wd_node_function_json(WD_FUNCTION_START_RECOVERY, NULL, 0, 0,
-                                                                                                shared_key ? *shared_key : 0, pool_config->wd_authkey);
-
-       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_ONLINE_RECOVERY_COMMAND,
-                                                                                                          pool_config->recovery_timeout + WD_DEFAULT_IPC_COMMAND_TIMEOUT,
-                                                                                                          func, strlen(func), true);
-
-       pfree(func);
-
-       if (result == NULL)
-       {
-               ereport(WARNING,
-                               (errmsg("start recovery command lock failed"),
-                                errdetail("issue command to watchdog returned NULL")));
-               return COMMAND_FAILED;
-       }
-
-       type = result->type;
-       FreeCmdResult(result);
-       if (type == WD_IPC_CMD_CLUSTER_IN_TRAN)
-       {
-               ereport(WARNING,
-                               (errmsg("start recovery command lock failed"),
-                                errdetail("watchdog cluster is not in stable state"),
-                                errhint("try again when the cluster is fully initialized")));
-               return CLUSTER_IN_TRANSATIONING;
-       }
-       else if (type == WD_IPC_CMD_TIMEOUT)
-       {
-               ereport(WARNING,
-                               (errmsg("start recovery command lock failed"),
-                                errdetail("ipc command timeout")));
-               return COMMAND_TIMEOUT;
-       }
-       else if (type == WD_IPC_CMD_RESULT_OK)
-       {
-               return COMMAND_OK;
-       }
-       return COMMAND_FAILED;
-}
-
-WdCommandResult
-wd_end_recovery(void)
-{
-       char            type;
-       unsigned int *shared_key = get_ipc_shared_key();
-
-       char       *func = get_wd_node_function_json(WD_FUNCTION_END_RECOVERY, NULL, 0, 0,
-                                                                                                shared_key ? *shared_key : 0, pool_config->wd_authkey);
-
-
-       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_ONLINE_RECOVERY_COMMAND,
-                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
-                                                                                                          func, strlen(func), true);
-
-       pfree(func);
-
-       if (result == NULL)
-       {
-               ereport(WARNING,
-                               (errmsg("end recovery command lock failed"),
-                                errdetail("issue command to watchdog returned NULL")));
-               return COMMAND_FAILED;
-       }
-
-       type = result->type;
-       FreeCmdResult(result);
-
-       if (type == WD_IPC_CMD_CLUSTER_IN_TRAN)
-       {
-               ereport(WARNING,
-                               (errmsg("end recovery command lock failed"),
-                                errdetail("watchdog cluster is not in stable state"),
-                                errhint("try again when the cluster is fully initialized")));
-               return CLUSTER_IN_TRANSATIONING;
-       }
-       else if (type == WD_IPC_CMD_TIMEOUT)
-       {
-               ereport(WARNING,
-                               (errmsg("end recovery command lock failed"),
-                                errdetail("ipc command timeout")));
-               return COMMAND_TIMEOUT;
-       }
-       else if (type == WD_IPC_CMD_RESULT_OK)
-       {
-               return COMMAND_OK;
-       }
-       return COMMAND_FAILED;
-}
-
-static char *
-get_wd_failover_state_json(bool start)
-{
-       char       *json_str;
-       JsonNode   *jNode = jw_create_with_object(true);
-       unsigned int *shared_key = get_ipc_shared_key();
-
-       jw_put_int(jNode, WD_IPC_SHARED_KEY, shared_key ? *shared_key : 0); /* put the shared key */
-       if (pool_config->wd_authkey != NULL && strlen(pool_config->wd_authkey) > 0)
-               jw_put_string(jNode, WD_IPC_AUTH_KEY, pool_config->wd_authkey); /* put the auth key */
-
-       jw_put_int(jNode, "FailoverFuncState", start ? 0 : 1);
-       jw_finish_document(jNode);
-       json_str = pstrdup(jw_get_json_string(jNode));
-       jw_destroy(jNode);
-       return json_str;
-}
-
-static WDFailoverCMDResults
-wd_send_failover_func_status_command(bool start)
-{
-       WDFailoverCMDResults res;
-       unsigned int failover_id;
-
-       char       *json_data = get_wd_failover_state_json(start);
-
-       WDIPCCmdResult *result = issue_command_to_watchdog(WD_FAILOVER_INDICATION
-                                                                                                          ,WD_DEFAULT_IPC_COMMAND_TIMEOUT,
-                                                                                                          json_data, strlen(json_data), true);
-
-       pfree(json_data);
-
-       res = wd_get_failover_result_from_data(result, &failover_id);
-
-       FreeCmdResult(result);
-       return res;
-}
-
-static WDFailoverCMDResults wd_get_failover_result_from_data(WDIPCCmdResult * result, unsigned int *wd_failover_id)
-{
-       if (result == NULL)
-               return FAILOVER_RES_ERROR;
-
-       if (result == NULL)
-       {
-               ereport(WARNING,
-                               (errmsg("failover command on watchdog failed"),
-                                errdetail("issue command to watchdog returned NULL")));
-               return FAILOVER_RES_ERROR;
-       }
-       if (result->type == WD_IPC_CMD_CLUSTER_IN_TRAN)
-       {
-               ereport(WARNING,
-                               (errmsg("failover command on watchdog failed"),
-                                errdetail("watchdog cluster is not in stable state"),
-                                errhint("try again when the cluster is fully initialized")));
-               return FAILOVER_RES_TRANSITION;
-       }
-       else if (result->type == WD_IPC_CMD_TIMEOUT)
-       {
-               ereport(WARNING,
-                               (errmsg("failover command on watchdog failed"),
-                                errdetail("ipc command timeout")));
-               return FAILOVER_RES_TIMEOUT;
-       }
-       else if (result->type == WD_IPC_CMD_RESULT_OK)
-       {
-               WDFailoverCMDResults res = FAILOVER_RES_ERROR;
-               json_value *root;
-
-               root = json_parse(result->data, result->length);
-               /* The root node must be object */
-               if (root == NULL || root->type != json_object)
-               {
-                       ereport(NOTICE,
-                                       (errmsg("unable to parse json data from failover command result")));
-                       return res;
-               }
-               if (root && json_get_int_value_for_key(root, WD_FAILOVER_RESULT_KEY, (int *) &res))
-               {
-                       json_value_free(root);
-                       return FAILOVER_RES_ERROR;
-               }
-               if (root && json_get_int_value_for_key(root, WD_FAILOVER_ID_KEY, (int *) wd_failover_id))
-               {
-                       json_value_free(root);
-                       return FAILOVER_RES_ERROR;
-               }
-               return res;
-       }
-       return FAILOVER_RES_ERROR;
-}
-
-static WDFailoverCMDResults
-wd_issue_failover_command(char *func_name, int *node_id_set, int count, unsigned char flags)
-{
-       WDFailoverCMDResults res;
-       char       *func;
-       unsigned int *shared_key = get_ipc_shared_key();
-       unsigned int wd_failover_id;
-
-       func = get_wd_node_function_json(func_name, node_id_set, count, flags,
-                                                                        shared_key ? *shared_key : 0, pool_config->wd_authkey);
-
-       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_FAILOVER_COMMAND,
-                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
-                                                                                                          func, strlen(func), true);
-
-       pfree(func);
-       res = wd_get_failover_result_from_data(result, &wd_failover_id);
-       FreeCmdResult(result);
-       return res;
-}
-
-/*
- * send the degenerate backend request to watchdog.
- * now watchdog can respond to the request in following ways.
- *
- * 1 - It can tell the caller to procees with failover. This
- * happens when the current node is the master watchdog node.
- *
- * 2 - It can tell the caller to failover not allowed
- * this happens when either cluster does not have the quorum
- *
- */
-WDFailoverCMDResults
-wd_degenerate_backend_set(int *node_id_set, int count, unsigned char flags)
-{
-       if (pool_config->use_watchdog)
-               return wd_issue_failover_command(WD_FUNCTION_DEGENERATE_REQUEST, node_id_set, count, flags);
-       return FAILOVER_RES_PROCEED;
-}
-
-WDFailoverCMDResults
-wd_promote_backend(int node_id, unsigned char flags)
-{
-       if (pool_config->use_watchdog)
-               return wd_issue_failover_command(WD_FUNCTION_PROMOTE_REQUEST, &node_id, 1, flags);
-       return FAILOVER_RES_PROCEED;
-}
-
-WDFailoverCMDResults
-wd_send_failback_request(int node_id, unsigned char flags)
-{
-       if (pool_config->use_watchdog)
-               return wd_issue_failover_command(WD_FUNCTION_FAILBACK_REQUEST, &node_id, 1, flags);
-       return FAILOVER_RES_PROCEED;
-}
-
 /*
  * Function returns the JSON of watchdog nodes
  * pass nodeID = -1 to get list of all nodes
  */
 char *
-wd_get_watchdog_nodes(int nodeID)
+wd_get_watchdog_nodes_json(char *wd_authkey, int nodeID)
 {
        WDIPCCmdResult *result;
        char       *json_str;
-       unsigned int *shared_key = get_ipc_shared_key();
 
        JsonNode   *jNode = jw_create_with_object(true);
 
        jw_put_int(jNode, "NodeID", nodeID);
 
-       jw_put_int(jNode, WD_IPC_SHARED_KEY, shared_key ? *shared_key : 0); /* put the shared key */
-
-       if (pool_config->wd_authkey != NULL && strlen(pool_config->wd_authkey) > 0)
-               jw_put_string(jNode, WD_IPC_AUTH_KEY, pool_config->wd_authkey); /* put the auth key */
+       if (wd_authkey != NULL && strlen(wd_authkey) > 0)
+               jw_put_string(jNode, WD_IPC_AUTH_KEY, wd_authkey); /* put the auth key */
 
        jw_finish_document(jNode);
 
        json_str = jw_get_json_string(jNode);
 
        result = issue_command_to_watchdog(WD_GET_NODES_LIST_COMMAND
-                                                                          ,WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                          ,wd_command_timeout_sec,
                                                                           json_str, strlen(json_str), true);
 
        jw_destroy(jNode);
@@ -895,63 +329,114 @@ wd_get_watchdog_nodes(int nodeID)
        return NULL;
 }
 
-static int
-open_wd_command_sock(bool throw_error)
+WDNodeInfo *
+parse_watchdog_node_info_from_wd_node_json(json_value * source)
 {
-       size_t          len;
-       struct sockaddr_un addr;
-       int                     sock = -1;
-
-       /* We use unix domain stream sockets for the purpose */
-       if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) < 0)
-       {
-               /* socket create failed */
-               ereport(throw_error ? ERROR : LOG,
-                               (errmsg("failed to connect to watchdog command server socket"),
-                                errdetail("connect on \"%s\" failed with reason: \"%s\"", addr.sun_path, strerror(errno))));
-               return -1;
-       }
-
-       memset(&addr, 0, sizeof(addr));
-       addr.sun_family = AF_UNIX;
-       snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", watchdog_ipc_address);
-       len = sizeof(struct sockaddr_un);
-
-       if (connect(sock, (struct sockaddr *) &addr, len) == -1)
-       {
-               close(sock);
-               ereport(throw_error ? ERROR : LOG,
-                               (errmsg("failed to connect to watchdog command server socket"),
-                                errdetail("connect on \"%s\" failed with reason: \"%s\"", addr.sun_path, strerror(errno))));
-               return -1;
-       }
-       return sock;
+       char       *ptr;
+       WDNodeInfo *wdNodeInfo = palloc0(sizeof(WDNodeInfo));
+       
+       if (source->type != json_object)
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("node is not of object type")));
+       
+       if (json_get_int_value_for_key(source, "ID", &wdNodeInfo->id))
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find Watchdog Node ID")));
+       }
+       
+       ptr = json_get_string_value_for_key(source, "NodeName");
+       if (ptr == NULL)
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find Watchdog Node Name")));
+       }
+       strncpy(wdNodeInfo->nodeName, ptr, sizeof(wdNodeInfo->nodeName) - 1);
+       
+       ptr = json_get_string_value_for_key(source, "HostName");
+       if (ptr == NULL)
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find Watchdog Host Name")));
+       }
+       strncpy(wdNodeInfo->hostName, ptr, sizeof(wdNodeInfo->hostName) - 1);
+       
+       ptr = json_get_string_value_for_key(source, "DelegateIP");
+       if (ptr == NULL)
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find Watchdog delegate IP")));
+       }
+       strncpy(wdNodeInfo->delegate_ip, ptr, sizeof(wdNodeInfo->delegate_ip) - 1);
+       
+       if (json_get_int_value_for_key(source, "WdPort", &wdNodeInfo->wd_port))
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find WdPort")));
+       }
+       
+       if (json_get_int_value_for_key(source, "PgpoolPort", &wdNodeInfo->pgpool_port))
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find PgpoolPort")));
+       }
+       
+       if (json_get_int_value_for_key(source, "State", &wdNodeInfo->state))
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find state")));
+       }
+       
+       ptr = json_get_string_value_for_key(source, "StateName");
+       if (ptr == NULL)
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find Watchdog State Name")));
+       }
+       strncpy(wdNodeInfo->stateName, ptr, sizeof(wdNodeInfo->stateName) - 1);
+       
+       if (json_get_int_value_for_key(source, "Priority", &wdNodeInfo->wd_priority))
+       {
+               ereport(ERROR,
+                               (errmsg("invalid json data"),
+                                errdetail("unable to find state")));
+       }
+       
+       return wdNodeInfo;
+       
 }
 
-WDFailoverCMDResults
-wd_failover_start(void)
+extern void set_wd_command_timeout(int sec)
 {
-       if (pool_config->use_watchdog)
-               return wd_send_failover_func_status_command(true);
-       return FAILOVER_RES_PROCEED;
+       wd_command_timeout_sec = sec;
 }
 
-WDFailoverCMDResults
-wd_failover_end(void)
+/* The function returs the simple JSON string that contains
+ * only one KEY,VALUE along with the authkey key value if provided
+ */
+char *
+get_request_json(char *key, char *value, char *authKey)
 {
-       if (pool_config->use_watchdog)
-               return wd_send_failover_func_status_command(false);
-       return FAILOVER_RES_PROCEED;
-}
+       char       *json_str;
 
+       JsonNode   *jNode = jw_create_with_object(true);
 
-static void
-FreeCmdResult(WDIPCCmdResult * res)
-{
-       if (res == NULL)
-               return;
+       if (authKey != NULL && strlen(authKey) > 0)
+               jw_put_string(jNode, WD_IPC_AUTH_KEY, authKey); /* put the auth key */
 
-       if (res->data)
-               pfree(res->data);
-       pfree(res);
+       jw_put_string(jNode, key, value);
+       jw_finish_document(jNode);
+       json_str = pstrdup(jw_get_json_string(jNode));
+       jw_destroy(jNode);
+       return json_str;
 }
+
diff --git a/src/watchdog/wd_internal_commands.c b/src/watchdog/wd_internal_commands.c
new file mode 100644 (file)
index 0000000..e394023
--- /dev/null
@@ -0,0 +1,496 @@
+/*
+ * $Header$
+ *
+ * Handles watchdog connection, and protocol communication with pgpool-II
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ */
+#include <stdio.h>
+#include <errno.h>
+#include <ctype.h>
+#include <time.h>
+#include <string.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+#include "pool.h"
+#include "utils/elog.h"
+#include "utils/json_writer.h"
+#include "utils/json.h"
+#include "pool_config.h"
+#include "watchdog/wd_json_data.h"
+#include "watchdog/wd_internal_commands.h"
+
+#define WD_DEFAULT_IPC_COMMAND_TIMEOUT 8       /* default number of seconds to
+                                                                                        * wait for IPC command results */
+#define WD_INTERLOCK_WAIT_MSEC         500
+#define WD_INTERLOCK_TIMEOUT_SEC       10
+#define WD_INTERLOCK_WAIT_COUNT ((int) ((WD_INTERLOCK_TIMEOUT_SEC * 1000)/WD_INTERLOCK_WAIT_MSEC))
+
+/* shared memory variables */
+bool                   *watchdog_require_cleanup = NULL;       /* shared memory variable set
+                                                                                                        * to true when watchdog
+                                                                                                        * process terminates
+                                                                                                        * abnormally */
+bool                   *watchdog_node_escalated = NULL; /* shared memory variable set to
+                                                                                                 * true when watchdog process has
+                                                                                                 * performed escalation */
+unsigned int   *ipc_shared_key = NULL;         /* key lives in shared memory used to
+                                                                                        * identify the ipc internal clients */
+
+static char *get_wd_failover_state_json(bool start);
+static WDFailoverCMDResults wd_get_failover_result_from_data(WDIPCCmdResult * result,
+                                                                                                                        unsigned int *wd_failover_id);
+static WDFailoverCMDResults wd_issue_failover_command(char *func_name, int *node_id_set,
+                                                                                                                       int count, unsigned char flags);
+
+
+void
+wd_ipc_initialize_data(void)
+{
+       wd_ipc_conn_initialize();
+
+       if (ipc_shared_key == NULL)
+       {
+               ipc_shared_key = pool_shared_memory_create(sizeof(unsigned int));
+               *ipc_shared_key = 0;
+               while (*ipc_shared_key == 0)
+               {
+                       pool_random_salt((char *) ipc_shared_key);
+               }
+       }
+
+       if (watchdog_require_cleanup == NULL)
+       {
+               watchdog_require_cleanup = pool_shared_memory_create(sizeof(bool));
+               *watchdog_require_cleanup = false;
+       }
+
+       if (watchdog_node_escalated == NULL)
+       {
+               watchdog_node_escalated = pool_shared_memory_create(sizeof(bool));
+               *watchdog_node_escalated = false;
+       }
+}
+
+
+/*
+ * function gets the PG backend status of all attached nodes from
+ * the master watchdog node.
+ */
+WDPGBackendStatus *
+get_pg_backend_status_from_master_wd_node(void)
+{
+       unsigned int *shared_key = get_ipc_shared_key();
+       char       *data = get_data_request_json(WD_DATE_REQ_PG_BACKEND_DATA,
+                                                                                        shared_key ? *shared_key : 0, pool_config->wd_authkey);
+
+       WDIPCCmdResult *result = issue_command_to_watchdog(WD_GET_MASTER_DATA_REQUEST,
+                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          data, strlen(data), true);
+
+       pfree(data);
+
+       if (result == NULL)
+       {
+               ereport(WARNING,
+                               (errmsg("get backend node status from master watchdog failed"),
+                                errdetail("issue command to watchdog returned NULL")));
+               return NULL;
+       }
+       if (result->type == WD_IPC_CMD_CLUSTER_IN_TRAN)
+       {
+               ereport(WARNING,
+                               (errmsg("get backend node status from master watchdog failed"),
+                                errdetail("watchdog cluster is not in stable state"),
+                                errhint("try again when the cluster is fully initialized")));
+               FreeCmdResult(result);
+               return NULL;
+       }
+       else if (result->type == WD_IPC_CMD_TIMEOUT)
+       {
+               ereport(WARNING,
+                               (errmsg("get backend node status from master watchdog failed"),
+                                errdetail("ipc command timeout")));
+               FreeCmdResult(result);
+               return NULL;
+       }
+       else if (result->type == WD_IPC_CMD_RESULT_OK)
+       {
+               WDPGBackendStatus *backendStatus = get_pg_backend_node_status_from_json(result->data, result->length);
+
+               /*
+                * Watchdog returns the zero length data when the node itself is a
+                * master watchdog node
+                */
+               if (result->length <= 0)
+               {
+                       backendStatus = palloc0(sizeof(WDPGBackendStatus));
+                       backendStatus->node_count = -1;
+               }
+               else
+               {
+                       backendStatus = get_pg_backend_node_status_from_json(result->data, result->length);
+               }
+               FreeCmdResult(result);
+               return backendStatus;
+       }
+
+       ereport(WARNING,
+                       (errmsg("get backend node status from master watchdog failed")));
+       FreeCmdResult(result);
+       return NULL;
+}
+
+WdCommandResult
+wd_start_recovery(void)
+{
+       char            type;
+       unsigned int *shared_key = get_ipc_shared_key();
+
+       char       *func = get_wd_node_function_json(WD_FUNCTION_START_RECOVERY, NULL, 0, 0,
+                                                                                                shared_key ? *shared_key : 0, pool_config->wd_authkey);
+
+       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_ONLINE_RECOVERY_COMMAND,
+                                                                                                          pool_config->recovery_timeout + WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          func, strlen(func), true);
+
+       pfree(func);
+
+       if (result == NULL)
+       {
+               ereport(WARNING,
+                               (errmsg("start recovery command lock failed"),
+                                errdetail("issue command to watchdog returned NULL")));
+               return COMMAND_FAILED;
+       }
+
+       type = result->type;
+       FreeCmdResult(result);
+       if (type == WD_IPC_CMD_CLUSTER_IN_TRAN)
+       {
+               ereport(WARNING,
+                               (errmsg("start recovery command lock failed"),
+                                errdetail("watchdog cluster is not in stable state"),
+                                errhint("try again when the cluster is fully initialized")));
+               return CLUSTER_IN_TRANSATIONING;
+       }
+       else if (type == WD_IPC_CMD_TIMEOUT)
+       {
+               ereport(WARNING,
+                               (errmsg("start recovery command lock failed"),
+                                errdetail("ipc command timeout")));
+               return COMMAND_TIMEOUT;
+       }
+       else if (type == WD_IPC_CMD_RESULT_OK)
+       {
+               return COMMAND_OK;
+       }
+       return COMMAND_FAILED;
+}
+
+WdCommandResult
+wd_end_recovery(void)
+{
+       char            type;
+       unsigned int *shared_key = get_ipc_shared_key();
+
+       char       *func = get_wd_node_function_json(WD_FUNCTION_END_RECOVERY, NULL, 0, 0,
+                                                                                                shared_key ? *shared_key : 0, pool_config->wd_authkey);
+
+
+       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_ONLINE_RECOVERY_COMMAND,
+                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          func, strlen(func), true);
+
+       pfree(func);
+
+       if (result == NULL)
+       {
+               ereport(WARNING,
+                               (errmsg("end recovery command lock failed"),
+                                errdetail("issue command to watchdog returned NULL")));
+               return COMMAND_FAILED;
+       }
+
+       type = result->type;
+       FreeCmdResult(result);
+
+       if (type == WD_IPC_CMD_CLUSTER_IN_TRAN)
+       {
+               ereport(WARNING,
+                               (errmsg("end recovery command lock failed"),
+                                errdetail("watchdog cluster is not in stable state"),
+                                errhint("try again when the cluster is fully initialized")));
+               return CLUSTER_IN_TRANSATIONING;
+       }
+       else if (type == WD_IPC_CMD_TIMEOUT)
+       {
+               ereport(WARNING,
+                               (errmsg("end recovery command lock failed"),
+                                errdetail("ipc command timeout")));
+               return COMMAND_TIMEOUT;
+       }
+       else if (type == WD_IPC_CMD_RESULT_OK)
+       {
+               return COMMAND_OK;
+       }
+       return COMMAND_FAILED;
+}
+
+static char *
+get_wd_failover_state_json(bool start)
+{
+       char       *json_str;
+       JsonNode   *jNode = jw_create_with_object(true);
+       unsigned int *shared_key = get_ipc_shared_key();
+
+       jw_put_int(jNode, WD_IPC_SHARED_KEY, shared_key ? *shared_key : 0); /* put the shared key */
+       if (pool_config->wd_authkey != NULL && strlen(pool_config->wd_authkey) > 0)
+               jw_put_string(jNode, WD_IPC_AUTH_KEY, pool_config->wd_authkey); /* put the auth key */
+
+       jw_put_int(jNode, "FailoverFuncState", start ? 0 : 1);
+       jw_finish_document(jNode);
+       json_str = pstrdup(jw_get_json_string(jNode));
+       jw_destroy(jNode);
+       return json_str;
+}
+
+static WDFailoverCMDResults
+wd_send_failover_func_status_command(bool start)
+{
+       WDFailoverCMDResults res;
+       unsigned int failover_id;
+
+       char       *json_data = get_wd_failover_state_json(start);
+
+       WDIPCCmdResult *result = issue_command_to_watchdog(WD_FAILOVER_INDICATION
+                                                                                                          ,WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          json_data, strlen(json_data), true);
+
+       pfree(json_data);
+
+       res = wd_get_failover_result_from_data(result, &failover_id);
+
+       FreeCmdResult(result);
+       return res;
+}
+
+static WDFailoverCMDResults wd_get_failover_result_from_data(WDIPCCmdResult * result, unsigned int *wd_failover_id)
+{
+       if (result == NULL)
+               return FAILOVER_RES_ERROR;
+
+       if (result == NULL)
+       {
+               ereport(WARNING,
+                               (errmsg("failover command on watchdog failed"),
+                                errdetail("issue command to watchdog returned NULL")));
+               return FAILOVER_RES_ERROR;
+       }
+       if (result->type == WD_IPC_CMD_CLUSTER_IN_TRAN)
+       {
+               ereport(WARNING,
+                               (errmsg("failover command on watchdog failed"),
+                                errdetail("watchdog cluster is not in stable state"),
+                                errhint("try again when the cluster is fully initialized")));
+               return FAILOVER_RES_TRANSITION;
+       }
+       else if (result->type == WD_IPC_CMD_TIMEOUT)
+       {
+               ereport(WARNING,
+                               (errmsg("failover command on watchdog failed"),
+                                errdetail("ipc command timeout")));
+               return FAILOVER_RES_TIMEOUT;
+       }
+       else if (result->type == WD_IPC_CMD_RESULT_OK)
+       {
+               WDFailoverCMDResults res = FAILOVER_RES_ERROR;
+               json_value *root;
+
+               root = json_parse(result->data, result->length);
+               /* The root node must be object */
+               if (root == NULL || root->type != json_object)
+               {
+                       ereport(NOTICE,
+                                       (errmsg("unable to parse json data from failover command result")));
+                       return res;
+               }
+               if (root && json_get_int_value_for_key(root, WD_FAILOVER_RESULT_KEY, (int *) &res))
+               {
+                       json_value_free(root);
+                       return FAILOVER_RES_ERROR;
+               }
+               if (root && json_get_int_value_for_key(root, WD_FAILOVER_ID_KEY, (int *) wd_failover_id))
+               {
+                       json_value_free(root);
+                       return FAILOVER_RES_ERROR;
+               }
+               return res;
+       }
+       return FAILOVER_RES_ERROR;
+}
+
+static WDFailoverCMDResults
+wd_issue_failover_command(char *func_name, int *node_id_set, int count, unsigned char flags)
+{
+       WDFailoverCMDResults res;
+       char       *func;
+       unsigned int *shared_key = get_ipc_shared_key();
+       unsigned int wd_failover_id;
+
+       func = get_wd_node_function_json(func_name, node_id_set, count, flags,
+                                                                        shared_key ? *shared_key : 0, pool_config->wd_authkey);
+
+       WDIPCCmdResult *result = issue_command_to_watchdog(WD_IPC_FAILOVER_COMMAND,
+                                                                                                          WD_DEFAULT_IPC_COMMAND_TIMEOUT,
+                                                                                                          func, strlen(func), true);
+
+       pfree(func);
+       res = wd_get_failover_result_from_data(result, &wd_failover_id);
+       FreeCmdResult(result);
+       return res;
+}
+
+/*
+ * send the degenerate backend request to watchdog.
+ * now watchdog can respond to the request in following ways.
+ *
+ * 1 - It can tell the caller to procees with failover. This
+ * happens when the current node is the master watchdog node.
+ *
+ * 2 - It can tell the caller to failover not allowed
+ * this happens when either cluster does not have the quorum
+ *
+ */
+WDFailoverCMDResults
+wd_degenerate_backend_set(int *node_id_set, int count, unsigned char flags)
+{
+       if (pool_config->use_watchdog)
+               return wd_issue_failover_command(WD_FUNCTION_DEGENERATE_REQUEST, node_id_set, count, flags);
+       return FAILOVER_RES_PROCEED;
+}
+
+WDFailoverCMDResults
+wd_promote_backend(int node_id, unsigned char flags)
+{
+       if (pool_config->use_watchdog)
+               return wd_issue_failover_command(WD_FUNCTION_PROMOTE_REQUEST, &node_id, 1, flags);
+       return FAILOVER_RES_PROCEED;
+}
+
+WDFailoverCMDResults
+wd_send_failback_request(int node_id, unsigned char flags)
+{
+       if (pool_config->use_watchdog)
+               return wd_issue_failover_command(WD_FUNCTION_FAILBACK_REQUEST, &node_id, 1, flags);
+       return FAILOVER_RES_PROCEED;
+}
+
+/*
+ * Function returns the JSON of watchdog nodes
+ * pass nodeID = -1 to get list of all nodes
+ */
+char *
+wd_internal_get_watchdog_nodes_json(int nodeID)
+{
+       return wd_get_watchdog_nodes_json(pool_config->wd_authkey, nodeID);
+}
+
+WDFailoverCMDResults
+wd_failover_start(void)
+{
+       if (pool_config->use_watchdog)
+               return wd_send_failover_func_status_command(true);
+       return FAILOVER_RES_PROCEED;
+}
+
+WDFailoverCMDResults
+wd_failover_end(void)
+{
+       if (pool_config->use_watchdog)
+               return wd_send_failover_func_status_command(false);
+       return FAILOVER_RES_PROCEED;
+}
+
+/* These functions are not available for frontend utilities */
+unsigned int *
+get_ipc_shared_key(void)
+{
+       return ipc_shared_key;
+}
+
+void
+set_watchdog_process_needs_cleanup(void)
+{
+       *watchdog_require_cleanup = true;
+}
+
+void
+reset_watchdog_process_needs_cleanup(void)
+{
+       *watchdog_require_cleanup = false;
+}
+
+bool
+get_watchdog_process_needs_cleanup(void)
+{
+       return *watchdog_require_cleanup;
+}
+
+
+void
+set_watchdog_node_escalated(void)
+{
+       *watchdog_node_escalated = true;
+}
+
+void
+reset_watchdog_node_escalated(void)
+{
+       *watchdog_node_escalated = false;
+}
+
+bool
+get_watchdog_node_escalation_state(void)
+{
+       return *watchdog_node_escalated;
+}
+
+int
+wd_internal_get_watchdog_quorum_state(void)
+{
+       return get_watchdog_quorum_state(pool_config->wd_authkey);
+}
+
+WD_STATES
+wd_internal_get_watchdog_local_node_state(void)
+{
+       return get_watchdog_local_node_state(pool_config->wd_authkey);
+}
diff --git a/src/watchdog/wd_ipc_conn.c b/src/watchdog/wd_ipc_conn.c
new file mode 100644 (file)
index 0000000..4515747
--- /dev/null
@@ -0,0 +1,282 @@
+/*
+ * $Header$
+ *
+ * Handles watchdog connection, and protocol communication with pgpool-II
+ *
+ * pgpool: a language independent connection pool server for PostgreSQL
+ * written by Tatsuo Ishii
+ *
+ * Copyright (c) 2003-2019     PgPool Global Development Group
+ *
+ * Permission to use, copy, modify, and distribute this software and
+ * its documentation for any purpose and without fee is hereby
+ * granted, provided that the above copyright notice appear in all
+ * copies and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * author not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. The author makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ */
+#include <stdio.h>
+#include <errno.h>
+#include <ctype.h>
+#include <time.h>
+#include <string.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+#include "pool.h"
+
+#ifndef POOL_PRIVATE
+#include "utils/elog.h"
+#else
+#include "utils/fe_ports.h"
+#endif
+
+#include "utils/socket_stream.h"
+#include "pool_config.h"
+#include "watchdog/wd_ipc_conn.h"
+#include "watchdog/wd_ipc_defines.h"
+
+static int     open_wd_command_sock(bool throw_error);
+/* shared memory variables */
+char      *watchdog_ipc_address = NULL;
+
+void wd_set_ipc_address(char *socket_dir, int port)
+{
+       if (watchdog_ipc_address == NULL)
+       {
+               char            wd_ipc_sock_addr[255];
+
+               snprintf(wd_ipc_sock_addr, sizeof(wd_ipc_sock_addr), "%s/.s.PGPOOLWD_CMD.%d",
+                                socket_dir,
+                                port);
+
+#ifndef POOL_PRIVATE
+               watchdog_ipc_address = pool_shared_memory_create(strlen(wd_ipc_sock_addr) + 1);
+               strcpy(watchdog_ipc_address, wd_ipc_sock_addr);
+#else
+               watchdog_ipc_address = pstrdup(wd_ipc_sock_addr);
+#endif
+       }
+}
+
+void
+wd_ipc_conn_initialize(void)
+{
+       if (watchdog_ipc_address == NULL)
+       {
+               wd_set_ipc_address(pool_config->wd_ipc_socket_dir, pool_config->wd_port);
+       }
+}
+
+char *
+get_watchdog_ipc_address(void)
+{
+       return watchdog_ipc_address;
+}
+
+/*
+ * function issues the command to watchdog process over the watchdog
+ * IPC command socket.
+ * type:            command type to send. valid command
+ *                  types are defined in wd_ipc_defines.h
+ * timeout_sec:     number of seconds to wait for the command response
+ *                  from watchdog
+ * data:            command data
+ * data_len:        length of data
+ * blocking:        send true if caller wants to wait for the results
+ *                  when blocking is false the timeout_sec is ignored
+ */
+WDIPCCmdResult *
+issue_command_to_watchdog(char type, int timeout_sec, char *data, int data_len, bool blocking)
+{
+       struct timeval start_time,
+                               tv;
+       int                     sock;
+       WDIPCCmdResult *result = NULL;
+       char            res_type = 'P';
+       int                     res_length,
+                               len;
+
+       gettimeofday(&start_time, NULL);
+
+       /* open the watchdog command socket for IPC */
+       sock = open_wd_command_sock(false);
+       if (sock < 0)
+               return NULL;
+
+       len = htonl(data_len);
+
+       if (socket_write(sock, &type, sizeof(char)) <= 0)
+       {
+               close(sock);
+               return NULL;
+       }
+
+       if (socket_write(sock, &len, sizeof(int)) <= 0)
+       {
+               close(sock);
+               return NULL;
+       }
+       if (data && data_len > 0)
+       {
+               if (socket_write(sock, data, data_len) <= 0)
+               {
+                       close(sock);
+                       return NULL;
+               }
+       }
+
+       if (blocking)
+       {
+               /* if we are asked to wait for results */
+               fd_set          fds;
+               struct timeval *timeout_st = NULL;
+
+               if (timeout_sec > 0)
+               {
+                       tv.tv_sec = timeout_sec;
+                       tv.tv_usec = 0;
+                       timeout_st = &tv;
+               }
+               FD_ZERO(&fds);
+               FD_SET(sock, &fds);
+               for (;;)
+               {
+                       int                     select_res;
+
+                       select_res = select(sock + 1, &fds, NULL, NULL, timeout_st);
+                       if (select_res == 0)
+                       {
+                               close(sock);
+                               result = palloc(sizeof(WDIPCCmdResult));
+                               result->type = WD_IPC_CMD_TIMEOUT;
+                               result->length = 0;
+                               result->data = NULL;
+                               return result;
+                       }
+                       if (select_res < 0)
+                       {
+                               if (errno == EAGAIN || errno == EINTR)
+                                       continue;
+                               ereport(WARNING,
+                                               (errmsg("error reading from IPC command socket for ipc command %c", type),
+                                                errdetail("select system call failed with error \"%s\"", strerror(errno))));
+                               close(sock);
+                               return NULL;
+                       }
+                       if (select_res > 0)
+                       {
+                               /* read the result type char */
+                               if (socket_read(sock, &res_type, 1, 0) <= 0)
+                               {
+                                       ereport(WARNING,
+                                                       (errmsg("error reading from IPC command socket for ipc command %c", type),
+                                                        errdetail("read from socket failed with error \"%s\"", strerror(errno))));
+                                       close(sock);
+                                       return result;
+                               }
+                               /* read the result data length */
+                               if (socket_read(sock, &res_length, sizeof(int), 0) <= 0)
+                               {
+                                       ereport(WARNING,
+                                                       (errmsg("error reading from IPC command socket for ipc command %c", type),
+                                                        errdetail("read from socket failed with error \"%s\"", strerror(errno))));
+                                       close(sock);
+                                       return result;
+                               }
+
+                               result = palloc(sizeof(WDIPCCmdResult));
+                               result->type = res_type;
+                               result->length = ntohl(res_length);
+                               result->data = NULL;
+
+                               if (result->length > 0)
+                               {
+                                       result->data = palloc(result->length);
+                                       if (socket_read(sock, result->data, result->length, 0) <= 0)
+                                       {
+                                               pfree(result->data);
+                                               pfree(result);
+                                               ereport(DEBUG1,
+                                                               (errmsg("error reading from IPC command socket for ipc command %c", type),
+                                                                errdetail("read from socket failed with error \"%s\"", strerror(errno))));
+                                               close(sock);
+                                               return NULL;
+                                       }
+                               }
+                               break;
+                       }
+               }
+       }
+       else
+       {
+               /*
+                * For non blocking mode if we are sucessful in sending the command
+                * that means the command is success
+                */
+               result = palloc0(sizeof(WDIPCCmdResult));
+               result->type = WD_IPC_CMD_RESULT_OK;
+       }
+       close(sock);
+       return result;
+}
+
+static int
+open_wd_command_sock(bool throw_error)
+{
+       size_t          len;
+       struct sockaddr_un addr;
+       int                     sock = -1;
+
+       /* We use unix domain stream sockets for the purpose */
+       if ((sock = socket(AF_UNIX, SOCK_STREAM, 0)) < 0)
+       {
+               /* socket create failed */
+               ereport(throw_error ? ERROR : LOG,
+                               (errmsg("failed to connect to watchdog command server socket"),
+                                errdetail("connect on \"%s\" failed with reason: \"%s\"", addr.sun_path, strerror(errno))));
+               return -1;
+       }
+
+       memset(&addr, 0, sizeof(addr));
+       addr.sun_family = AF_UNIX;
+       snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", watchdog_ipc_address);
+       len = sizeof(struct sockaddr_un);
+
+       if (connect(sock, (struct sockaddr *) &addr, len) == -1)
+       {
+               close(sock);
+               ereport(throw_error ? ERROR : LOG,
+                               (errmsg("failed to connect to watchdog command server socket"),
+                                errdetail("connect on \"%s\" failed with reason: \"%s\"", addr.sun_path, strerror(errno))));
+               return -1;
+       }
+       return sock;
+}
+
+void
+FreeCmdResult(WDIPCCmdResult * res)
+{
+       if (res == NULL)
+               return;
+       
+       if (res->data)
+               pfree(res->data);
+       pfree(res);
+}
index 4fa8dd77a379aa68593a4bdb35849307155e3cdd..c296baefe8a8eb2d09457252c7de4bb1edb3a2de 100644 (file)
@@ -645,92 +645,6 @@ ERROR_EXIT:
        return false;
 }
 
-WDNodeInfo *
-get_WDNodeInfo_from_wd_node_json(json_value * source)
-{
-       char       *ptr;
-       WDNodeInfo *wdNodeInfo = palloc0(sizeof(WDNodeInfo));
-
-       if (source->type != json_object)
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("node is not of object type")));
-
-       if (json_get_int_value_for_key(source, "ID", &wdNodeInfo->id))
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find Watchdog Node ID")));
-       }
-
-       ptr = json_get_string_value_for_key(source, "NodeName");
-       if (ptr == NULL)
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find Watchdog Node Name")));
-       }
-       strncpy(wdNodeInfo->nodeName, ptr, sizeof(wdNodeInfo->nodeName) - 1);
-
-       ptr = json_get_string_value_for_key(source, "HostName");
-       if (ptr == NULL)
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find Watchdog Host Name")));
-       }
-       strncpy(wdNodeInfo->hostName, ptr, sizeof(wdNodeInfo->hostName) - 1);
-
-       ptr = json_get_string_value_for_key(source, "DelegateIP");
-       if (ptr == NULL)
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find Watchdog delegate IP")));
-       }
-       strncpy(wdNodeInfo->delegate_ip, ptr, sizeof(wdNodeInfo->delegate_ip) - 1);
-
-       if (json_get_int_value_for_key(source, "WdPort", &wdNodeInfo->wd_port))
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find WdPort")));
-       }
-
-       if (json_get_int_value_for_key(source, "PgpoolPort", &wdNodeInfo->pgpool_port))
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find PgpoolPort")));
-       }
-
-       if (json_get_int_value_for_key(source, "State", &wdNodeInfo->state))
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find state")));
-       }
-
-       ptr = json_get_string_value_for_key(source, "StateName");
-       if (ptr == NULL)
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find Watchdog State Name")));
-       }
-       strncpy(wdNodeInfo->stateName, ptr, sizeof(wdNodeInfo->stateName) - 1);
-
-       if (json_get_int_value_for_key(source, "Priority", &wdNodeInfo->wd_priority))
-       {
-               ereport(ERROR,
-                               (errmsg("invalid json data"),
-                                errdetail("unable to find state")));
-       }
-
-       return wdNodeInfo;
-
-}
-
 char *
 get_wd_node_function_json(char *func_name, int *node_id_set, int count, unsigned char flags, unsigned int sharedKey, char *authKey)
 {
index 54c45c16bd639b1d6ed93406fa7440e00440860d..75d2e835df81839dce967fe3f27eb3a6a822a9a8 100644 (file)
@@ -43,7 +43,7 @@
 #include "watchdog/wd_utils.h"
 #include "watchdog/wd_lifecheck.h"
 #include "watchdog/wd_ipc_defines.h"
-#include "watchdog/wd_ipc_commands.h"
+#include "watchdog/wd_internal_commands.h"
 #include "watchdog/wd_json_data.h"
 
 #include "libpq-fe.h"
@@ -551,7 +551,7 @@ inform_node_status(LifeCheckNode * node, char *message)
 static bool
 fetch_watchdog_nodes_data(void)
 {
-       char       *json_data = wd_get_watchdog_nodes(-1);
+       char       *json_data = wd_internal_get_watchdog_nodes_json(-1);
 
        if (json_data == NULL)
        {
@@ -620,9 +620,11 @@ load_watchdog_nodes_from_json(char *json_data, int len)
        gslifeCheckCluster->lifeCheckNodes = pool_shared_memory_create(sizeof(LifeCheckNode) * gslifeCheckCluster->nodeCount);
        for (i = 0; i < nodeCount; i++)
        {
-               WDNodeInfo *nodeInfo = get_WDNodeInfo_from_wd_node_json(value->u.array.values[i]);
-
-               gslifeCheckCluster->lifeCheckNodes[i].nodeState = NODE_EMPTY;
+               WDNodeInfo *nodeInfo = parse_watchdog_node_info_from_wd_node_json(value->u.array.values[i]);
+               
+               gslifeCheckCluster->lifeCheckNodes[i].wdState = nodeInfo->state;
+               strcpy(gslifeCheckCluster->lifeCheckNodes[i].stateName, nodeInfo->stateName);
+               gslifeCheckCluster->lifeCheckNodes[i].nodeState = NODE_EMPTY; /* This is local health check state*/
                gslifeCheckCluster->lifeCheckNodes[i].ID = nodeInfo->id;
                strcpy(gslifeCheckCluster->lifeCheckNodes[i].hostName, nodeInfo->hostName);
                strcpy(gslifeCheckCluster->lifeCheckNodes[i].nodeName, nodeInfo->nodeName);