Add health check statistics collection/display feature.
authorTatsuo Ishii <ishii@sraoss.co.jp>
Sun, 26 Jan 2020 11:01:18 +0000 (20:01 +0900)
committerTatsuo Ishii <ishii@sraoss.co.jp>
Sun, 26 Jan 2020 11:01:18 +0000 (20:01 +0900)
The health check process now collects statistics data such as number
of total health check performed, number of health check retry count
and health check duration and so on in the shared memory area. This
commit also adds new "show pool_health_check_stats" SQL
command. Corresponding PCP command and pgpool_adm function will be
added in subsequent commit.

doc/src/sgml/ref/allfiles.sgml
doc/src/sgml/reference.sgml
doc/src/sgml/version.sgml
src/include/pcp/libpcp_ext.h
src/include/pool.h
src/include/utils/pool_process_reporting.h
src/main/health_check.c
src/main/pgpool_main.c
src/protocol/pool_proto_modules.c
src/utils/pool_process_reporting.c

index 4b9897a0785dfddf7330b7949c3a05bae0c9ad08..7a643ccaf51442ecfd51ca940ce8049fd4ab2003 100644 (file)
@@ -1,6 +1,6 @@
 <!--
 doc/src/sgml/ref/allfiles.sgml
-PostgreSQL documentation
+Pgpool-II documentation
 Complete list of usable sgml source files in this directory.
 -->
 
@@ -31,6 +31,7 @@ Complete list of usable sgml source files in this directory.
 <!ENTITY showPoolPools       SYSTEM "show_pool_pools.sgml">
 <!ENTITY showPoolVersion     SYSTEM "show_pool_version.sgml">
 <!ENTITY showPoolCache       SYSTEM "show_pool_cache.sgml">
+<!ENTITY showPoolHealthCheckStats SYSTEM "show_pool_health_check_stats.sgml">
 <!ENTITY pgpoolAdmPcpNodeInfo SYSTEM "pgpool_adm_pcp_node_info.sgml">
 <!ENTITY pgpoolAdmPcpPoolStatus SYSTEM "pgpool_adm_pcp_pool_status.sgml">
 <!ENTITY pgpoolAdmPcpNodeCount SYSTEM "pgpool_adm_pcp_node_count.sgml">
index c8b4be282bcd52beed3411ad0dd90e26a31edc80..17cb8843a0f6ee21a4db072772842c91cf07a090 100644 (file)
   &showPoolPools
   &showPoolVersion
   &showPoolCache
-
+  &showPoolHealthCheckStats
  </reference>
 
  <reference id="pgpool-adm">
index 997e1f0d51a582db9071f0a2bf81f891b14e0b94..0db052f62de220cbd93f83082c127b889fb6c524 100644 (file)
@@ -1 +1 @@
-<!ENTITY version "4.1devel">
+<!ENTITY version "4.2devel">
index 430a75247a454a913ef827f9c1c02363b8ad141f..8b3a91ab16960ec1ac79060e0b9da0c6abeb2db5 100644 (file)
@@ -171,6 +171,7 @@ typedef struct
 #define POOLCONFIG_MAXWEIGHTLEN 20
 #define POOLCONFIG_MAXDATELEN 128
 #define POOLCONFIG_MAXCOUNTLEN 16
+#define POOLCONFIG_MAXLONGCOUNTLEN 20
 
 /* config report struct*/
 typedef struct
@@ -231,6 +232,31 @@ typedef struct
        char            version[POOLCONFIG_MAXVALLEN + 1];
 }                      POOL_REPORT_VERSION;
 
+/* health check statistics report struct */
+typedef struct
+{
+       char            node_id[POOLCONFIG_MAXIDLEN + 1];
+       char            hostname[MAX_DB_HOST_NAMELEN + 1];
+       char            port[POOLCONFIG_MAXPORTLEN + 1];
+       char            status[POOLCONFIG_MAXSTATLEN + 1];
+       char            role[POOLCONFIG_MAXWEIGHTLEN + 1];
+       char            last_status_change[POOLCONFIG_MAXDATELEN];
+       char            total_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            success_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            fail_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            skip_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            retry_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            average_retry_count[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            max_retry_count[POOLCONFIG_MAXCOUNTLEN+1];
+       char            max_health_check_duration[POOLCONFIG_MAXCOUNTLEN+1];
+       char            min_health_check_duration[POOLCONFIG_MAXCOUNTLEN+1];
+       char            average_health_check_duration[POOLCONFIG_MAXLONGCOUNTLEN+1];
+       char            last_health_check[POOLCONFIG_MAXDATELEN];
+       char            last_successful_health_check[POOLCONFIG_MAXDATELEN];
+       char            last_skip_health_check[POOLCONFIG_MAXDATELEN];
+       char            last_failed_health_check[POOLCONFIG_MAXDATELEN];
+}                      POOL_HEALTH_CHECK_STATS;
+
 typedef enum
 {
        PCP_CONNECTION_OK,
index 8995b961803c86a6f767c6e9fc416c984e674eb4..794837b50637f5265c92c9339db18bddf9d6257e 100644 (file)
@@ -4,7 +4,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2019     PgPool Global Development Group
+ * Copyright (c) 2003-2020     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -358,6 +358,27 @@ typedef struct
        UNIT            unit;
 } Interval;
 
+/*
+ * Health check statistics per node
+*/
+typedef struct {
+       uint64  total_count;    /* total count of health check */
+       uint64  success_count;  /* total count of successful health check */
+       uint64  fail_count;             /* total count of failed health check */
+       uint64  skip_count;             /* total count of skipped health check */
+       uint64  retry_count;    /* total count of health check retries */
+       uint32  max_retry_count;        /* max retry count in a health check session */
+       uint64  total_health_check_duration;    /* sum of health check duration */
+       int32   max_health_check_duration;      /* maximum duration spent for a health check session in milli seconds */
+       int32   min_health_check_duration;      /* minimum duration spent for a health check session in milli seconds */
+       time_t  last_health_check;      /* last health check timestamp */
+       time_t  last_successful_health_check;   /* last succesfull health check timestamp */
+       time_t  last_skip_health_check;                 /* last skipped health check timestamp */
+       time_t  last_failed_health_check;               /* last failed health check timestamp */
+} POOL_HEALTH_CHECK_STATISTICS;
+
+extern volatile POOL_HEALTH_CHECK_STATISTICS   *health_check_stats;    /* health check stats area in shared memory */
+
 /* Defined in pool_session_context.h */
 extern int     pool_get_major_version(void);
 
@@ -867,4 +888,8 @@ extern POOL_NODE_STATUS * pool_get_node_status(void);
 extern void pool_set_backend_status_changed_time(int backend_id);
 extern int     get_next_master_node(void);
 
+/* health_check.c */
+extern size_t  health_check_stats_shared_memory_size(void);
+extern void            health_check_stats_init(POOL_HEALTH_CHECK_STATISTICS *addr);
+
 #endif                                                 /* POOL_H */
index eb75f3def9805308bed31e3e4beb84400f1333fb..45f717d88b5c9a73a4f1dcde93b074f137178acd 100644 (file)
@@ -6,7 +6,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2012     PgPool Global Development Group
+ * Copyright (c) 2003-2020     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -34,12 +34,15 @@ extern POOL_REPORT_POOLS * get_pools(int *nrows);
 extern POOL_REPORT_PROCESSES * get_processes(int *nrows);
 extern POOL_REPORT_NODES * get_nodes(int *nrows);
 extern POOL_REPORT_VERSION * get_version(void);
+POOL_HEALTH_CHECK_STATS *get_health_check_stats(int *nrows);
+
 extern void config_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 extern void pools_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 extern void processes_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 extern void nodes_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 extern void version_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 extern void cache_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
+extern void show_health_check_stats(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend);
 
 extern void send_config_var_detail_row(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend, const char *name, const char *value, const char *description);
 extern void send_config_var_value_only_row(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend, const char *value);
index f341c50d238a4a270cb6a7332c70f5864d52273f..7071ba3d103f24aee627b7f9f33b1b1884218587 100644 (file)
@@ -5,7 +5,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2019     PgPool Global Development Group
+ * Copyright (c) 2003-2020     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
 #include "auth/pool_hba.h"
 #include "utils/pool_stream.h"
 
+volatile POOL_HEALTH_CHECK_STATISTICS  *health_check_stats;    /* health check stats area in shared memory */
+
 char           remote_ps_data[NI_MAXHOST]; /* used for set_ps_display */
 static POOL_CONNECTION_POOL_SLOT * slot;
 static volatile sig_atomic_t reload_config_request = 0;
 static volatile sig_atomic_t restart_request = 0;
+/*static POOL_HEALTH_CHECK_STATISTICS  stats;*/
+volatile POOL_HEALTH_CHECK_STATISTICS *stats;
+
 static bool establish_persistent_connection(int node);
 static void discard_persistent_connection(int node);
 static RETSIGTYPE my_signal_handler(int sig);
@@ -112,6 +117,12 @@ do_health_check_child(int *node_id)
        sigjmp_buf      local_sigjmp_buf;
        MemoryContext HealthCheckMemoryContext;
        char            psbuffer[NI_MAXHOST];
+       static struct timeval   start_time;
+       static struct timeval   end_time;
+       long    diff_t;
+
+       POOL_HEALTH_CHECK_STATISTICS    mystat;
+       stats = &health_check_stats[*node_id];
 
        ereport(DEBUG1,
                        (errmsg("I am health check process pid:%d DB node id:%d", getpid(), *node_id)));
@@ -158,16 +169,20 @@ do_health_check_child(int *node_id)
        /* We can now handle ereport(ERROR) */
        PG_exception_stack = &local_sigjmp_buf;
 
+       /* Initialize health check stats area */
+       stats->min_health_check_duration = INT_MAX;
 
        for (;;)
        {
                MemoryContextSwitchTo(HealthCheckMemoryContext);
                MemoryContextResetAndDeleteChildren(HealthCheckMemoryContext);
+               bool    skipped = false;
 
                CHECK_REQUEST;
 
                if (pool_config->health_check_params[*node_id].health_check_period <= 0)
                {
+                       stats->min_health_check_duration = 0;
                        sleep(30);
                }
 
@@ -180,10 +195,19 @@ do_health_check_child(int *node_id)
                        bool            result;
                        BackendInfo *bkinfo = pool_get_node_info(*node_id);
 
+                       stats->total_count++;
+                       gettimeofday(&start_time, NULL);
+                       ereport(LOG,
+                                       (errmsg("node: %d start time: %ld %ld", *node_id, start_time.tv_sec, start_time.tv_usec)));
+
+                       stats->last_health_check = time(NULL);
+
                        result = establish_persistent_connection(*node_id);
 
                        if (result && slot == NULL)
                        {
+                               stats->last_failed_health_check = time(NULL);
+
                                if (POOL_DISALLOW_TO_FAILOVER(BACKEND_INFO(*node_id).flag))
                                {
                                        ereport(LOG,
@@ -194,6 +218,8 @@ do_health_check_child(int *node_id)
                                {
                                        bool            partial;
 
+                                       stats->fail_count++;
+
                                        ereport(LOG, (errmsg("health check failed on node %d (timeout:%d)",
                                                                                 *node_id, health_check_timer_expired)));
 
@@ -213,14 +239,57 @@ do_health_check_child(int *node_id)
                        }
                        else if (slot && bkinfo->backend_status == CON_DOWN && bkinfo->quarantine == true)
                        {
+                               stats->success_count++;
+                               stats->last_successful_health_check = time(NULL);
+
                                /* The node has become reachable again. Reset
                                 * the quarantine state
                                 */
                                send_failback_request(*node_id, false, REQ_DETAIL_UPDATE | REQ_DETAIL_WATCHDOG);
                        }
+                       else if (result && slot)
+                       {
+                               /* Health check succeeded */
+                               stats->success_count++;
+                               stats->last_successful_health_check = time(NULL);
+                       }
+                       else if (!result)
+                       {
+                               /* Health check skipped */
+                               stats->skip_count++;
+                               stats->last_skip_health_check = time(NULL);
+                               skipped = true;
+                       }
 
                        /* Discard persistent connections */
                        discard_persistent_connection(*node_id);
+
+                       /*
+                        Update health check duration only if health check was not skipped
+                        since the duration could be very small (probably 0) if health
+                        check is skipped.
+                        */
+
+                       if (!skipped)
+                       {
+                               gettimeofday(&end_time, NULL);
+
+                               if (end_time.tv_sec > start_time.tv_sec)
+                                       diff_t = end_time.tv_usec - start_time.tv_usec + 1000000 * (end_time.tv_sec - start_time.tv_sec);
+                               else
+                                       diff_t = end_time.tv_usec - start_time.tv_usec;
+
+                               diff_t /= 1000;
+                               stats->total_health_check_duration += diff_t;
+
+                               if (diff_t > stats->max_health_check_duration)
+                                       stats->max_health_check_duration = diff_t;
+                               if (diff_t < stats->min_health_check_duration)
+                                       stats->min_health_check_duration = diff_t;
+                       }
+
+                       memcpy(&mystat, (void *)stats, sizeof(mystat));
+
                        sleep(pool_config->health_check_params[*node_id].health_check_period);
                }
        }
@@ -276,11 +345,11 @@ establish_persistent_connection(int node)
         */
        if (slot == NULL)
        {
-               retry_cnt = pool_config->health_check_params[node].health_check_max_retries;
-
                char       *password = get_pgpool_config_user_password(pool_config->health_check_params[node].health_check_user,
                                                                                                                           pool_config->health_check_params[node].health_check_password);
 
+               retry_cnt = pool_config->health_check_params[node].health_check_max_retries;
+
                do
                {
                        /*
@@ -331,6 +400,8 @@ establish_persistent_connection(int node)
 
                        if (retry_cnt >= 0)
                        {
+                               stats->retry_count++;
+
                                ereport(LOG,
                                                (errmsg("health check retrying on DB node: %d (round:%d)",
                                                                node,
@@ -340,6 +411,19 @@ establish_persistent_connection(int node)
                        }
                } while (retry_cnt >= 0);
 
+               /* Check if we need to refresh max retry count */
+
+               if (retry_cnt != pool_config->health_check_params[node].health_check_max_retries)
+               {
+                       int ret_cnt;
+
+                       retry_cnt++;
+                       ret_cnt = pool_config->health_check_params[node].health_check_max_retries - retry_cnt;
+
+                       if (ret_cnt > stats->max_retry_count)
+                               stats->max_retry_count = ret_cnt;
+               }
+
                if (password)
                        pfree(password);
 
@@ -354,6 +438,7 @@ establish_persistent_connection(int node)
                                send_failback_request(node, true, REQ_DETAIL_CONFIRMED);
                }
        }
+
        /* if check_failback is true, backend_status is DOWN or UNUSED. */
        if (check_failback)
        {
@@ -441,6 +526,36 @@ static RETSIGTYPE health_check_timer_handler(int sig)
        errno = save_errno;
 }
 
+/*
+ * Returns the byte size of health check statistics area
+ */
+size_t
+health_check_stats_shared_memory_size(void)
+{
+       size_t  size;
+
+       size =  MAXALIGN(sizeof(POOL_HEALTH_CHECK_STATISTICS) * MAX_NUM_BACKENDS);
+       elog(LOG, "health_check_stats_shared_memory_size: requested size: %lu", size);
+       return size;
+}
+
+/*
+ * Initialize health check statistics area
+ */
+void
+health_check_stats_init(POOL_HEALTH_CHECK_STATISTICS *addr)
+{
+       int     i;
+
+       health_check_stats = addr;
+       memset((void *) health_check_stats, 0, health_check_stats_shared_memory_size());
+
+       for (i = 0 ;i < MAX_NUM_BACKENDS; i++)
+       {
+               health_check_stats[i].min_health_check_duration = INT_MAX;
+       }
+}
+
 #ifdef HEALTHCHECK_DEBUG
 
 /*
index 8bfcfbc9cc13e0ce5bcf8ecd9c7be6fc12609ecf..fd2c218033f52a2ca97a90d54dfcad7e08d3c181 100644 (file)
@@ -3707,6 +3707,10 @@ initialize_shared_mem_objects(bool clear_memcache_oidmaps)
        /* Initialize statistics area */
        stat_set_stat_area(pool_shared_memory_create(stat_shared_memory_size()));
        stat_init_stat_area();
+
+       /* Initialize health check statistics area */
+       health_check_stats_init(pool_shared_memory_create(health_check_stats_shared_memory_size()));
+
        /* initialize watchdog IPC unix domain socket address */
        if (pool_config->use_watchdog)
        {
index e1aa52cf216fb6c5ffdfaa129f9ea5d056c39cbf..39e906e5a2a8768a4431d573476d8fb85b246483 100644 (file)
@@ -3,7 +3,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2019     PgPool Global Development Group
+ * Copyright (c) 2003-2020     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -175,6 +175,7 @@ SimpleQuery(POOL_CONNECTION * frontend,
        static char *sq_nodes = "pool_nodes";
        static char *sq_version = "pool_version";
        static char *sq_cache = "pool_cache";
+       static char *sq_health_check_stats = "pool_health_check_stats";
        int                     commit;
        List       *parse_tree_list;
        Node       *node = NULL;
@@ -430,6 +431,14 @@ SimpleQuery(POOL_CONNECTION * frontend,
                                                 errdetail("cache reporting")));
                                cache_reporting(frontend, backend);
                        }
+                       else if (!strcmp(sq_health_check_stats, vnode->name))
+                       {
+                               is_valid_show_command = true;
+                               ereport(DEBUG1,
+                                               (errmsg("SimpleQuery"),
+                                                errdetail("health check stats")));
+                               show_health_check_stats(frontend, backend);
+                       }
 
                        if (is_valid_show_command)
                        {
index 10d0ec12eccce348cfe5054d1a9574c9c285f689..f15367685073a124c6d3af500dbf68f88854e36a 100644 (file)
@@ -5,7 +5,7 @@
  * pgpool: a language independent connection pool server for PostgreSQL
  * written by Tatsuo Ishii
  *
- * Copyright (c) 2003-2019     PgPool Global Development Group
+ * Copyright (c) 2003-2020     PgPool Global Development Group
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -32,6 +32,8 @@
 #include <string.h>
 #include <netinet/in.h>
 
+static void write_one_field(POOL_CONNECTION * frontend, char *field);
+static void write_one_field_v2(POOL_CONNECTION * frontend, char *field);
 
 void
 send_row_description(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend,
@@ -2005,3 +2007,230 @@ cache_reporting(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend)
 
        pfree(strp);
 }
+
+/*
+ * for SHOW health_check_stats
+ */
+POOL_HEALTH_CHECK_STATS *
+get_health_check_stats(int *nrows)
+{
+       int                     i;
+       POOL_HEALTH_CHECK_STATS *stats = palloc0(NUM_BACKENDS * sizeof(POOL_HEALTH_CHECK_STATS));
+       BackendInfo *bi = NULL;
+       struct tm       tm;
+       time_t          t;
+       double          f;
+
+       for (i = 0; i < NUM_BACKENDS; i++)
+       {
+               bi = pool_get_node_info(i);
+
+               snprintf(stats[i].node_id, POOLCONFIG_MAXIDLEN, "%d", i);
+               StrNCpy(stats[i].hostname, bi->backend_hostname, strlen(bi->backend_hostname) + 1);
+               snprintf(stats[i].port, POOLCONFIG_MAXPORTLEN, "%d", bi->backend_port);
+               snprintf(stats[i].status, POOLCONFIG_MAXSTATLEN, "%s", backend_status_to_str(bi));
+
+               if (STREAM)
+               {
+                       if (i == REAL_PRIMARY_NODE_ID)
+                       {
+                               snprintf(stats[i].role, POOLCONFIG_MAXWEIGHTLEN, "%s", "primary");
+                       }
+                       else
+                       {
+                               snprintf(stats[i].role, POOLCONFIG_MAXWEIGHTLEN, "%s", "standby");
+                       }
+               }
+               else
+               {
+                       if (i == REAL_MASTER_NODE_ID)
+                               snprintf(stats[i].role, POOLCONFIG_MAXWEIGHTLEN, "%s", "master");
+                       else
+                               snprintf(stats[i].role, POOLCONFIG_MAXWEIGHTLEN, "%s", "slave");
+               }
+
+               /* status last changed */
+               localtime_r(&bi->status_changed_time, &tm);
+               strftime(stats[i].last_status_change, POOLCONFIG_MAXDATELEN, "%F %T", &tm);
+
+               snprintf(stats[i].total_count, POOLCONFIG_MAXLONGCOUNTLEN, UINT64_FORMAT, health_check_stats[i].total_count);
+               snprintf(stats[i].success_count, POOLCONFIG_MAXLONGCOUNTLEN, UINT64_FORMAT, health_check_stats[i].success_count);
+               snprintf(stats[i].fail_count, POOLCONFIG_MAXLONGCOUNTLEN, UINT64_FORMAT, health_check_stats[i].fail_count);
+               snprintf(stats[i].skip_count, POOLCONFIG_MAXLONGCOUNTLEN, UINT64_FORMAT, health_check_stats[i].skip_count);
+               snprintf(stats[i].retry_count, POOLCONFIG_MAXLONGCOUNTLEN, UINT64_FORMAT, health_check_stats[i].retry_count);
+               snprintf(stats[i].max_retry_count, POOLCONFIG_MAXCOUNTLEN, "%d", health_check_stats[i].max_retry_count);
+
+               if (pool_config->health_check_params[i].health_check_period > 0)
+                       f = (double)health_check_stats[i].retry_count /
+                               (health_check_stats[i].total_count - health_check_stats[i].skip_count);
+               else
+                       f = 0.0;
+               snprintf(stats[i].average_retry_count, POOLCONFIG_MAXWEIGHTLEN, "%f", f);
+
+               if (pool_config->health_check_params[i].health_check_period > 0)
+                       f = (double)health_check_stats[i].total_health_check_duration /
+                               (health_check_stats[i].total_count - health_check_stats[i].skip_count);
+               else
+                       f = 0.0;
+               snprintf(stats[i].average_health_check_duration, POOLCONFIG_MAXWEIGHTLEN, "%f", f);
+
+               snprintf(stats[i].max_health_check_duration, POOLCONFIG_MAXCOUNTLEN, "%d", health_check_stats[i].max_health_check_duration);
+               snprintf(stats[i].min_health_check_duration, POOLCONFIG_MAXCOUNTLEN, "%d", health_check_stats[i].min_health_check_duration);
+
+               t = health_check_stats[i].last_health_check;
+               if (t > 0)
+                       strftime(stats[i].last_health_check, POOLCONFIG_MAXDATELEN, "%F %T", localtime(&t));
+
+               t = health_check_stats[i].last_successful_health_check;
+               if (t > 0)
+                       strftime(stats[i].last_successful_health_check, POOLCONFIG_MAXDATELEN, "%F %T", localtime(&t));
+
+               t = health_check_stats[i].last_skip_health_check;
+               if (t > 0)
+                       strftime(stats[i].last_skip_health_check, POOLCONFIG_MAXDATELEN, "%F %T", localtime(&t));
+
+               t = health_check_stats[i].last_failed_health_check;
+               if (t > 0)
+                       strftime(stats[i].last_failed_health_check, POOLCONFIG_MAXDATELEN, "%F %T", localtime(&t));
+       }
+
+       *nrows = i;
+
+       return stats;
+}
+
+/*
+ * SHOW health_check_stats;
+ */
+void
+show_health_check_stats(POOL_CONNECTION * frontend, POOL_CONNECTION_POOL * backend)
+{
+       static char *field_names[] = {"node_id", "hostname", "port", "status", "role", "last_status_change",
+                                                                 "total_count", "success_count", "fail_count", "skip_count", "retry_count",
+                                                                 "average_retry_count", "max_retry_count", "max_duration", "min_duration",
+                                                                 "average_duration", "last_health_check", "last_successful_health_check",
+                                                                 "last_skip_health_check", "last_failed_health_check"};
+       short           num_fields = sizeof(field_names) / sizeof(char *);
+       int                     i;
+       short           s;
+       int                     len;
+       int                     nrows;
+       static unsigned char nullmap[2] = {0xff, 0xff};
+       int                     nbytes = (num_fields + 7) / 8;
+
+       POOL_HEALTH_CHECK_STATS *stats = get_health_check_stats(&nrows);
+
+       send_row_description(frontend, backend, num_fields, field_names);
+
+       if (MAJOR(backend) == PROTO_MAJOR_V2)
+       {
+               /* ascii row */
+               for (i = 0; i < nrows; i++)
+               {
+                       pool_write(frontend, "D", 1);
+                       pool_write_and_flush(frontend, nullmap, nbytes);
+
+                       write_one_field_v2(frontend, stats[i].node_id);
+                       write_one_field_v2(frontend, stats[i].hostname);
+                       write_one_field_v2(frontend, stats[i].port);
+                       write_one_field_v2(frontend, stats[i].status);
+                       write_one_field_v2(frontend, stats[i].role);
+                       write_one_field_v2(frontend, stats[i].last_status_change);
+                       write_one_field_v2(frontend, stats[i].total_count);
+                       write_one_field_v2(frontend, stats[i].success_count);
+                       write_one_field_v2(frontend, stats[i].fail_count);
+                       write_one_field_v2(frontend, stats[i].skip_count);
+                       write_one_field_v2(frontend, stats[i].retry_count);
+                       write_one_field_v2(frontend, stats[i].average_retry_count);
+                       write_one_field_v2(frontend, stats[i].max_retry_count);
+                       write_one_field_v2(frontend, stats[i].max_health_check_duration);
+                       write_one_field_v2(frontend, stats[i].min_health_check_duration);
+                       write_one_field_v2(frontend, stats[i].average_health_check_duration);
+                       write_one_field_v2(frontend, stats[i].last_health_check);
+                       write_one_field_v2(frontend, stats[i].last_successful_health_check);
+                       write_one_field_v2(frontend, stats[i].last_skip_health_check);
+                       write_one_field_v2(frontend, stats[i].last_failed_health_check);
+               }
+       }
+       else
+       {
+               /* data row */
+               for (i = 0; i < nrows; i++)
+               {
+                       pool_write(frontend, "D", 1);
+                       len = 6;                        /* int32 + int16; */
+                       len += 4 + strlen(stats[i].node_id);    /* int32 + data; */
+                       len += 4 + strlen(stats[i].hostname);   /* int32 + data; */
+                       len += 4 + strlen(stats[i].port);       /* int32 + data; */
+                       len += 4 + strlen(stats[i].status);     /* int32 + data; */
+                       len += 4 + strlen(stats[i].role);       /* int32 + data; */
+                       len += 4 + strlen(stats[i].last_status_change); /* int32 + data; */
+                       len += 4 + strlen(stats[i].total_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].success_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].fail_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].skip_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].retry_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].average_retry_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].max_retry_count); /* int32 + data; */
+                       len += 4 + strlen(stats[i].max_health_check_duration); /* int32 + data; */
+                       len += 4 + strlen(stats[i].min_health_check_duration); /* int32 + data; */
+                       len += 4 + strlen(stats[i].average_health_check_duration); /* int32 + data; */
+                       len += 4 + strlen(stats[i].last_health_check); /* int32 + data; */
+                       len += 4 + strlen(stats[i].last_successful_health_check); /* int32 + data; */
+                       len += 4 + strlen(stats[i].last_skip_health_check); /* int32 + data; */
+                       len += 4 + strlen(stats[i].last_failed_health_check); /* int32 + data; */
+
+                       len = htonl(len);
+                       pool_write(frontend, &len, sizeof(len));
+                       s = htons(num_fields);
+                       pool_write(frontend, &s, sizeof(s));
+
+                       write_one_field(frontend, stats[i].node_id);
+                       write_one_field(frontend, stats[i].hostname);
+                       write_one_field(frontend, stats[i].port);
+                       write_one_field(frontend, stats[i].status);
+                       write_one_field(frontend, stats[i].role);
+                       write_one_field(frontend, stats[i].last_status_change);
+                       write_one_field(frontend, stats[i].total_count);
+                       write_one_field(frontend, stats[i].success_count);
+                       write_one_field(frontend, stats[i].fail_count);
+                       write_one_field(frontend, stats[i].skip_count);
+                       write_one_field(frontend, stats[i].retry_count);
+                       write_one_field(frontend, stats[i].average_retry_count);
+                       write_one_field(frontend, stats[i].max_retry_count);
+                       write_one_field(frontend, stats[i].max_health_check_duration);
+                       write_one_field(frontend, stats[i].min_health_check_duration);
+                       write_one_field(frontend, stats[i].average_health_check_duration);
+                       write_one_field(frontend, stats[i].last_health_check);
+                       write_one_field(frontend, stats[i].last_successful_health_check);
+                       write_one_field(frontend, stats[i].last_skip_health_check);
+                       write_one_field(frontend, stats[i].last_failed_health_check);
+               }
+       }
+
+       send_complete_and_ready(frontend, backend, "SELECT", nrows);
+
+       pfree(stats);
+}
+
+/* Write one field to frontend (v3) */
+static void write_one_field(POOL_CONNECTION * frontend, char *field)
+{
+       int     size, hsize;
+
+       size = strlen(field);
+       hsize = htonl(size);
+       pool_write(frontend, &hsize, sizeof(hsize));
+       pool_write(frontend, field, size);
+}
+
+/* Write one field to frontend (v2) */
+static void write_one_field_v2(POOL_CONNECTION * frontend, char *field)
+{
+       int     size, hsize;
+
+       size = strlen(field);
+       hsize = htonl(size + 4);
+       pool_write(frontend, &hsize, sizeof(hsize));
+       pool_write(frontend, field, size);
+}