diff --git a/contribs/pmi2/pmi2.h b/contribs/pmi2/pmi2.h
index ddcc36c..8bee03d 100644
--- a/contribs/pmi2/pmi2.h
+++ b/contribs/pmi2/pmi2.h
@@ -52,6 +52,8 @@ static const char NAMEUNPUBLISH_CMD[]     = "name-unpublish";
 static const char NAMEUNPUBLISHRESP_CMD[] = "name-unpublish-response";
 static const char NAMELOOKUP_CMD[]        = "name-lookup";
 static const char NAMELOOKUPRESP_CMD[]    = "name-lookup-response";
+static const char ALLGATHER_CMD[]         = "allgather";
+static const char ALLGATHERRESP_CMD[]     = "allgather-response";
 
 static const char PMIJOBID_KEY[]          = "pmijobid";
 static const char PMIRANK_KEY[]           = "pmirank";
@@ -451,6 +453,12 @@ int PMI2_KVS_Put(const char key[], const char value[]);
 @*/
 int PMI2_KVS_Fence(void);
 
+int PMI2_KVS_Ifence(void);
+int PMI2_KVS_Wait(void);
+
+int PMI2_Iallgather(const char value[]);
+int PMI2_Iallgather_wait(void *result);
+
 /*@
   PMI2_KVS_Get - returns the value associated with key in the key-value
       space associated with the job ID jobid
diff --git a/contribs/pmi2/pmi2_api.c b/contribs/pmi2/pmi2_api.c
index 2789fbc..4d51268 100644
--- a/contribs/pmi2/pmi2_api.c
+++ b/contribs/pmi2/pmi2_api.c
@@ -92,6 +92,7 @@ static void init_kv_strdup_intsuffix(PMI2_Keyvalpair *kv, const char key_prefix[
 static int getPMIFD(void);
 static int PMIi_ReadCommandExp( int fd, PMI2_Command *cmd, const char *exp, int* rc, const char **errmsg );
 static int PMIi_ReadCommand( int fd, PMI2_Command *cmd );
+static int PMIi_ReadRaw( int fd, char *buf );
 
 static int PMIi_WriteSimpleCommand( int fd, PMI2_Command *resp, const char cmd[], PMI2_Keyvalpair *pairs[], int npairs);
 static int PMIi_WriteSimpleCommandStr( int fd, PMI2_Command *resp, const char cmd[], ...);
@@ -743,6 +744,47 @@ fn_fail:
     goto fn_exit;
 }
 
+int PMI2_KVS_Ifence(void)
+{
+    int pmi2_errno = PMI2_SUCCESS;
+    PMI2_Command cmd = {0};
+
+    PMI2U_printf("[BEGIN]");
+
+    pmi2_errno = PMIi_WriteSimpleCommandStr(PMI2_fd, &cmd, KVSFENCE_CMD, NULL);
+    if (pmi2_errno) PMI2U_ERR_SETANDJUMP(1, pmi2_errno, "PMIi_WriteSimpleCommandStr");
+
+fn_exit:
+    free(cmd.command);
+    freepairs(cmd.pairs, cmd.nPairs);
+    PMI2U_printf("[END]");
+    return pmi2_errno;
+fn_fail:
+    goto fn_exit;
+}
+
+int PMI2_KVS_Wait(void)
+{
+    int pmi2_errno = PMI2_SUCCESS;
+    PMI2_Command cmd = {0};
+    int rc;
+    const char *errmsg;
+
+    PMI2U_printf("[BEGIN]");
+
+    pmi2_errno = PMIi_ReadCommandExp(PMI2_fd, &cmd, KVSFENCERESP_CMD, &rc, &errmsg);
+    if (pmi2_errno) PMI2U_ERR_SETANDJUMP(1, pmi2_errno, "PMIi_ReadCommandExp");
+    PMI2U_ERR_CHKANDJUMP(rc, pmi2_errno, PMI2_ERR_OTHER, "**pmi2_kvsfence %s", errmsg ? errmsg : "unknown");
+
+fn_exit:
+    free(cmd.command);
+    freepairs(cmd.pairs, cmd.nPairs);
+    PMI2U_printf("[END]");
+    return pmi2_errno;
+fn_fail:
+    goto fn_exit;
+}
+
 int PMI2_KVS_Get(const char *jobid, int src_pmi_id, const char key[], char value [], int maxValue, int *valLen)
 {
     int pmi2_errno = PMI2_SUCCESS;
@@ -787,6 +829,43 @@ int PMI2_KVS_Get(const char *jobid, int src_pmi_id, const char key[], char value
     goto fn_exit;
 }
 
+int PMI2_Iallgather(const char value[])
+{
+    int pmi2_errno = PMI2_SUCCESS;
+    PMI2_Command cmd = {0};
+    int rc;
+
+    PMI2U_printf("[BEGIN]");
+
+    pmi2_errno = PMIi_WriteSimpleCommandStr(PMI2_fd, &cmd, ALLGATHER_CMD, VALUE_KEY, value, NULL);
+    if (pmi2_errno) PMI2U_ERR_SETANDJUMP(1, pmi2_errno, "PMIi_WriteSimpleCommandStr");
+
+fn_exit:
+    free(cmd.command);
+    freepairs(cmd.pairs, cmd.nPairs);
+    PMI2U_printf("[END]");
+    return pmi2_errno;
+fn_fail:
+    goto fn_exit;
+}
+
+int PMI2_Iallgather_wait(void *results)
+{
+    int pmi2_errno = PMI2_SUCCESS;
+
+    PMI2U_printf("[BEGIN]");
+
+    pmi2_errno = PMIi_ReadRaw(PMI2_fd, results);
+    if (pmi2_errno) PMI2U_ERR_SETANDJUMP(1, pmi2_errno, "PMIi_ReadRaw");
+
+fn_exit:
+    PMI2U_printf("[END]");
+    return pmi2_errno;
+fn_fail:
+    goto fn_exit;
+}
+
+
 int PMI2_Info_GetNodeAttr(const char name[], char value[], int valuelen, int *flag, int waitfor)
 {
     int pmi2_errno = PMI2_SUCCESS;
@@ -1315,6 +1394,39 @@ fn_fail:
 }
 
 
+static int PMIi_ReadRaw( int fd, char *buf )
+{
+    int pmi2_errno = PMI2_SUCCESS;
+	int data_len, nbytes, offset;
+
+    PMI2U_printf("[BEGIN]");
+
+	do {
+		nbytes = read(fd, &data_len, sizeof data_len);
+	} while (nbytes == -1 && errno == EINTR);
+
+    PMI2U_printf("[PMIi_ReadRaw] data_len=%d", data_len);
+
+	offset = 0;
+	do
+	{
+		do {
+			nbytes = read(fd, &buf[offset], data_len - offset);
+		} while (nbytes == -1 && errno == EINTR);
+
+		PMI2U_ERR_CHKANDJUMP(nbytes <= 0, pmi2_errno, PMI2_ERR_OTHER, "**read %s", strerror(errno));
+
+		offset += nbytes;
+	}
+	while (offset < data_len);
+
+fn_exit:
+    PMI2U_printf("[END]");
+    return pmi2_errno;
+fn_fail:
+    goto fn_exit;
+}
+
 /* Note that we fill in the fields in a command that is provided.
    We may want to share these routines with the PMI version 2 server */
 int PMIi_ReadCommand( int fd, PMI2_Command *cmd )
diff --git a/contribs/pmi2/slurm/pmi2.h b/contribs/pmi2/slurm/pmi2.h
index 59bdad6..4fe58f9 100644
--- a/contribs/pmi2/slurm/pmi2.h
+++ b/contribs/pmi2/slurm/pmi2.h
@@ -52,6 +52,8 @@ static const char NAMEUNPUBLISH_CMD[]     = "name-unpublish";
 static const char NAMEUNPUBLISHRESP_CMD[] = "name-unpublish-response";
 static const char NAMELOOKUP_CMD[]        = "name-lookup";
 static const char NAMELOOKUPRESP_CMD[]    = "name-lookup-response";
+static const char ALLGATHER_CMD[]         = "allgather";
+static const char ALLGATHERRESP_CMD[]     = "allgather-response";
 static const char RING_CMD[]              = "ring";
 static const char RINGRESP_CMD[]          = "ring-response";
 
@@ -483,6 +485,12 @@ int PMI2_KVS_Put(const char key[], const char value[]);
 @*/
 int PMI2_KVS_Fence(void);
 
+int PMI2_KVS_IFence(void);
+int PMI2_KVS_IWait(void);
+
+int PMI2_Iallgather(const char value[]);
+int PMI2_Iallgather_Wait(void *result);
+
 /*@
   PMI2_KVS_Get - returns the value associated with key in the key-value
       space associated with the job ID jobid
diff --git a/src/plugins/mpi/pmi2/client.c b/src/plugins/mpi/pmi2/client.c
index f4ba6fe..667ebe7 100644
--- a/src/plugins/mpi/pmi2/client.c
+++ b/src/plugins/mpi/pmi2/client.c
@@ -524,6 +524,17 @@ client_req_get_bool(client_req_t *req, const char *key, bool *pval)
 
 /* ************************************************************ */
 
+extern int
+client_resp_send_raw(int fd, const void *buf, int len_buf)
+{
+	safe_write(fd, &len_buf, sizeof len_buf);
+	safe_write(fd, buf, len_buf);
+	return SLURM_SUCCESS;
+
+rwfail:
+	return SLURM_ERROR;
+}
+
 extern client_resp_t *
 client_resp_new(void)
 {
@@ -620,3 +631,17 @@ send_kvs_fence_resp_to_clients(int rc, char *errmsg)
 	client_resp_free(resp);
 	return rc;
 }
+
+extern int
+send_allgather_resp_to_clients(const void *buf, int len_buf)
+{
+	int i;
+	int rc = SLURM_SUCCESS;
+	debug2("send_allgather_resp_to_clients: %d bytes", len_buf);
+
+	for (i = 0; i < job_info.ltasks; i ++) {
+		rc = client_resp_send_raw(STEPD_PMI_SOCK(i), buf, len_buf);
+	}
+
+	return rc;
+}
diff --git a/src/plugins/mpi/pmi2/client.h b/src/plugins/mpi/pmi2/client.h
index 85bbe77..08a4677 100644
--- a/src/plugins/mpi/pmi2/client.h
+++ b/src/plugins/mpi/pmi2/client.h
@@ -93,6 +93,7 @@ extern spawn_subcmd_t *client_req_parse_spawn_subcmd(client_req_t *req);
 
 extern client_resp_t *client_resp_new(void);
 extern int  client_resp_send(client_resp_t *req, int fd);
+extern int  client_resp_send_raw(int fd, const void *buf, int len_buf);
 extern void client_resp_free(client_resp_t *resp);
 /* XXX: this requires CPP */
 #define client_resp_append(msg, fmt, ...) do { \
@@ -101,5 +102,6 @@ extern void client_resp_free(client_resp_t *resp);
 
 
 extern int send_kvs_fence_resp_to_clients(int rc, char *errmsg);
+extern int send_allgather_resp_to_clients(const void *buf, int len_buf);
 
 #endif	/* _CLIENT_H */
diff --git a/src/plugins/mpi/pmi2/kvs.c b/src/plugins/mpi/pmi2/kvs.c
index 0a0950c..d75e063 100644
--- a/src/plugins/mpi/pmi2/kvs.c
+++ b/src/plugins/mpi/pmi2/kvs.c
@@ -51,6 +51,8 @@ int children_to_wait = 0;
 int kvs_seq = 1; /* starting from 1 */
 int waiting_kvs_resp = 0;
 
+/* for allgather */
+int allg_seq = 1; /* starting from 1 */
 
 /* bucket of key-value pairs */
 typedef struct kvs_bucket {
@@ -66,6 +68,10 @@ static char *temp_kvs_buf = NULL;
 static int temp_kvs_cnt = 0;
 static int temp_kvs_size = 0;
 
+static char *allgather_buf = NULL;
+static int allgather_cnt = 0;
+static int allgather_size = 0;
+
 static int no_dup_keys = 0;
 
 #define TASKS_PER_BUCKET 8
@@ -334,3 +340,154 @@ kvs_clear(void)
 
 	return SLURM_SUCCESS;
 }
+
+/**************************************************************/
+
+extern int
+allgather_init(void)
+{
+	uint16_t cmd;
+	uint32_t nodeid, num_children, size;
+	Buf buf = NULL;
+
+	xfree(allgather_buf);
+	allgather_cnt = 0;
+	allgather_size = TEMP_KVS_SIZE_INC;
+	allgather_buf = xmalloc(allgather_size);
+
+	/* put the tree cmd here to simplify message sending */
+	if (in_stepd()) {
+		cmd = TREE_CMD_ALLGATHER;
+	} else {
+		cmd = TREE_CMD_ALLGATHER_RESP;
+	}
+
+	buf = init_buf(1024);
+	pack16(cmd, buf);
+	if (in_stepd()) {
+		nodeid = job_info.nodeid;
+		/* XXX: TBC */
+		num_children = tree_info.num_children + 1;
+
+		pack32((uint32_t)nodeid, buf); /* from_nodeid */
+		packstr(tree_info.this_node, buf); /* from_node */
+		pack32((uint32_t)num_children, buf); /* num_children */
+		pack32(allg_seq, buf);
+	} else {
+		pack32(allg_seq, buf);
+	}
+	size = get_buf_offset(buf);
+	if (allgather_cnt + size > allgather_size) {
+		allgather_size += TEMP_KVS_SIZE_INC;
+		xrealloc(allgather_buf, allgather_size);
+	}
+	memcpy(&allgather_buf[allgather_cnt], get_buf_data(buf), size);
+	allgather_cnt += size;
+	free_buf(buf);
+
+	tasks_to_wait = 0;
+	children_to_wait = 0;
+
+	return SLURM_SUCCESS;
+}
+
+extern int
+allgather_add(int rank, char *val)
+{
+	Buf buf;
+	uint32_t size;
+
+	if ( val == NULL )
+		return SLURM_SUCCESS;
+
+	buf = init_buf(PMI2_MAX_VALLEN + 3 * sizeof(uint32_t));
+	pack32(rank, buf);
+	packstr(val, buf);
+	size = get_buf_offset(buf);
+	if (allgather_cnt + size > allgather_size) {
+		allgather_size += TEMP_KVS_SIZE_INC;
+		xrealloc(allgather_buf, allgather_size);
+	}
+	memcpy(&allgather_buf[allgather_cnt], get_buf_data(buf), size);
+	allgather_cnt += size;
+	free_buf(buf);
+
+	return SLURM_SUCCESS;
+}
+
+extern int
+allgather_merge(Buf buf)
+{
+	char *data;
+	uint32_t offset, size;
+
+	size = remaining_buf(buf);
+	if (size == 0) {
+		return SLURM_SUCCESS;
+	}
+	data = get_buf_data(buf);
+	offset = get_buf_offset(buf);
+
+	if (allgather_cnt + size > allgather_size) {
+		allgather_size += size;
+		xrealloc(allgather_buf, allgather_size);
+	}
+	memcpy(&allgather_buf[allgather_cnt], &data[offset], size);
+	allgather_cnt += size;
+
+	return SLURM_SUCCESS;
+}
+
+extern int
+allgather_send(void)
+{
+	int rc = SLURM_ERROR, retry = 0;
+	unsigned int delay = 1;
+	hostlist_t hl = NULL;
+	char free_hl = 0;
+
+	if (! in_stepd()) {	/* srun */
+		hl = hostlist_create(job_info.step_nodelist);
+		free_hl = 1;
+	} else if (tree_info.parent_node != NULL) {
+		hl = hostlist_create(tree_info.parent_node);
+		free_hl = 1;
+	}
+
+
+	/* cmd included in allgather_buf */
+	allg_seq ++; /* expecting new kvs after now */
+
+	while (1) {
+		if (retry == 1) {
+			verbose("failed to send allgather kvs, rc=%d, retrying", rc);
+		}
+
+		if (! in_stepd()) {	/* srun */
+			rc = tree_msg_to_stepds(hl,
+						allgather_cnt,
+						allgather_buf);
+		} else if (tree_info.parent_node != NULL) {
+			/* non-first-level stepds */
+			rc = tree_msg_to_stepds(hl,
+						allgather_cnt,
+						allgather_buf);
+		} else {		/* first level stepds */
+			rc = tree_msg_to_srun(allgather_cnt, allgather_buf);
+		}
+		if (rc == SLURM_SUCCESS)
+			break;
+		retry ++;
+		if (retry >= MAX_RETRIES)
+			break;
+		/* wait, in case parent stepd / srun not ready */
+		sleep(delay);
+		delay *= 2;
+	}
+	allgather_init();	/* clear old allgather kvs */
+	if( free_hl ){
+		hostlist_destroy(hl);
+	}
+	return rc;
+}
+
diff --git a/src/plugins/mpi/pmi2/kvs.h b/src/plugins/mpi/pmi2/kvs.h
index f2e3f42..2024313 100644
--- a/src/plugins/mpi/pmi2/kvs.h
+++ b/src/plugins/mpi/pmi2/kvs.h
@@ -51,6 +51,7 @@
 extern int tasks_to_wait;
 extern int children_to_wait;
 extern int kvs_seq;
+extern int allg_seq;
 extern int waiting_kvs_resp;
 
 extern int   temp_kvs_init(void);
@@ -58,6 +59,11 @@ extern int   temp_kvs_add(char *key, char *val);
 extern int   temp_kvs_merge(Buf buf);
 extern int   temp_kvs_send(void);
 
+extern int   allgather_init(void);
+extern int   allgather_add(int rank, char *val);
+extern int   allgather_merge(Buf buf);
+extern int   allgather_send(void);
+
 extern int   kvs_init(void);
 extern char *kvs_get(char *key);
 extern int   kvs_put(char *key, char *val);
diff --git a/src/plugins/mpi/pmi2/pmi.h b/src/plugins/mpi/pmi2/pmi.h
index b60592b..d7d0376 100644
--- a/src/plugins/mpi/pmi2/pmi.h
+++ b/src/plugins/mpi/pmi2/pmi.h
@@ -152,6 +152,8 @@
 #define NAMELOOKUPRESP_CMD     "name-lookup-response"
 #define SPAWN_CMD              "spawn"
 #define SPAWNRESP_CMD          "spawn-response"
+#define ALLGATHER_CMD          "allgather"
+#define ALLGATHERRESP_CMD      "allgather-response"
 #define RING_CMD               "ring"
 #define RINGRESP_CMD           "ring-response"
 
diff --git a/src/plugins/mpi/pmi2/pmi2.c b/src/plugins/mpi/pmi2/pmi2.c
index 8423450..33cfd37 100644
--- a/src/plugins/mpi/pmi2/pmi2.c
+++ b/src/plugins/mpi/pmi2/pmi2.c
@@ -80,6 +80,7 @@ static int _handle_name_publish(int fd, int lrank, client_req_t *req);
 static int _handle_name_unpublish(int fd, int lrank, client_req_t *req);
 static int _handle_name_lookup(int fd, int lrank, client_req_t *req);
 static int _handle_spawn(int fd, int lrank, client_req_t *req);
+static int _handle_allgather(int fd, int lrank, client_req_t *req);
 
 
 static struct {
@@ -103,6 +104,7 @@ static struct {
 	{ NAMEUNPUBLISH_CMD,     _handle_name_unpublish },
 	{ NAMELOOKUP_CMD,        _handle_name_lookup },
 	{ SPAWN_CMD,             _handle_spawn },
+	{ ALLGATHER_CMD,         _handle_allgather },
 	{ NULL, NULL},
 };
 
@@ -230,6 +232,34 @@ _handle_job_disconnect(int fd, int lrank, client_req_t *req)
 }
 
 static int
+_handle_allgather(int fd, int lrank, client_req_t *req)
+{
+	int rc = SLURM_SUCCESS;
+	client_resp_t *resp;
+	char *val = NULL;
+
+	debug2("mpi/pmi2: in _handle_allgather");
+	client_req_parse_body(req);
+	client_req_get_str(req, VALUE_KEY, &val);
+
+	rc = allgather_add(job_info.gtids[lrank], val);
+
+	if (tasks_to_wait == 0 && children_to_wait == 0) {
+		tasks_to_wait = job_info.ltasks;
+		children_to_wait = tree_info.num_children;
+	}
+	tasks_to_wait --;
+
+	if (tasks_to_wait == 0 && children_to_wait == 0) {
+		rc = allgather_send();
+		waiting_kvs_resp = 1;
+	}
+
+	debug2("mpi/pmi2: out _handle_allgather");
+	return rc;
+}
+
+static int
 _handle_ring(int fd, int lrank, client_req_t *req)
 {
 	int rc = SLURM_SUCCESS;
diff --git a/src/plugins/mpi/pmi2/setup.c b/src/plugins/mpi/pmi2/setup.c
index 1d18455..0c34dad 100644
--- a/src/plugins/mpi/pmi2/setup.c
+++ b/src/plugins/mpi/pmi2/setup.c
@@ -254,7 +254,9 @@ _setup_stepd_tree_info(const stepd_step_rec_t *job, char ***env)
 	/* init kvs seq to 0. TODO: reduce array size */
 	tree_info.children_kvs_seq = xmalloc(sizeof(uint32_t) *
 					     job_info.nnodes);
-
+	tree_info.children_allg_seq = xmalloc(sizeof(uint32_t) *
+					     job_info.nnodes);
+	
 	return SLURM_SUCCESS;
 }
 
@@ -313,6 +315,10 @@ _setup_stepd_kvs(const stepd_step_rec_t *job, char ***env)
 	if (rc != SLURM_SUCCESS)
 		return rc;
 
+	rc = allgather_init();
+	if (rc != SLURM_SUCCESS)
+		return rc;
+
 	rc = kvs_init();
 	if (rc != SLURM_SUCCESS)
 		return rc;
@@ -634,7 +640,9 @@ _setup_srun_tree_info(const mpi_plugin_client_info_t *job)
 	/* init kvs seq to 0. TODO: reduce array size */
 	tree_info.children_kvs_seq = xmalloc(sizeof(uint32_t) *
 					     job_info.nnodes);
-
+	tree_info.children_allg_seq = xmalloc(sizeof(uint32_t) *
+					     job_info.nnodes);
+	
 	return SLURM_SUCCESS;
 }
 
@@ -658,6 +666,7 @@ _setup_srun_kvs(const mpi_plugin_client_info_t *job)
 
 	kvs_seq = 1;
 	rc = temp_kvs_init();
+	rc = allgather_init();
 	return rc;
 }
 
diff --git a/src/plugins/mpi/pmi2/setup.h b/src/plugins/mpi/pmi2/setup.h
index 0e0a91c..ae5fc0b 100644
--- a/src/plugins/mpi/pmi2/setup.h
+++ b/src/plugins/mpi/pmi2/setup.h
@@ -100,6 +100,7 @@ typedef struct pmi2_tree_info {
 	uint16_t pmi_port;	 /* PMI2 comm port of this srun */
 	slurm_addr_t *srun_addr; /* PMI2 comm address parent srun */
 	uint32_t *children_kvs_seq; /* sequence number of children nodes */
+	uint32_t *children_allg_seq; /* sequence number of children nodes */
 } pmi2_tree_info_t;
 
 
diff --git a/src/plugins/mpi/pmi2/tree.c b/src/plugins/mpi/pmi2/tree.c
index bf6a1ac..e59ce23 100644
--- a/src/plugins/mpi/pmi2/tree.c
+++ b/src/plugins/mpi/pmi2/tree.c
@@ -69,6 +69,8 @@ static int _handle_spawn_resp(int fd, Buf buf);
 static int _handle_name_publish(int fd, Buf buf);
 static int _handle_name_unpublish(int fd, Buf buf);
 static int _handle_name_lookup(int fd, Buf buf);
+static int _handle_allgather(int fd, Buf buf);
+static int _handle_allgather_resp(int fd, Buf buf);
 static int _handle_ring(int fd, Buf buf);
 static int _handle_ring_resp(int fd, Buf buf);
 
@@ -84,6 +86,8 @@ static int (*tree_cmd_handlers[]) (int fd, Buf buf) = {
 	_handle_name_publish,
 	_handle_name_unpublish,
 	_handle_name_lookup,
+	_handle_allgather,
+	_handle_allgather_resp,
 	_handle_ring,
 	_handle_ring_resp,
 	NULL
@@ -97,11 +101,15 @@ static char *tree_cmd_names[] = {
 	"TREE_CMD_NAME_PUBLISH",
 	"TREE_CMD_NAME_UNPUBLISH",
 	"TREE_CMD_NAME_LOOKUP",
+	"TREE_CMD_ALLGATHER",
+	"TREE_CMD_ALLGATHER_RESP",
 	"TREE_CMD_RING",
 	"TREE_CMD_RING_RESP",
 	NULL,
 };
 
+/**************************************************************/
+
 static int
 _handle_kvs_fence(int fd, Buf buf)
 {
@@ -226,6 +234,150 @@ unpack_error:
 	goto resp;
 }
 
+/**************************************************************/
+
+static int
+_get_sorted_result(Buf buf, char **newbuf, int *buflen)
+{
+	int maxlen = PMI2_MAX_VALLEN, result_len;
+	int rc = SLURM_SUCCESS;
+	uint32_t rank, temp32;
+	char val[maxlen];
+	char *result;
+
+	debug3("mpi/pmi2: _get_sorted_result");
+	result_len = job_info.ntasks * maxlen * sizeof (char);
+	result = xmalloc(result_len);
+	memset(result, '\0', result_len);
+
+	temp32 = remaining_buf(buf);
+	while (remaining_buf(buf) > 0) {
+		safe_unpack32(&rank, buf);
+		unpackmem(val, &temp32, buf);
+		memcpy(&result[rank*maxlen], val, temp32);
+	}
+
+	*newbuf = result;
+	*buflen = result_len;
+out:
+	return rc;
+
+unpack_error:
+	error("mpi/pmi2: unpack error in allgather resp");
+	rc = SLURM_ERROR;
+	goto out;
+}
+
+static int
+_handle_allgather(int fd, Buf buf)
+{
+	uint32_t from_nodeid, num_children, temp32, seq;
+	char *from_node = NULL;
+	int rc = SLURM_SUCCESS;
+
+	safe_unpack32(&from_nodeid, buf);
+	safe_unpackstr_xmalloc(&from_node, &temp32, buf);
+	safe_unpack32(&num_children, buf);
+	safe_unpack32(&seq, buf);
+
+	debug3("mpi/pmi2: in _handle_allgather, from node %u(%s) representing"
+	       " %u offspring, seq=%u", from_nodeid, from_node, num_children,
+	       seq);
+	if (seq != allg_seq) {
+		error("mpi/pmi2: invalid kvs seq from node %u(%s) ignored, "
+		      "expect %u got %u", 
+		      from_nodeid, from_node, allg_seq, seq);
+		goto out;
+	}
+	if (seq == tree_info.children_allg_seq[from_nodeid]) {
+		info("mpi/pmi2: duplicate ALLGATHER request from node %u(%s) "
+		      "ignored, seq=%u", from_nodeid, from_node, seq);
+		goto out;
+	}
+	tree_info.children_allg_seq[from_nodeid] = seq;
+	
+	if (tasks_to_wait == 0 && children_to_wait == 0) {
+		tasks_to_wait = job_info.ltasks;
+		children_to_wait = tree_info.num_children;
+	}
+	children_to_wait -= num_children;
+
+	allgather_merge(buf);
+
+	if (children_to_wait == 0 && tasks_to_wait == 0) {
+
+		rc = allgather_send();
+		if (rc != SLURM_SUCCESS) {
+			if (in_stepd()) {
+				error("mpi/pmi2: failed to send allgather kvs"
+				      " to %s",
+				      tree_info.parent_node ?: "srun");
+			} else {
+				error("mpi/pmi2: failed to send allgather kvs"
+				      " to compute nodes");
+			}
+			/* cancel the step to avoid tasks hang */
+			slurm_kill_job_step(job_info.jobid, job_info.stepid,
+					    SIGKILL);
+		} else {
+			if (in_stepd())
+				waiting_kvs_resp = 1;
+		}
+	}
+	debug3("mpi/pmi2: out _handle_allgather, tasks_to_wait=%d, "
+	       "children_to_wait=%d", tasks_to_wait, children_to_wait);
+out:
+	xfree(from_node);
+	return rc;
+
+unpack_error:
+	error("mpi/pmi2: failed to unpack allgather message");
+	rc = SLURM_ERROR;
+	goto out;
+}
+
+static int
+_handle_allgather_resp(int fd, Buf buf)
+{
+	int rc = SLURM_SUCCESS;
+	uint32_t rank, temp32, seq;
+	char *newbuf;
+	int len;
+
+	debug3("mpi/pmi2: in _handle_allgather_resp");
+
+	safe_unpack32(&seq, buf);
+	debug("_handle_allgather_resp, seq: %lu", seq);
+	if (seq != allg_seq - 1) {
+		error("mpi/pmi2: invalid kvs seq from srun, expect %u"
+		      " got %u", allg_seq - 1, seq);
+		return SLURM_ERROR;
+	}
+	if (! waiting_kvs_resp) {
+		debug("mpi/pmi2: duplicate ALLGATHER_RESP from srun ignored");
+		return rc;
+	} else {
+		waiting_kvs_resp = 0;
+	}
+
+	_get_sorted_result(buf, &newbuf, &len);
+
+resp:
+	rc = send_allgather_resp_to_clients(newbuf, len);
+	if (rc != SLURM_SUCCESS) {
+		slurm_kill_job_step(job_info.jobid, job_info.stepid, SIGKILL);
+	}
+	xfree(newbuf);
+	return rc;
+
+unpack_error:
+	error("mpi/pmi2: unpack error in allgather resp");
+	rc = SLURM_ERROR;
+	goto resp;
+}
+
+/**************************************************************/
+
 /* only called in srun */
 static int
 _handle_spawn(int fd, Buf buf)
diff --git a/src/plugins/mpi/pmi2/tree.h b/src/plugins/mpi/pmi2/tree.h
index ccc254c..80c1df5 100644
--- a/src/plugins/mpi/pmi2/tree.h
+++ b/src/plugins/mpi/pmi2/tree.h
@@ -50,6 +50,8 @@ enum {
 	TREE_CMD_NAME_PUBLISH,
 	TREE_CMD_NAME_UNPUBLISH,
 	TREE_CMD_NAME_LOOKUP,
+	TREE_CMD_ALLGATHER,
+	TREE_CMD_ALLGATHER_RESP,
 	TREE_CMD_RING,
 	TREE_CMD_RING_RESP,
 	TREE_CMD_COUNT
