Gatherv

IMPI_Int4 gatherv_is_short(int *count, MPI_Datatype dtype, 
                           int root, MPI_Comm comm)
{
  IMPI_Int4   maxsize;
  int         myrank, nprocs, size;
  int         nprocs, size;

  MPI_Comm_rank(comm, &myrank);
  MPI_Comm_size(comm, &nprocs);
  MPI_Pack_size(1, dtype, comm, &size);

  if (myrank == root) {
    maxsize = count[0] * size;
    for (i = 1; i < nprocs; i++) {
      if (count[i] * size > maxsize) {
        maxsize = count[i] * size;
    }

    if (maxsize > 2048) {
      maxsize = 0;
    }
  }

  MPI_Bcast(&maxsize, 1, IMPI_INT4, root, comm);
  return(maxsize);
}

int MPI_Gatherv(void *sbuf, int scount, MPI_Datatype sdtype, 
                void *rbuf, int *rcounts, int *disps, MPI_Datatype rdtype, 
                int root, MPI_Comm comm)
{
  IMPI_Int4 maxsize;

  maxsize = gatherv_is_short(rcounts, rdtype, root, comm);
  if (maxsize) {
    gatherv_short(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, 
                  root, comm, maxsize);
  } else {
    gatherv_long(sbuf, scount, sdtype, rbuf, rcounts, disps, rdtype, 
                 root, comm);
  }

  return(MPI_SUCCESS);
}

int gatherv_short(void *sbuf, int scount, MPI_Datatype sdtype, 
                  void *rbuf, int *rcounts, int *disps, MPI_Datatype rdtype, 
                  int root, MPI_Comm comm, IMPI_Int4 maxsize)
{
  MPI_Status status;
  int myrank, nmasters, packsize, vnum, rootnum, nmasters;
  int mask, nprocs, count, size;

  MPI_Comm_rank(comm, &myrank);
  MPI_Comm_size(comm, &nprocs);

  if (is_master(myrank, comm)) {
    allocate a temporary buffer tmpbuf of size nprocs*maxsize;
  }

  nmasters = num_masters(comm);

  /* local phase */
  if (are_local(myrank, root, comm)) {
    gather the send buffers of the local processes into the
      root's receive buffer;
  } else {
    gather send buffers at the local master into tmpbuf;
    on local master set size equal to the # of bytes gathered in tmpbuf;
    /* At this point the master must have a buffer tmpbuf
     * containing a concatenation in rank order of the
     * local processes packed send buffers.
     */
  }

  /* global phase */
  if ((myrank == root) || (is_master(myrank, comm) 
      && !are_local(myrank, root, comm))) {

    if (nmasters <= MAXLINEARGATHER) {
      /* linear gather to root */
      if (myrank == root) {
        for (i = 0, size = 0; i < nmasters; i++) {
          if (i == local_master_num(root, comm)) {
            continue;            /* skip root's node */
          }

          MPI_Recv(tmpbuf+size, nprocs*maxsize, MPI_BYTE, 
                   master_rank(i, comm), IMPI_GATHERV_TAG, comm, &status);
          MPI_Get_count(status, MPI_BYTE, &count);
          size += count;
      } else {
        MPI_Send(tmpbuf, size, MPI_BYTE, root, IMPI_GATHERV_TAG, comm); 
      }
    } else {
      /* tree gather to root */
      mynum = local_master_num(myrank, comm);
      rootnum = master_num(root, comm);
      vnum = (mynum - rootnum + nmasters) % nmasters;

      if (myrank == root) {
          size = 0;
      }

      for (mask = 1; mask < nprocs; mask <<= 1) {
        if (vnum & mask) {
          peer = master_rank(((vnum & ~mask) + rootnum) % nmasters, comm);
          if (are_local(peer, root, comm)) {
            peer = root;
          }

          MPI_Send(tmpbuf, size, MPI_BYTE, peer, IMPI_GATHERV_TAG, comm);
          break;
        }
        else {
          peer = vnum | mask;
          if (peer >= nmasters) continue;
          peer = master_rank((peer + rootnum) % nmasters, comm);
          if (are_local(peer, root, comm)) {
            peer = root;
          }

          MPI_Recv(tmpbuf+size, nprocs*maxsize, MPI_BYTE, peer, 
                   IMPI_GATHERV_TAG, comm, &status);
          MPI_Get_count(status, MPI_BYTE, &count);
          size += count;
        }
      }
    }
  }

  /* local phase */
  if (myrank == root) {
    /* tmpbuf contains concatenated in order of master rank the 
     * concatenations of the process send buffers created in the first
     * local phase
     */
    unpack the data in tmpbuf into the receive buffer;
  }

  if (is_master(myrank, comm)) {
    free(tmpbuf);
  }
  return(MPI_SUCCESS);
}

int gatherv_long(void *sbuf, int scount, MPI_Datatype sdtype, 
                 void *rbuf, int *rcounts, int *disps, MPI_Datatype rdtype, 
                 int root, MPI_Comm comm)
{
  MPI_Status status;
  MPI_Aint extent;
  int i, myrank, nprocs;
  char *p;

  MPI_Comm_rank(comm, &myrank);
  MPI_Comm_size(comm, &nprocs);

  if (myrank != root) {
    MPI_Send(sbuf, scount, sdtype, root, IMPI_GATHER_TAG, comm);
  }

  MPI_Type_extent(rdtype, &extent);

  for (i = 0; i < nprocs; i++) {

    p = ((char *) rbuf) + (extent * disps[i]);

    if (i == myrank) {
      MPI_Sendrecv(sbuf, scount, sdtype, i, IMPI_GATHERV_TAG,
                   p, rcounts[i], rdtype, i, IMPI_GATHERV_TAG,
									 comm, &status);
    } else {
      MPI_Recv(p, rcounts[i], rdtype, i, IMPI_GATHERV_TAG, comm, &status);
    }
  }

  return(MPI_SUCCESS);
}