Research Article

Performance Optimization and Modeling of Fine-Grained Irregular Communication in UPC

Listing 7

The halo data exchange function of an existing 2D heat equation solver.
#define idx(i, k) ((i) ∗ n + (k))
#define rank(ip, kp) ((ip) ∗ nprocs + (kp))
void halo_exchange_intrinsic()
{
double phi = (double) xphi[MYTHREAD];
int i, k;
 /∗ packing messages for the horizontal direction ∗/
if (kproc > 0) {
  doublephivec_coord1first = (double) xphivec_coord1first[MYTHREAD];
  for (i = 0; i < m − 2; i++)
   phivec_coord1first[i] = phi[idx(i + 1, 1)];
 }
if (kproc < nprocs − 1) {
  doublephivec_coord1last = (double) xphivec_coord1last[MYTHREAD];
  for (i = 0; i < m − 2; i++)
   phivec_coord1last[i] = phi[idx(i + 1, n − 2)];
 }
 upc_barrier;
 /∗ message transfer and unpacking (needed for the horizontal direction) ∗/
if (kproc > 0) {
  upc_memget(halovec_coord1first, xphivec_coord1last[rank(iproc, kproc − 1)], (m − 2) ∗ sizeof(double));
  for (i = 1; i < m − 1; i++)
   phi[idx(i, 0)] = halovec_coord1first[i − 1];
if (kproc > nprocs − 1) {
  upc_memget(halovec_coord1last, xphivec_coord1first[rank(iproc,kproc + 1)], (m − 2) ∗ sizeof(double));
  for (i = 1; i < m − 1; i++)
   phi[idx(i, n − 1)] = halovec_coord1last[i − 1];
 }
if (iproc > 0)
  upc_memget(&phi[idx(0, 1)], &(xphi[rank(iproc − 1, kproc)][idx(m − 2, 1)]), (n − 2) ∗ sizeof(double));
if (iproc < mprocs − 1)
  upc_memget(&phi[idx(m − 1, 1)], &(xphi[rank(iproc + 1, kproc)][idx(1, 1)]), (n − 2) ∗ sizeof(double));
}