MPICH2 Guide

From Storrs HPC Wiki
Revision as of 15:01, 26 June 2012 by Stc07008 (talk | contribs) (Created page with "==MPICH2 jobs through LSF== MPICH2 is located in /apps/mpich2/1.4.1p1-ics/ and can be loaded with modules as module load mpi/mpich2/1.4.1p1-ics To automatically load mpich2 ...")
(diff) ← Older revision | Latest revision (diff) | Newer revision → (diff)
Jump to: navigation, search

MPICH2 jobs through LSF

MPICH2 is located in /apps/mpich2/1.4.1p1-ics/ and can be loaded with modules as

module load mpi/mpich2/1.4.1p1-ics

To automatically load mpich2 on login, execute the following:

echo "module load mpi/mpich2/1.4.1p1-ics" >> /home/$USER/.bashrc

Example code 1

#include <iostream>
#include <string>
#include <sys/unistd.h>
#include <sys/socket.h>
#include <netdb.h>
#include "mpi.h"
using namespace std;
void funkywork(){
 int i;
 for(i=0;i<100; i++){
  //sleep(1);
 }
}
int main(int argc, char* argv[]){
    int mytid, numprocs;
    MPI_Status status;
    MPI_Init(&argc,&argv);
    MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
    MPI_Comm_rank(MPI_COMM_WORLD,&mytid);
    char name[100];
    gethostname(name, sizeof(name));
    if(mytid>0){
     funkywork();
    }
    cout << "Hello, " << mytid <<  " and " << name << " say hi in a  C++ statement \n";
    MPI_Finalize();
}

To compile your code using MPICH2 you need to use Intel ICS (for now at least)

module load intelics/2012.0.032

To compile the example code, save it as main.cpp and execute the following

mpicxx -lm -O2 -Wall main.cpp -o mpich2test

To submit this for LSF using MPD, use the following template:

# the name of your job on the queue system
#BSUB -J mpich2_test
# the queue that you will use, the example here use the queue called "normal"
# please use bqueus command to check the available queues
#BSUB -q normal
# the system output and error message output, %J will show as your jobID
#BSUB -o %J.out
#BSUB -e %J.err
# the number of processors that you will use (in this example a 100)
#BSUB -n 100
# when job finish get email notification
# BSUB -u myemail@address.com
# BSUB -N
############ enter your working directory, change to the location of where the executable is ###
work_dir="/home/$USER/TestMPICH2"
cd $work_dir
############ create mpd ring, DO NOT modify this section unless you really know ####
nproc=0
for proc in $LSB_HOSTS ; do
echo $proc >> mpd.procs
nproc=`expr $nproc + 1`
done
echo $LSB_HOSTS
echo $nproc
`sort -u mpd.procs > mpd.nodes`
nhosts=`less mpd.nodes | wc -l`
mpdboot -n $nhosts -v -f mpd.nodes
############ ONLY change the mpich2test to your own application and add necessary arguments (if needed)  #####
mpiexec -machinefile mpd.procs -np $nproc ./mpich2test
############ exit the mpd ring and clean off the nodes ###################
mpdallexit
mpdcleanup
`rm mpd.nodes`
`rm mpd.procs`

Save the template as "mpich2.lsf". You can submit it with the following command:

bsub < mpich2.lsf

Results will be saved to the <LSF_JOB_ID>.out file, any error encountered during the execution will be saved to the <LSF_JOB_ID>.err file in the defined work_dir folder. You can monitor the execution of the job with

bjobs

The output of the example should look something like this

Hello, 90 and cn37 say hi in a  C++ statement 
Hello, 78 and cn34 say hi in a  C++ statement 
Hello, 28 and cn43 say hi in a  C++ statement 

Example code 2

This example illustrates information exchange through MPI between the nodes.

#include <stdio.h>
#include <math.h>
#include <unistd.h>
#include "mpi.h"
#define MAX 1024000
int main (int argc, char *argv[]) {
 char message[MAX] ="Hellooo";
 int count=0;
 char hostname[129];
 int rank, size, i, tag, node, mlen;
 double t0, time, ticks;
 MPI_Status status;
 for (count=7;count<MAX;count++) {
    message[count]='c';
 }
 message[MAX-1]='\0';
 MPI_Init (&argc, &argv);      /* starts MPI */
 MPI_Comm_rank (MPI_COMM_WORLD, &rank);        /* get current process id */
 MPI_Comm_size (MPI_COMM_WORLD, &size);        /* get number of processes */
 tag = 100;
 t0 = 0.0;
 time = 0.0;
 ticks = 0.0;
   ticks = MPI_Wtick();
 mlen = sizeof(message);
 printf("Start---> rank: %d size: %d Clk Resolution is %f sec/tick\n", rank, size, ticks);
 if (gethostname(hostname, 129) < 0)
 {
   printf ("gethostname failed \n");
 }
 else
 {
   printf("running on %s\n",hostname );
 }
 message[mlen] = '\0';
 if (rank == 0)
   {
     for (i = 1; i <   size; i++)
     {
         t0 = MPI_Wtime();
         MPI_Send (message, mlen, MPI_CHAR, i, tag, MPI_COMM_WORLD);
         time = MPI_Wtime() - t0;
         printf ("End--> Snd hostname: %s Length: %d bytes  Snd Time: %.4f sec. i: %d \n",hostname,mlen, time, i);
     }
   }
 else
   {
      printf ("Rcv  rank: %d \n", rank);
      t0 = MPI_Wtime();
      MPI_Recv (message, mlen, MPI_CHAR, 0, tag, MPI_COMM_WORLD, &status);
      time = MPI_Wtime() - t0;
      printf("End--> Rcv hostname: %s Length: %d bytes  Rcv Time: %.4f sec.  node: %d   %.12s\n",hostname, mlen, time, rank, message);
   }
 MPI_Finalize ();
 return(0);
}

Save it as big_message_mpi.c and to compile execute

mpicxx -O2 -Wall big_message_mpi.c -o mpich_big_msg -lm

You can use the template from Example 1 to submit it to LSF. This example exchanges pretty big messages between the nodes so don't spam it. In fact, change the template from Example one so that you only submit it to a handful of nodes (say 2)

#BSUB -n 2

The output of the example should look something like this

cn12 cn12
2
running mpdallexit on cn12
LAUNCHED mpd on cn12  via  
RUNNING: mpd on cn12
Start---> rank: 0 size: 2 Clk Resolution is 0.000001 sec/tick
running on cn12
Start---> rank: 1 size: 2 Clk Resolution is 0.000001 sec/tick
running on cn12
Rcv  rank: 1 
End--> Snd hostname: cn12 Length: 1024000 bytes  Snd Time: 0.0003 sec. i: 1 
End--> Rcv hostname: cn12 Length: 1024000 bytes  Rcv Time: 0.0003 sec.  node: 1   cccccccccccc

Using Hydra

In the above examples we used an MPD template for LSF but there is an alternative and easier way to submit MPI jobs through LSF with the Hydra Process Manager. The following will submit the compiled code from Example 2 to LSF using the Hydra PM:

bsub -n 20 -I mpiexec.hydra -prepend-rank -iface ib0 -rmk lsf ./mpich_big_msg

We asked LSF for 20 processors and specified to use the Infiniband network (ib0) for the communication in interactive mode (-I). More information on running with Hydra can be found here.

MPJ (aka Java MPI)

MPJ v 0.38 is installed. You need to copy it to your home directory so that the logfiles are writable by you. In order to use MPJ, the first step is.

$ cp -rp /apps/mpj/0.38 /home/$USER/.mpj/

Now you want to load the MPJ module every time you log in with:

$ module initadd mpj/0.38

After doing this, you can either log out and log back in again or simply load the module manually (first time only)

$ module load mpj/0.38

A simple example is Foo.java:

import java.net.*;
import java.io.*;

public class Foo {
  public static String getHostName() {
    String hostname;
    try {
      InetAddress addr = InetAddress.getLocalHost();
      byte[] ipAddr = addr.getAddress();
      hostname = addr.getHostName();
    } catch (UnknownHostException e) {
      hostname="undef";
    }
    return(hostname);
  }

  public static void main(String args[]) throws Exception {
     System.out.println("hi the hostname is "+getHostName());
  }
}

Compile your program with:

$ javac -cp .:$MPJ_HOME/lib/mpj.jar Foo.java

Next, you create a bsub file. The following is an example file! You should name it bsub.mpj. This code assumes your working directory is /home/$USER/mpj/ if it does not exist you should create it with: mkdir ~/mpj You will also need to update your email address (line 12), and update the java class name (line 29) which gets run (our example uses Foo).

#the name of your job on the queue system
#BSUB -J MPJ_test
# the queue that you will use, the example here use the queue called "normal"
# please use bqueus command to check the available queues
#BSUB -q normal
# the system output and error message output, %J will show as your jobID
#BSUB -o %J.out
#BSUB -e %J.err
# the number of processors that you will use (in this example a 36)
#BSUB -n 36
# when job finish get email notification
# BSUB -u yourEmail@domain.com
# BSUB -N
############ enter your working directory, change to the location of where the executable (compiled program) is ###
work_dir="/home/$USER/mpj"
cd $work_dir
############ create machines file, DO NOT modify this section unless you really know ####
nproc=0
for proc in $LSB_HOSTS ; do
echo $proc >> mpd.procs
nproc=`expr $nproc + 1`
done
echo $LSB_HOSTS
echo $nproc
`sort -u mpd.procs |uniq >machines`
nhosts=`less machines | wc -l`
mpjboot machines
############  Replace Foo with the name of your class that has main() method ###
mpjrun.sh -np $nproc -dev niodev Foo
############ exit the mpd ring and clean off the nodes ###################
mpjhalt machines
`rm mpd.procs`
`rm machines`

Then, to submit your job just type:

$ bsub < bsub.mpj

Here is a HelloWorld program that utilizes some functions of MPI:

import mpi.*;
public class HelloWorld {
     public static void main(String args[]) throws Exception {
          MPI.Init(args);
          int me = MPI.COMM_WORLD.Rank();
          int size = MPI.COMM_WORLD.Size();
          System.out.println("Hi from <"+me+">");
          MPI.Finalize();
     }
}