// matmul_pthread4.c  full featured, not all used
// threadID 0 for main, 1, 2, 3, 4 same for SlaveID   
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <time.h>

#define DIM 2048
static  double a[DIM][DIM];
static  double b[DIM][DIM];
static  double c[DIM][DIM];

#define nthread  5           // includes main
static int results[nthread];
static pthread_mutex_t ready[nthread]; // allows slave to proceed  
static pthread_mutex_t done [nthread]; // allows master to proceed 
static pthread_mutex_t ack  [nthread]; // acknowlege proceeding    

void master_init(int slaveID)
{
  int t = slaveID;
  pthread_mutex_init(&ready[t], NULL); // holds up slave 
  pthread_mutex_lock(&ready[t]);
  pthread_mutex_init(&done[t], NULL);  // holds up master 
  pthread_mutex_unlock(&done[t]);
  pthread_mutex_init(&ack[t], NULL);   // holds up slave 
  pthread_mutex_unlock(&ack[t]);
} // end master_init 

void master_release(int slaveID)
{
  int t = slaveID;
  pthread_mutex_lock(&ack[t]);     // for hold after ready 
  pthread_mutex_unlock(&ready[t]); // let slaves run 
} // end master_release 

void master_barrier(int slaveID)
{
  int t = slaveID;
  pthread_mutex_lock(&done[t]);    // waits for slave to be done 
  pthread_mutex_lock(&ready[t]);   // hold slaves until everybody done  
  pthread_mutex_unlock(&done[t]);  // ready for slaves next task 
  pthread_mutex_unlock(&ack[t]);   // ready for slaves next task 
} // end master_barrier 

void slave_init(int slaveID)
{
  int i = slaveID;
  pthread_mutex_lock(&done[i]);    // no wait, master keeps unlocked 
  pthread_mutex_lock(&ready[i]);   // wait for master to release 
  pthread_mutex_unlock(&ready[i]); // unlock 
} // end slave_init 

void slave_task_done(int slaveID)
{
  int i = slaveID;
  pthread_mutex_unlock(&done[i]); // this task done 
} // end slave_task_done 

void slave_barrier(int slaveID)
{
  int i = slaveID;
  pthread_mutex_unlock(&done[i]);  // master can test for done 
  pthread_mutex_lock(&ack[i]);     // wait for master ack 
  pthread_mutex_unlock(&ack[i]);   // unlock 
  pthread_mutex_lock(&ready[i]);   // waiting for master to release 
  pthread_mutex_unlock(&ready[i]); // unlock 
  pthread_mutex_lock(&done[i]);    // no wait, master keeps unlocked 
} // end slave barrier 

void * slave_code(void * threadID)
{
  int part; // slaves 1,2,3,4  main is nthread 0 (unused here)
  double sum;
  int i, j, k, i1, i2;


  part = (int)(long int)threadID;
  if(part==0)
  {
    slave_init(part);
    slave_task_done(part);
    pthread_exit(NULL); // done 
  } // nothing for main

  printf("   thread %d started, waiting \n", part);
  fflush(stdout);
  slave_init(part);

  printf("   thread %d computing \n", part);
  fflush(stdout);

  i1 = (part-1)*DIM/(nthread-1);
  i2 = (part)*DIM/(nthread-1);
  printf("   thread %d  i1=%d  i2=%d \n", part, i1, i2);
  fflush(stdout);

  for(i=i1; i<i2; i++)
  {
    for(j=0; j<DIM; j++)
    {
      sum=0.0;
      for(k=0; k<DIM; k++)
      {
        sum += a[i][k]*b[k][j];
      } // end k
      c[i][j]=sum;
    } // end j
  } // end i

  printf("   thread %d finished computing \n", part);
  fflush(stdout);

  slave_task_done(part);

  printf("   thread %d, done, exiting \n", part);
  fflush(stdout);
  pthread_exit(NULL); // done 
} // end slave_code 

int main(int argc, char * argv[])
{
  int rc, t;
  double wallstart, wallnow;
  double cpustart, cpunow;
  pthread_t threads[nthread];
  int i, j, k;

  printf("matmul_pthread4.c running, DIM = %d \n", DIM);

  for(i=0; i<DIM; i++)
  {
    for(j=0; j<DIM; j++)
    {
      a[i][j] = (double)(i+j);
      b[i][j] = (double)(i+j+1);
    }
  }

  printf("matmul_pthread4.c initialized \n");
  wallstart = (double)time(NULL);
  cpustart = (double)clock()/(double)CLOCKS_PER_SEC;

  for(t=0; t<nthread; t++)
  {
    printf("creating thread %d \n", t);
    results[t] = 0;
    master_init(t);
    rc = pthread_create(&threads[t], NULL, slave_code, (void *)(long int)t);
    if(rc)
    {
      printf("error, thread not created, code=%d, at t=%d \n", rc, t);
      exit(-1);
    }
  }
  fflush(stdout);
  printf("let all slaves run\n");
  fflush(stdout);
  // let threads run 
  for(t=0; t<nthread; t++) // start all threads 
  {
    master_release(t);
  }

  printf("waiting for threads to be done computing \n");
  fflush(stdout);
  for(t=0; t<nthread; t++) // wait for all threads to continue 
  {
    master_barrier(t);
    printf("thread %d done. \n", t);
    fflush(stdout);
  }

  printf("all threads finished  \n");
  fflush(stdout);

  printf("c[1][1]= %e, c[2][2]= %e \n",
	 c[1][1], c[2][2]);
  printf("c[513][513]= %e, c[514][514]= %e \n",
	 c[513][513], c[514][514]);
  printf("c[1025][1025]= %e, c[1026][1026]= %e \n",
	 c[1025][1025], c[1026][1026]);
  printf("c[1538][1538]= %e, c[1539][1539]= %e \n",
	 c[1538][1538], c[1539][1539]);

  wallnow = (double)time(NULL);
  cpunow = (double)clock()/(double)CLOCKS_PER_SEC;
  printf("master exiting, wall time = %f seconds. \n", wallnow-wallstart);
  fflush(stdout);
  printf("cpu time = %f seconds \n", cpunow-cpustart);
  printf("average number of parallel threads %f\n",
	 (cpunow-cpustart)/(wallnow-wallstart));
  fflush(stdout);
  return 0;
} // end matmul_pthread4.c