From c6cafed31e53092971744899fa60fca1fd3c5877 Mon Sep 17 00:00:00 2001
From: ignis <ignis@postech.ac.kr>
Date: Fri, 14 Jun 2019 17:42:56 +0900
Subject: [PATCH] load balanced main subroutine, processor cumulative
 performance log, only master print finish message, clean trailing w/s

---
 code/post.f90 | 162 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 153 insertions(+), 9 deletions(-)

diff --git a/code/post.f90 b/code/post.f90
index 395e5a8..df842a3 100644
--- a/code/post.f90
+++ b/code/post.f90
@@ -8,6 +8,7 @@
      IMPLICIT NONE
      PRIVATE
      INTEGER :: countnum
+     INTEGER :: nprogress
      INTEGER :: num_, dummyu_   ! hybrid
      REAL :: tnow
 
@@ -15,7 +16,7 @@
 
      PUBLIC :: main
     CONTAINS
- 
+
      SUBROUTINE main
       INTEGER :: fread,i
 
@@ -46,11 +47,11 @@
           CALL READ_FILE(fread)
 
           CALL m_terms_calculate_pass1
-         
+
          END IF
 
         ENDIF
-          
+
       ENDDO firstloop
 
       write(*,*) '1st loop finished'
@@ -84,17 +85,157 @@
       ENDDO secondloop
 
       CALL m_terms_average_pass2(countnum)
-      
-      if (iammaster) CALL m_terms_write_result
+
+      if (iammaster) then
+          CALL m_terms_write_result
+          WRITE(*,*) ' Avergaing RAW data is FINISHED'
+          WRITE(*,*)  'qEdge_X.dat is generated'
+      end if
 
       CALL DEALLOCATES_CLOSE
 
 
-      WRITE(*,*) ' Avergaing RAW data is FINISHED'
-      WRITE(*,*)  'qEdge_X.dat is generated' 
 
      END SUBROUTINE main
 
+
+
+     SUBROUTINE main_lb
+      INTEGER :: fread,i
+
+      CALL READ_INTRO
+
+      CALL ALLOCATE_ARRAYS
+
+      if (iammaster) CALL PRINT_BANNER
+
+
+    ! PASS 1
+
+      tag = 1
+      countnum = 0
+      nprogress = 0
+
+      if (iammaster) then
+
+          dispatchloop1: DO fread=startnum,endnum,skipnum
+
+            IF ( to_omit(fread) ) THEN
+
+                WRITE(*,'(a47,i7,a4,i5,a3,i5)') &
+                     ' Current fullsavenum = ', fread, ' || ', (fread-startnum+1), ' / ', (endnum-startnum+1)
+
+                WRITE(*,'(a12,i6,a20,i6)') &
+                     '      Skip. ', omit_t(i,1), ' <= fullsavenum <= ', omit_t(i,2)
+
+            ELSE
+
+                countnum=countnum+1
+
+                CALL MPI_RECV(id_to, 1, MPI_INTEGER, MPI_ANY_SOURCE, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+                CALL MPI_SEND(fread, 1, MPI_INTEGER, id_to, tag, MPI_COMM_TASK, mpi_err)
+
+            ENDIF
+
+          ENDDO dispatchloop1
+
+          end_of_dispatch1: DO i=1,numprocs-1
+
+              CALL MPI_RECV(id_to, 1, MPI_INTEGER, MPI_ANY_SOURCE, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+              CALL MPI_SEND(-1,    1, MPI_INTEGER, id_to,          tag, MPI_COMM_TASK, mpi_err)
+
+          ENDDO end_of_dispatch1
+
+      else ! slave
+
+          fetchloop1: do
+
+              CALL MPI_SEND(myid,  1, MPI_INTEGER, master, tag, MPI_COMM_TASK, mpi_err)
+              CALL MPI_RECV(fread, 1, MPI_INTEGER, master, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+
+              if (fread < 0) exit
+
+              nprogress = nprogress + 1
+
+              CALL READ_FILE(fread)
+
+              CALL m_terms_calculate_pass1
+
+          end do fetchloop1
+
+      end if
+
+      CALL MPI_BCAST(countnum, 1, MPI_INTEGER, master, MPI_COMM_TASK, mpi_err)
+
+      CALL m_terms_average_pass1(countnum)
+
+      if (iammaster) write(*,*) '1st loop finished'
+
+    ! PASS 2
+
+      tag = 2
+      nprogress = 0
+
+      if (iammaster) then
+
+          dispatchloop2: DO fread=startnum,endnum,skipnum
+
+              IF ( to_omit(fread) ) THEN
+
+                  WRITE(*,'(a40,i7,a4,i4,a3,i4)') &
+                     ' Current fullsavenum = ', fread, ' || ', (fread-startnum+1), ' / ', (endnum-startnum+1)
+
+                  WRITE(*,'(a12,i6,a20,i6)') &
+                     '     Skip. ', omit_t(i,1), ' <= fullsavenum <= ', omit_t(i,2)
+
+              ELSE
+
+                  CALL MPI_RECV(id_to, 1, MPI_INTEGER, MPI_ANY_SOURCE, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+                  CALL MPI_SEND(fread, 1, MPI_INTEGER, id_to,          tag, MPI_COMM_TASK, mpi_err)
+
+              ENDIF
+
+          ENDDO dispatchloop2
+
+          end_of_dispatch2: DO i=1,numprocs-1
+
+              CALL MPI_RECV(id_to, 1, MPI_INTEGER, MPI_ANY_SOURCE, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+              CALL MPI_SEND(-1,    1, MPI_INTEGER, id_to,          tag, MPI_COMM_TASK, mpi_err)
+
+          ENDDO end_of_dispatch2
+
+      else ! slave
+
+          fetchloop2: do
+
+              CALL MPI_SEND(myid,  1, MPI_INTEGER, master, tag, MPI_COMM_TASK, mpi_err)
+              CALL MPI_RECV(fread, 1, MPI_INTEGER, master, tag, MPI_COMM_TASK, mpi_status, mpi_err)
+
+              if (fread < 0) exit
+
+              nprogress = nprogress + 1
+
+              CALL READ_FILE(fread)
+
+              CALL m_terms_calculate_pass2
+
+          end do fetchloop2
+
+      end if
+
+      CALL m_terms_average_pass2(countnum)
+
+      if (iammaster) then
+          CALL m_terms_write_result
+          WRITE(*,*) ' Avergaing RAW data is FINISHED'
+          WRITE(*,*)  'qEdge_X.dat is generated'
+      end if
+
+      CALL DEALLOCATES_CLOSE
+
+
+     END SUBROUTINE main_lb
+
 !========================================================================================
 ! End of main routine
 !========================================================================================
@@ -131,8 +272,11 @@
 
       WRITE(*,'(a4,i3,a40,f8.3,a2,i7,a2,i5,a4,i5,a3,i5)') ' p ', myid, ': time / NCYC / fullsavenum = ',tnow,&
                 ' /',ncyc,' /',num,' || ',(num-startnum+1),' / ',(endnum-startnum+1)
+
       WRITE(*,*) '    Reading current data file and processing'
 
+      WRITE(*,'(a10,i3,a10,i5,a10)') 'processor ', myid, ' processed ', nprogress, ' files'
+
       num_=num
 
       IF(num.le.shiftnum) THEN
@@ -180,11 +324,11 @@
 
       DEALLOCATE(old_scalar)
       DEALLOCATE(new_scalar)
-      
+
       IF(omitnum.gt.0)   DEALLOCATE(omit_t)
 
       DEALLOCATE(file_dist)
 
      END SUBROUTINE DEALLOCATES_CLOSE
 
-    END MODULE post   
+    END MODULE post