diff --git a/Makefile b/Makefile index a636b29..4f7f142 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,34 @@ MPIF90 = blah FFTW_HOME = $(FFTW_DIR) MPIF90 = mpif90 FCFLAGS = -O3 -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include - LDFLAGS = -O3 -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm + LDFLAGS = -O3 -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPIPP_LD_FLAGS) $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm FCFLAGS_F77 = MODULES += m_openmpi.o + +# compile for mpiP MPI Profiling +ifeq ($(PROFILER), mpiP) + FFTW_HOME = $(FFTW_DIR) + MPIF90 = mpif90 + FCFLAGS = -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include + MPIP_LD_FLAGS = -L$(MPIP_DIR)/lib -lmpiP -lm -lbfd -liberty -lunwind + LDFLAGS = -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPIPP_LD_FLAGS) $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm + FCFLAGS_F77 = + +endif + +# tau source instrumentation +ifeq ($(PROFILER), tau_source) + FFTW_HOME = $(FFTW_DIR) + MPIF90 = tau_f90.sh + FCFLAGS = -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include + LDFLAGS = -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm + FCFLAGS_F77 = + +endif + + # Program name PROG = hit3d.x diff --git a/m_fdm_calc.f90 b/m_fdm_calc.f90 index 0ce9382..0cce4b4 100644 --- a/m_fdm_calc.f90 +++ b/m_fdm_calc.f90 @@ -25,6 +25,8 @@ module m_fdm_calc real*8 :: umax,umin,vmax,vmin,wmax,wmin ! J. Kwon real*8, dimension(3) :: velmax, velmin, velmax1, velmin1 + integer, parameter :: neq = 2 + !=========================================================================== !=========================================================================== @@ -234,9 +236,9 @@ module m_fdm_calc fdmtime=0. fullsavenum=1000 !full save file - allocate(y1(2,nx,ny,nz)) - allocate(y2(2,nx,ny,nz)) - allocate(yf(2,nx,ny,nz)) + allocate(y1(nx,ny,nz,neq)) + allocate(y2(nx,ny,nz,neq)) + allocate(yf(nx,ny,nz,neq)) allocate(fz(4*nx*ny,nz)) allocate(dfz(4*nx*ny,nz)) @@ -278,8 +280,8 @@ module m_fdm_calc do j=1,nz do i=1,ny do ii=1,nx - y1(1,ii,i,j)=1. ! rho initializing - y1(2,ii,i,j)=yy(ii) ! Yr initializing + y1(ii,i,j,1)=1. ! rho initializing + y1(ii,i,j,2)=yy(ii) ! Yr initializing enddo enddo enddo @@ -290,7 +292,7 @@ module m_fdm_calc DO ii=1,nz DO j=1,ny DO i=1,nx - yr=y1(2,i,j,ii)/y1(1,i,j,ii) + yr=y1(i,j,ii,2)/y1(i,j,ii,1) c=1.-yr IF (c.lt.0.) c=0. wrate=pre*yr*exp(-ac/(1.+bc*c)) @@ -404,7 +406,7 @@ module m_fdm_calc do k=1,nz do j=1,ny do i=1,nx - yr=y1(2,i,j,k)/y1(1,i,j,k) + yr=y1(i,j,k,2)/y1(i,j,k,1) wrate=pre*yr*exp(-ac/(1.+bc*(1.-yr))) IF((1.-yr).le.c_ref) THEN wrate=min_wr @@ -469,7 +471,7 @@ module m_fdm_calc integer :: n integer :: idx1, idx2 real*8 :: wrate,yr,yp - real*8 :: r1_(2,xx,yy,zz),f_(2,xx,yy,zz) + real*8 :: r1_(xx,yy,zz,neq),f_(xx,yy,zz,neq) real*8 :: uu_(xx,yy,zz),vv_(xx,yy,zz),ww_(xx,yy,zz) real*8 :: ux(4,xx),dux(4,xx),d2ux(xx) real*8 :: uy(4,yy),duy(4,yy),d2uy(yy) @@ -485,7 +487,7 @@ module m_fdm_calc DO j=1,yy DO i=1,xx - y=r1_(2,i,j,k)/r1_(1,i,j,k) ! 2:Y + y=r1_(i,j,k,2)/r1_(i,j,k,1) ! 2:Y wrate=pre*y*exp(-ac/(1.+bc*(1.-y))) !wrate IF ((1.-y).le.c_ref) THEN @@ -494,8 +496,8 @@ module m_fdm_calc min_wr-refwr*exp(prof_wr*(c_cut-c_ref)))/(1.-exp(prof_wr*(c_cut-c_ref))) ENDIF - f_(1,i,j,k) = 0.0 ! continuity - f_(2,i,j,k) = - wrate ! species conservation + f_(i,j,k,1) = 0.0 ! continuity + f_(i,j,k,2) = - wrate ! species conservation ENDDO ENDDO @@ -508,8 +510,8 @@ module m_fdm_calc DO j=1,yy DO i=1,xx DO k=1,zz - uz(1,k)=r1_(1,i,j,k) ! 1:rho - uz(2,k)=r1_(2,i,j,k)/r1_(1,i,j,k) ! 2:Y + uz(1,k)=r1_(i,j,k,1) ! 1:rho + uz(2,k)=r1_(i,j,k,2)/r1_(i,j,k,1) ! 2:Y uz(3,k)=uz(1,k)*ww_(i,j,k) ! 3:rho*w uz(4,k)=uz(3,k)*uz(2,k) ! 4:rho*w*Y uuz (k)=uz(2,k) @@ -520,12 +522,12 @@ module m_fdm_calc DO k=1,zz ! -( d(rho*w)/dz ) - f_(1,i,j,k) = f_(1,i,j,k) - duz(3,k) ! continuity + f_(i,j,k,1) = f_(i,j,k,1) - duz(3,k) ! continuity ! -( d(rho*w*Yr)/dz ) + d(rho*D* d(Yr)/dz)/dz ! = -( d(rho*w*Yr)/dz ) ! + D* (rho* d2(Yr)/dz2 + d(rho)/dz * d(Yr)/dz ) - f_(2,i,j,k) = f_(2,i,j,k) - duz(4,k) + diff*(uz(1,k)*d2uz(k)+duz(1,k)*duz(2,k)) ! species conserv. + f_(i,j,k,2) = f_(i,j,k,2) - duz(4,k) + diff*(uz(1,k)*d2uz(k)+duz(1,k)*duz(2,k)) ! species conserv. ENDDO ENDDO ENDDO @@ -537,11 +539,13 @@ module m_fdm_calc DO i=1,xx idx2 = xx*(j-1)+i idx1 = (idx2-1)*4 - fz(idx1+1,k) = r1_(1,i,j,k) ! 1:rho - fz(idx1+2,k) = r1_(2,i,j,k)/r1_(1,i,j,k) ! 2:Y - fz(idx1+3,k) = r1_(1,i,j,k)*ww_(i,j,k) ! 3:rho*w - fz(idx1+4,k) = r1_(2,i,j,k)*ww_(i,j,k) ! 4:rho*w*Y - fzz(idx2,k) = r1_(2,i,j,k)/r1_(1,i,j,k) + fz(idx1+1,k) = r1_(i,j,k,1) ! 1:rho + fz(idx1+2,k) = r1_(i,j,k,2)/r1_(i,j,k,1) ! 2:Y + fz(idx1+3,k) = r1_(i,j,k,1)*ww_(i,j,k) ! 3:rho*w + fz(idx1+4,k) = r1_(i,j,k,2)*ww_(i,j,k) ! 4:rho*w*Y + fzz(idx2,k) = r1_(i,j,k,2)/r1_(i,j,k,1) + !fz(idx1+4,k) = r1_(2,i,j,k)*ww_(i,j,k) ! :rho*w*YO + !fzz(idx2,k) = r1_(2,i,j,k)/r1_(1,i,j,k) ! :YO ENDDO ENDDO ENDDO @@ -555,12 +559,12 @@ module m_fdm_calc idx2 = xx*(j-1)+i idx1 = (idx2-1)*4 ! -( d(rho*w)/dz ) - f_(1,i,j,k) = f_(1,i,j,k) - dfz(idx1+3,k) ! continuity + f_(i,j,k,1) = f_(i,j,k,1) - dfz(idx1+3,k) ! continuity ! -( d(rho*w*Yr)/dz ) + d(rho*D* d(Yr)/dz)/dz ! = -( d(rho*w*Yr)/dz ) ! + D* (rho* d2(Yr)/dz2 + d(rho)/dz * d(Yr)/dz ) - f_(2,i,j,k) = f_(2,i,j,k) - dfz(idx1+4,k) + diff*(fz(idx1+1,k)*dfzz(idx2,k)+dfz(idx1+1,k)*dfz(idx1+2,k)) ! species conserv. + f_(i,j,k,2) = f_(i,j,k,2) - dfz(idx1+4,k) + diff*(fz(idx1+1,k)*dfzz(idx2,k)+dfz(idx1+1,k)*dfz(idx1+2,k)) ! species conserv. ENDDO ENDDO @@ -572,8 +576,8 @@ module m_fdm_calc DO k=1,zz DO j=1,yy DO i=1,xx - ux(1,i)=r1_(1,i,j,k) ! 1:rho - ux(2,i)=r1_(2,i,j,k)/r1_(1,i,j,k) ! 2:Y + ux(1,i)=r1_(i,j,k,1) ! 1:rho + ux(2,i)=r1_(i,j,k,2)/r1_(i,j,k,1) ! 2:Y ux(3,i)=ux(1,i)*uu_(i,j,k) ! 3:rho*u ux(4,i)=ux(3,i)*ux(2,i) ! 4:rho*u*Y uux (i)=ux(2,i) @@ -585,12 +589,12 @@ module m_fdm_calc DO i=1,xx ! -( d(rho*u)/dx ) - f_(1,i,j,k) = f_(1,i,j,k) - dux(3,i) ! continuity + f_(i,j,k,1) = f_(i,j,k,1) - dux(3,i) ! continuity ! -( d(rho*u*Yr)/dx ) + d(rho*D* d(Yr)/dx)/dx ! = -( d(rho*u*Yr)/dx ) ! + D* (rho* d2(Yr)/dx2 + d(rho)/dx * d(Yr)/dx ) - f_(2,i,j,k) = f_(2,i,j,k) - dux(4,i) + diff*(ux(1,i)*d2ux(i)+dux(1,i)*dux(2,i)) ! species conservation + f_(i,j,k,2) = f_(i,j,k,2) - dux(4,i) + diff*(ux(1,i)*d2ux(i)+dux(1,i)*dux(2,i)) ! species conservation ENDDO ENDDO @@ -600,8 +604,8 @@ module m_fdm_calc DO k=1,zz DO i=1,xx DO j=1,yy - uy(1,j)=r1_(1,i,j,k) ! 1:rho - uy(2,j)=r1_(2,i,j,k)/r1_(1,i,j,k) ! 2:Y + uy(1,j)=r1_(i,j,k,1) ! 1:rho + uy(2,j)=r1_(i,j,k,2)/r1_(i,j,k,1) ! 2:Y uy(3,j)=uy(1,j)*vv_(i,j,k) ! 3:rho*v uy(4,j)=uy(3,j)*uy(2,j) ! 4:rho*v*Y uuy (j)=uy(2,j) @@ -612,12 +616,12 @@ module m_fdm_calc DO j=1,yy ! -( d(rho*v)/dy ) - f_(1,i,j,k)=f_(1,i,j,k)-duy(3,j) ! continuity + f_(i,j,k,1)=f_(i,j,k,1)-duy(3,j) ! continuity ! -( d(rho*v*Yr)/dy ) + d(rho*D* d(Yr)/dy)/dy ! = -( d(rho*v*Yr)/dy ) ! + D* (rho* d2(Yr)/dyy2 + d(rho)/dy * d(Yr)/dy ) - f_(2,i,j,k)=f_(2,i,j,k)-duy(4,j) + diff*(uy(1,j)*d2uy(j)+duy(1,j)*duy(2,j)) ! species conserv. + f_(i,j,k,2)=f_(i,j,k,2)-duy(4,j) + diff*(uy(1,j)*d2uy(j)+duy(1,j)*duy(2,j)) ! species conserv. ENDDO ENDDO ENDDO @@ -627,9 +631,9 @@ module m_fdm_calc DO k=1,zz DO j=1,yy DO i=1,yrsw - f_(2,i,j,k)=r1_(1,i,j,k)*0.+f_(1,i,j,k)*in_yr + f_(i,j,k,2)=r1_(i,j,k,1)*0.+f_(i,j,k,1)*in_yr ENDDO - IF (uu_(xx,j,k).lt.0.) f_(2,xx,j,k)=f_(1,xx,j,k)*r1_(2,xx,j,k)/r1_(1,xx,j,k) + IF (uu_(xx,j,k).lt.0.) f_(xx,j,k,2)=f_(xx,j,k,1)*r1_(xx,j,k,2)/r1_(xx,j,k,1) ENDDO ENDDO @@ -642,7 +646,7 @@ module m_fdm_calc integer :: istage,xx,yy,zz,i real*8 :: uu_(xx,yy,zz),vv_(xx,yy,zz),ww_(xx,yy,zz) - real*8 :: yy1(2,xx,yy,zz),yy2(2,xx,yy,zz),yyf(2,xx,yy,zz) + real*8 :: yy1(xx,yy,zz,neq),yy2(xx,yy,zz,neq),yyf(xx,yy,zz,neq) istage=1; CALL substep(yy1,yy1,yy2,yyf,xx,yy,zz,istage,uu_,vv_,ww_) @@ -660,7 +664,7 @@ module m_fdm_calc integer :: i,j,k,xx,yy,zz real*8 :: uu_(xx,yy,zz),vv_(xx,yy,zz),ww_(xx,yy,zz) - real*8 :: yy1(2,xx,yy,zz),yy2(2,xx,yy,zz),yyf(2,xx,yy,zz) + real*8 :: yy1(xx,yy,zz,neq),yy2(xx,yy,zz,neq),yyf(xx,yy,zz,neq) CALL RK4(xx,yy,zz,uu_,vv_,ww_,yy1,yy2,yyf) @@ -674,7 +678,7 @@ module m_fdm_calc integer :: i,j,k,xx,yy,zz,istage real*8 :: at,bt , wrate , yr - real*8 :: ri(2,xx,yy,zz),r1(2,xx,yy,zz),r2(2,xx,yy,zz),f(2,xx,yy,zz) + real*8 :: ri(xx,yy,zz,neq),r1(xx,yy,zz,neq),r2(xx,yy,zz,neq),f(xx,yy,zz,neq) real*8 :: a(5),b(5) real*8 :: uu_(xx,yy,zz),vv_(xx,yy,zz),ww_(xx,yy,zz) integer :: nfinal, iscr, mspec, mpict, msave, nmindt, nv @@ -699,9 +703,9 @@ module m_fdm_calc DO k=1,zz DO j=1,yy DO i=1,xx - DO nv=1,2 - r1(nv,i,j,k)=r1(nv,i,j,k)+at*f(nv,i,j,k) - r2(nv,i,j,k)=r1(nv,i,j,k)+bt*f(nv,i,j,k) + DO nv=1,neq + r1(i,j,k,nv)=r1(i,j,k,nv)+at*f(i,j,k,nv) + r2(i,j,k,nv)=r1(i,j,k,nv)+bt*f(i,j,k,nv) ENDDO ENDDO ENDDO @@ -711,15 +715,15 @@ module m_fdm_calc DO k=1,zz DO j=1,yy DO i=1,xx - DO nv=1,2 - r1(nv,i,j,k)=r1(nv,i,j,k)+bt*f(nv,i,j,k) + DO nv=1,neq + r1(i,j,k,nv)=r1(i,j,k,nv)+bt*f(i,j,k,nv) ENDDO !==========rho=1 treatment - r1(2,i,j,k)=r1(2,i,j,k)/r1(1,i,j,k) - r1(1,i,j,k)=1. + r1(i,j,k,2)=r1(i,j,k,2)/r1(i,j,k,1) + r1(i,j,k,1)=1. !==========Max Yr=1 treatment - r1(2,i,j,k)=MIN(in_yr,r1(2,i,j,k)) + r1(i,j,k,2)=MIN(in_yr,r1(i,j,k,2)) !==========Min Yr=0 treatment ! r1(2,i,j,k)=MAX(out_yr,r1(2,i,j,k))