research/dns-hit3d-fdm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

merge into: research:mpi_fdm
Branches Tags
research:mpi_fdm
research:two-step
research:more-iterations
research:diffusivity
research:test_branch
research:pipeline-compact
research:ppc
...
pull from: research:pipeline-compact
Branches Tags
research:two-step
research:more-iterations
research:diffusivity
research:test_branch
research:pipeline-compact
research:ppc
research:mpi_fdm
Sign in to create a new pull request.

27 commits
mpi_fdm ... pipeline-c

Author SHA1 Message Date
ignis
e16ead959d Adams-Bashforth method for reacting scalar time advancement 2018-01-21 06:24:41 +09:00
ignis
f5b0550436 pipeline td solve, distinct tag for each rhs 2017-12-24 20:42:30 +09:00
ignis
4a8a67d241 fixed bug in ialias array construction 2017-12-24 20:19:22 +09:00
ignis
db7732025a remove unnecessary lines in gather_4 and added comments in main 2017-12-24 20:18:40 +09:00
ignis
a56c4d978c remove loops in parallel z-dir fns 2017-11-11 00:43:27 +09:00
ignis
e659052e26 z-dir rhs exchange mpi subroutine change, rk substep advance opt 2017-11-10 23:39:52 +09:00
ignis
b426be89ad transpose subroutine convention change and minor mods 2017-11-10 16:37:35 +09:00
ignis
749e77de32 corse grained and term-wise derivative computation 2017-11-09 03:42:30 +09:00
ignis
6933297130 remove unnecessary ifs and optimize parallel td solve 2017-11-09 03:11:09 +09:00
ignis
75f0feaa02 Makefile update for optimization flags 2017-11-09 03:05:23 +09:00
ignis
27bf7ab6dd Makefile added option for profiling and solution array axis roll 2017-11-07 03:52:47 +09:00
ignis
3ee83a9fff Temperton circulant tridiagonal algorithm 2017-11-04 01:36:43 +09:00
ignis
56fc50e9f0 par_ptdslv performance tune 2017-11-03 22:29:37 +09:00
ignis
b39e4cbef4 Makefile fftw cluster edition 2017-11-03 03:35:07 +09:00
ignis
dac299c060 Makefile update to use gfortran compiler 2017-08-02 19:39:19 +09:00
ignis
3225cd757b fully mpi working 2017-02-06 12:49:15 +09:00
ignis
6c75483f1e pipeline parallel compact scheme 2017-01-24 15:50:20 +09:00
ignis
383b2c5aa7 init_velocity seperate n_shell range test and e_spec value test 2017-01-24 15:50:20 +09:00
ignis
6f0c1f44fa fdm restart for mpi 2017-01-24 15:50:20 +09:00
ignis
4f60757a9a mpi communication established 2017-01-24 15:50:20 +09:00
ignis
016bd9fb80 gather4 only performs inverse transform 2017-01-24 15:50:20 +09:00
ignis
804698f717 every process call prepare_fdm and fdm_exe, do not work correctly 2017-01-24 15:50:20 +09:00
ignis
4f7bb89461 remove omp 2017-01-24 15:50:20 +09:00
ignis
50bc5073c1 remove old HIT3D save after writing fdm save 2017-01-24 15:50:20 +09:00
ignis
9c48452b0f disable io 2017-01-24 15:50:20 +09:00
ignis
6d7b31c555 write tp_field flush after each x loop 2017-01-24 15:03:34 +09:00
CombustionLab.POSTECH
cfe19ffb17 job_runlimit default to one month 2017-01-24 14:09:18 +09:00
11 changed files with 1657 additions and 413 deletions
Show stats Expand all files Collapse all files

86
Makefile
View file

@ -4,77 +4,37 @@
MPIF90 = blah MPIF90 = blah
ifeq ($(OMPI), 1) FFTW_HOME = $(FFTW_DIR)
FFTW_HOME = ../libs/fftw-3.2.2/lib MPIF90 = mpif90
OPENMPI_HOME = /home/comb/ignis/openmpi/ FCFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include
MPIF90 = $(OPENMPI_HOME)/bin/mpif90 LDFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPIPP_LD_FLAGS) $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm
FCFLAGS = -i8 -r8 -c -openmp $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include -I$(OPENMPI_HOME)/include FCFLAGS_F77 =
LDFLAGS = -i8 -r8 -openmp $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -L$(OPENMPI_HOME)/lib -lfftw3 -lm -shared-intel
FCFLAGS_F77 = -Mextend
MODULES += m_openmpi.o MODULES += m_openmpi.o
else
FFTW_HOME = ../libs/fftw-3.2.2/lib
# compile for mpiP MPI Profiling
ifeq ($(PROFILER), mpiP)
FFTW_HOME = $(FFTW_DIR)
MPIF90 = mpif90 MPIF90 = mpif90
FCFLAGS = -i8 -r8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include FCFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include
LDFLAGS = -i8 -r8 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm MPIP_LD_FLAGS = -L$(MPIP_DIR)/lib -lmpiP -lm -lbfd -liberty -lunwind
FCFLAGS_F77 = -Mextend LDFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPIPP_LD_FLAGS) $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm
FCFLAGS_F77 =
MODULES += m_mpi.o
endif
# Franklin cluster, NERSC
ifeq ($(NERSC_HOST), franklin)
MPIF90 = ftn
FCFLAGS = -target=linux -i8 -r8 -O4 -c
LDFLAGS = -target=linux -i8 -r8 -O4 -lfftw3
FCFLAGS_F77 = -Mextend
endif
# Hopper cluster, NERSC
ifeq ($(NERSC_HOST), hopper)
MPIF90 = ftn
FCFLAGS = -target=linux -i8 -r8 -O4 -c
LDFLAGS = -target=linux -i8 -r8 -O4 -lfftw3
FCFLAGS_F77 = -Mextend
endif
# Yellowrail cluster, LANL
ifeq ($(HOSTNAME), yr-fe1.lanl.gov)
MPIF90 = mpif90
FCFLAGS = -i8 -r8 -O4 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_INCLUDE)
LDFLAGS = -i8 -r8 -O4 -lfftw3 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib
endif
# Coyote cluster, LANL
ifeq ($(HOSTNAME), cy-c2.lanl.gov)
MPIF90 = mpif90
FCFLAGS = -i8 -r8 -O4 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_INCLUDE)
LDFLAGS = -i8 -r8 -O4 -lfftw3 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib
endif
# Linux Cluster, CMU
ifeq ($(HOSTNAME), karman.me.cmu.edu)
FFTW_HOME = ../libs/fftw-3.2.2/lib
MPIF90 = mpif90
FCFLAGS = -i8 -r8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include
LDFLAGS = -i8 -r8 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm
FCFLAGS_F77 = -Mextend
endif endif
# sparrow.stanford.edu, Mac OS X box # tau source instrumentation
ifeq ($(HOSTNAME), sparrow.stanford.edu) ifeq ($(PROFILER), tau_source)
MPIF90 = mpif90 FFTW_HOME = $(FFTW_DIR)
FCFLAGS = -fdefault-real-8 -fdefault-integer-8 -finit-integer=0 -finit-real=zero -c MPIF90 = tau_f90.sh
FCFLAGS_F77 = -ffixed-form -ffixed-line-length-none FCFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 -c $(MPI_COMPILE_FLAGS) -I$(FFTW_HOME)/include
FCFLAGS_F90 = -ffree-form -ffree-line-length-none LDFLAGS = -Ofast -march=native -Wall -fdefault-integer-8 -fdefault-double-8 -fdefault-real-8 $(MPI_LD_FLAGS) -L$(FFTW_HOME)/lib -lfftw3 -lm
LDFLAGS = -fdefault-real-8 -fdefault-integer-8 -finit-integer=0 -finit-real=zero -lmpi -lmpi -lfftw3 -lm FCFLAGS_F77 =
endif endif
# Program name # Program name
PROG = hit3d.x PROG = hit3d.x

45
gather_4.f90
View file

@ -23,8 +23,6 @@ subroutine gather_4
! number of variables to write out ! number of variables to write out
n_out = 3 n_out = 3
if (int_scalars) n_out = n_out + n_scalars
if (les .and. n_les>0) n_out = n_out + n_les
! putting all variables in wrk array ! putting all variables in wrk array
do k = 1,nz do k = 1,nz
@ -44,50 +42,13 @@ subroutine gather_4
! velocities ! velocities
call xFFT3d(-1,1) call xFFT3d(-1,1)
fname = 'u.'//file_ext
tmp4(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,1)
call gather_tmp4(u_)
call xFFT3d(-1,2) call xFFT3d(-1,2)
fname = 'v.'//file_ext
tmp4(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,2)
call gather_tmp4(v_)
call xFFT3d(-1,3) call xFFT3d(-1,3)
fname = 'w.'//file_ext
tmp4(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,3)
call gather_tmp4(w_)
! scalars u_(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,1)
if (int_scalars) then v_(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,2)
do n = 1,n_scalars w_(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,3)
call xFFT3d(-1,3+n)
write(fname,"('sc',i2.2,'.',a6)") n,file_ext
tmp4(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,3+n)
call write_tmp4
end do
end if
! LES quantities
if (les) then
! turbulent viscosity
if (allocated(turb_visc)) then
write(fname,"('nu_t.',a6)") file_ext
tmp4 = turb_visc
call write_tmp4
end if
if (n_les > 0) then
do n = 1, n_les
call xFFT3d(-1,3+n_scalars+n)
write(fname,"('les',i1,'.',a6)") n,file_ext
tmp4(1:nx,1:ny,1:nz) = wrk(1:nx,1:ny,1:nz,3+n_scalars+n)
call write_tmp4
end do
end if
end if
return return
end subroutine gather_4 end subroutine gather_4

4
init_velocity.f90
View file

@ -266,8 +266,10 @@ subroutine init_velocity
do i = 1,nx+2 do i = 1,nx+2
n_shell = nint(sqrt(real(akx(i)**2 + aky(k)**2 + akz(j)**2, 4))) n_shell = nint(sqrt(real(akx(i)**2 + aky(k)**2 + akz(j)**2, 4)))
if (n_shell .gt. 0 .and. n_shell .le. kmax .and. e_spec(n_shell) .gt. zip) then if (n_shell .gt. 0 .and. n_shell .le. kmax ) then
if (e_spec(n_shell) .gt. zip) then
fields(i,j,k,1:3) = fields(i,j,k,1:3) * sqrt(e_spec1(n_shell)/e_spec(n_shell)) fields(i,j,k,1:3) = fields(i,j,k,1:3) * sqrt(e_spec1(n_shell)/e_spec(n_shell))
endif
else else
fields(i,j,k,1:3) = zip fields(i,j,k,1:3) = zip
end if end if

852
m_compact.f90
View file

@ -1,14 +1,20 @@
MODULE m_compact MODULE m_compact
use m_openmpi
IMPLICIT NONE IMPLICIT NONE
PRIVATE PRIVATE
REAL*8, DIMENSION(:), ALLOCATABLE :: lxf,lxs,wxf,wxs, & REAL*8, DIMENSION(:), ALLOCATABLE :: lxf,lxs,wxf,wxs, &
lyf,lys,wyf,wys, & lyf,lys,wyf,wys, &
lzf,lzs,wzf,wzs lzf,lzs,wzf,wzs
! lyzf,lyzs,wyzf,wyzs ! lyzf,lyzs,wyzf,wyzs
REAL*8, DIMENSION(:), ALLOCATABLE :: zz1, zz2, vz1, vz2
INTEGER :: nxc,nyc,nzc INTEGER :: nxc,nyc,nzc
REAL*8, PARAMETER :: ezero = 1.0e-14 REAL*8, PARAMETER :: ezero = 1.0e-14
PUBLIC :: ludcmp,dfnonp,d2fnonp,dfp,d2fp PUBLIC :: ludcmp,dfnonp,d2fnonp,dfp,d2fp,par_dfp,par_d2fp,pdfp,pd2fp
CONTAINS CONTAINS
@ -65,6 +71,22 @@
call nonp_lud(3,nzc) call nonp_lud(3,nzc)
ENDIF ENDIF
ALLOCATE(vz1(nzc),STAT=ierr)
IF(ierr /= 0) PRINT*, 'work array for lud allocation failed'
ALLOCATE(zz1(nzc),STAT=ierr)
IF(ierr /= 0) PRINT*, 'work array for lud allocation failed'
ALLOCATE(vz2(nzc),STAT=ierr)
IF(ierr /= 0) PRINT*, 'work array for lud allocation failed'
ALLOCATE(zz2(nzc),STAT=ierr)
IF(ierr /= 0) PRINT*, 'work array for lud allocation failed'
call ctdlu (3.0, nzc, vz1, zz1)
call ctdlu (11.0/2.0, nzc, vz2, zz2)
! CALL x_lud ! CALL x_lud
! CALL yz_lud ! CALL yz_lud
@ -141,6 +163,280 @@
l(n)=1./d l(n)=1./d
END SUBROUTINE ptdlu END SUBROUTINE ptdlu
SUBROUTINE ctdlu(lambda,n,v,z)
INTEGER, INTENT(IN) :: n
REAL*8, INTENT(IN) :: lambda
REAL*8, INTENT(OUT) :: v(n)
REAL*8, INTENT(OUT) :: z(n)
INTEGER :: i
REAL*8 :: sigma, alpha
v(1)=0.0
v(2)=lambda
DO i=3,n
v(i)=lambda - 1./v(i-1)
ENDDO
if (lambda > 0.0) then
alpha = (-lambda + sqrt(lambda**2 - 4.0))/2.0
else
alpha = (-lambda - sqrt(lambda**2 - 4.0))/2.0
endif
sigma = (1.0 + alpha**2) / (lambda * (1.0 - alpha**2) * (1.0 - alpha**n))
DO i=1,n
z(i) = sigma * (alpha**(i-1) + alpha**(n-i+1))
ENDDO
END SUBROUTINE ctdlu
SUBROUTINE pdfp(x,xu,xl,dx,h,nd,n,nall,dir)
INTEGER,INTENT(IN) :: nall,n,nd,dir
REAL*8,INTENT(IN) :: h
REAL*8,INTENT(IN),DIMENSION(nd,n) :: x
REAL*8,INTENT(OUT),DIMENSION(nd,2) :: xu, xl
REAL*8,INTENT(OUT),DIMENSION(nd,n) :: dx
INTEGER :: i,j
REAL*8 :: r1,r2,h1
integer (kind=MPI_INTEGER_KIND) :: idx
integer (kind=MPI_INTEGER_KIND),dimension(4) :: requests
logical :: recvLow, recvUpp
integer (kind=MPI_INTEGER_KIND), parameter :: rhstag = 0
h1=1./h
r1=7./3.
r2=1./12.
!if (myid.eq.0) write(*,*) "parallel dfp commuication"
if (myid.eq.master) then
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, numprocs-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, numprocs-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
elseif (myid.eq.numprocs-1) then
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, 0, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, 0, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
else
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
endif
call MPI_STARTALL (4, requests, mpi_err)
DO i=3,n-2
DO j=1,nd
dx(j,i)=(r1*(x(j,i+1)-x(j,i-1))+r2*(x(j,i+2)-x(j,i-2)))*h1
ENDDO
ENDDO
recvLow = .true.
recvUpp = .true.
do while (recvLow.or.recvUpp)
call MPI_WAITANY (2, requests(3:), idx, mpi_status, mpi_err)
if (idx.eq.1) then
DO j=1,nd
dx(j,1) =(r1*( x(j,2)-xl(j,2)) +r2*( x(j,3)-xl(j,1))) *h1
ENDDO
DO j=1,nd
dx(j,2) =(r1*( x(j,3)- x(j,1)) +r2*( x(j,4)-xl(j,2))) *h1
ENDDO
recvLow = .false.
endif
if (idx.eq.2) then
DO j=1,nd
dx(j,n-1)=(r1*( x(j,n)- x(j,n-2))+r2*(xu(j,1)- x(j,n-3)))*h1
ENDDO
DO j=1,nd
dx(j,n) =(r1*(xu(j,1)- x(j,n-1))+r2*(xu(j,2)- x(j,n-2)))*h1
ENDDO
recvUpp = .false.
endif
enddo
IF (dir.eq.1) CALL ctdslv(dx,vz1,zz1,nd,n,nall) ! x-direction
IF (dir.eq.2) CALL ctdslv(dx,vz1,zz1,nd,n,nall) ! x-direction
IF (dir.eq.3) CALL ctdslv(dx,vz1,zz1,nd,n,nall) ! x-direction
END SUBROUTINE pdfp
SUBROUTINE pd2fp(x,xu,xl,dx,h,nd,n,nall,dir)
INTEGER,INTENT(IN) :: nall,n,nd,dir
REAL*8,INTENT(IN) :: h
REAL*8,INTENT(IN),DIMENSION(nd,n) :: x
REAL*8,INTENT(OUT),DIMENSION(nd,n) :: dx
INTEGER :: i,j
REAL*8 :: h2,r1,r2,t1,t2
REAL*8,DIMENSION(nd,2) :: xu, xl
integer (kind=MPI_INTEGER_KIND) :: idx
integer (kind=MPI_INTEGER_KIND),dimension(4) :: requests
logical :: recvLow, recvUpp
integer (kind=MPI_INTEGER_KIND), parameter :: rhstag = 0
h2=1./(h*h)
r1=6.
r2=3./8.
!if (myid.eq.0) write(*,*) "parallel d2fp commuication"
if (myid.eq.master) then
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, numprocs-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, numprocs-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
elseif (myid.eq.numprocs-1) then
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, 0, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, 0, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
else
call MPI_SEND_INIT (x(1,n-1), 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(1), mpi_err)
call MPI_SEND_INIT (x(1,1), 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(2), mpi_err)
call MPI_RECV_INIT (xl, 2*nd, MPI_REAL8, myid-1, rhstag, &
MPI_COMM_TASK, requests(3), mpi_err)
call MPI_RECV_INIT (xu, 2*nd, MPI_REAL8, myid+1, rhstag, &
MPI_COMM_TASK, requests(4), mpi_err)
endif
call MPI_STARTALL (4, requests, mpi_err)
DO i=3,n-2
DO j=1,nd
t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1))
t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2))
dx(j,i)=(r1*t1+r2*t2)*h2
ENDDO
ENDDO
recvLow = .true.
recvUpp = .true.
do while (recvLow.or.recvUpp)
call MPI_WAITANY (2, requests(3:), idx, mpi_status, mpi_err)
if (idx.eq.1) then
DO j=1,nd
t1 = (x(j,2)-2.*x(j,1)+xl(j,2))
t2 = (x(j,3)-2.*x(j,1)+xl(j,1))
dx(j,1)=(r1*t1+r2*t2)*h2
ENDDO
DO j=1,nd
t1 = (x(j,3)-2.*x(j,2)+ x(j,1))
t2 = (x(j,4)-2.*x(j,2)+xl(j,2))
dx(j,2)=(r1*t1+r2*t2)*h2
ENDDO
recvLow = .false.
endif
if (idx.eq.2) then
DO j=1,nd
t1 = ( x(j,n)-2.*x(j,n-1)+x(j,n-2))
t2 = (xu(j,1)-2.*x(j,n-1)+x(j,n-3))
dx(j,n-1)=(r1*t1+r2*t2)*h2
ENDDO
DO j=1,nd
t1 = (xu(j,1)-2.*x(j,n)+x(j,n-1))
t2 = (xu(j,2)-2.*x(j,n)+x(j,n-2))
dx(j,n)=(r1*t1+r2*t2)*h2
ENDDO
recvUpp = .false.
endif
enddo
IF (dir.eq.1) CALL ctdslv(dx,vz2,zz2,nd,n,nall) ! x-direction
IF (dir.eq.2) CALL ctdslv(dx,vz2,zz2,nd,n,nall) ! y-direction
IF (dir.eq.3) CALL ctdslv(dx,vz2,zz2,nd,n,nall) ! z-direction
END SUBROUTINE pd2fp
SUBROUTINE dfnonp(n,h,x,dx,nd,dir) SUBROUTINE dfnonp(n,h,x,dx,nd,dir)
INTEGER,INTENT(IN) :: n,nd,dir INTEGER,INTENT(IN) :: n,nd,dir
REAL*8,INTENT(IN) :: h REAL*8,INTENT(IN) :: h
@ -219,6 +515,85 @@
END SUBROUTINE dfp END SUBROUTINE dfp
SUBROUTINE par_dfp(x,xu,xl,dx,h,nd,n,nall,dir)
INTEGER,INTENT(IN) :: nall,n,nd,dir
REAL*8,INTENT(IN) :: h
REAL*8,INTENT(IN),DIMENSION(nd,n) :: x
REAL*8,INTENT(OUT),DIMENSION(nd,n) :: dx
INTEGER :: i,j
REAL*8 :: r1,r2,h1
REAL*8,DIMENSION(nd,2) :: xu, xl
h1=1./h
r1=7./3.
r2=1./12.
!if (myid.eq.0) write(*,*) "parallel dfp commuication"
if (myid.eq.master) then
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, numprocs-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, myid+1, myid+1, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, myid+1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, numprocs-1, numprocs-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
elseif (myid.eq.numprocs-1) then
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, myid-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, 0, 0, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, 0, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, myid-1, myid-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
else
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, myid-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, myid+1, myid+1, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, myid+1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, myid-1, myid-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
endif
DO j=1,nd
dx(j,n-1)=(r1*( x(j,n)- x(j,n-2))+r2*(xu(j,1)- x(j,n-3)))*h1
dx(j,n)= (r1*(xu(j,1)- x(j,n-1))+r2*(xu(j,2)- x(j,n-2)))*h1
dx(j,1)= (r1*( x(j,2)-xl(j,2)) +r2*( x(j,3)-xl(j,1)))*h1
dx(j,2)= (r1*( x(j,3)- x(j,1)) +r2*( x(j,4)-xl(j,2)))*h1
ENDDO
DO i=3,n-2
DO j=1,nd
dx(j,i)=(r1*(x(j,i+1)-x(j,i-1))+r2*(x(j,i+2)-x(j,i-2)))*h1
ENDDO
ENDDO
IF (dir.eq.1) CALL par_ptdslv(dx,lxf,wxf,nd,n,nall) ! x-direction
IF (dir.eq.2) CALL par_ptdslv(dx,lyf,wyf,nd,n,nall) ! x-direction
IF (dir.eq.3) CALL par_ptdslv(dx,lzf,wzf,nd,n,nall) ! x-direction
END SUBROUTINE par_dfp
SUBROUTINE ptdslv(r,n,l,w,nd) SUBROUTINE ptdslv(r,n,l,w,nd)
INTEGER,INTENT(IN) :: n,nd INTEGER,INTENT(IN) :: n,nd
REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r
@ -247,6 +622,362 @@
ENDDO ENDDO
END SUBROUTINE ptdslv END SUBROUTINE ptdslv
SUBROUTINE ctdslv(r,va,za,nd,n,nall)
INTEGER,PARAMETER :: nb = 256
INTEGER,PARAMETER :: nb2 = 64
INTEGER,INTENT(IN) :: n,nd,nall
REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r
REAL*8,INTENT(IN),DIMENSION(nall) :: va
REAL*8,INTENT(IN),DIMENSION(nall) :: za
REAL*8,DIMENSION(n) :: v,z
INTEGER i,j
INTEGER ii,jj
REAL*8, DIMENSION(nd) :: x1, x0
INTEGER npart, nbase, nlow, nupp
INTEGER :: pid, np
REAL*8, DIMENSION(nb) :: r0, r1
pid = myid
np = numprocs
npart = nall / np
nbase = pid * npart
nlow = pid * npart + 1
nupp = (pid + 1) * npart
v = va(nlow:nupp)
z = za(nlow:nupp)
if (npart.lt.4) then
! assertion fail
endif
x1 = 0.0
x0 = 0.0
do i=1,n
x0 = x0 + z(i)*r(:,i)
enddo
call MPI_REDUCE(x0, x1, nd, MPI_REAL8, MPI_SUM, master, MPI_COMM_TASK, mpi_err)
! first process
if (myid.eq.0) then
CALL MPI_SSEND(x1, nd, MPI_REAL8, np-1, 100, MPI_COMM_TASK, mpi_err)
r(:,1) = x1
r(:,2) = r(:,2) - x1
! last process
elseif (pid.eq.(np-1)) then
CALL MPI_RECV(x1, nd, MPI_REAL8, master, 100, MPI_COMM_TASK, mpi_status, mpi_err)
r(:,n) = r(:,n) - x1
! intermediate process
else
endif
! first process
if (myid.eq.0) then
DO jj=1,nd,nb
DO j=jj,jj+nb-1
r(j,2)=r(j,2)/v(2)
ENDDO
DO i=3,n
DO j=jj,jj+nb-1
r(j,i)=(r(j,i)-r(j,i-1))/v(i)
ENDDO
ENDDO
r0 = r(jj:jj+nb-1,n)
CALL MPI_SEND(r0, nb, MPI_REAL8, pid+1, jj, MPI_COMM_TASK, mpi_err)
ENDDO
DO jj=1,nd,nb
CALL MPI_RECV(r1, nb, MPI_REAL8, pid+1, nd+jj, MPI_COMM_TASK, mpi_status, mpi_err)
i=n
DO j=jj,jj+nb-1
r(j,i) = r(j,i) - r1(j-jj+1)/v(i)
ENDDO
DO i=n-1,2,-1
DO j=jj,jj+nb-1
r(j,i) = r(j,i) - r(j,i+1)/v(i)
ENDDO
ENDDO
ENDDO
! last process
elseif (pid.eq.(np-1)) then
DO jj=1,nd,nb
CALL MPI_RECV(r1, nb, MPI_REAL8, pid-1, jj, MPI_COMM_TASK, mpi_status, mpi_err)
i=1
DO j=jj,jj+nb-1
r(j,i)=(r(j,i) - r1(j-jj+1))/v(i)
ENDDO
DO i=2,n
DO j=jj,jj+nb-1
r(j,i)=(r(j,i) - r(j,i-1))/v(i)
ENDDO
ENDDO
ENDDO
DO jj=1,nd,nb
DO i=n-1,1,-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r(j,i+1)/v(i)
ENDDO
ENDDO
r0 = r(jj:jj+nb-1,1)
CALL MPI_SEND(r0, nb, MPI_REAL8, pid-1, nd+jj, MPI_COMM_TASK, mpi_err)
ENDDO
! intermediate process
else
DO jj=1,nd,nb
CALL MPI_RECV(r1, nb, MPI_REAL8, pid-1, jj, MPI_COMM_TASK, mpi_status, mpi_err)
i=1
DO j=jj,jj+nb-1
r(j,i)=(r(j,i)-r1(j-jj+1))/v(i)
ENDDO
DO i=2,n
DO j=jj,jj+nb-1
r(j,i)=(r(j,i)-r(j,i-1))/v(i)
ENDDO
ENDDO
r0 = r(jj:jj+nb-1,n)
CALL MPI_SEND(r0, nb, MPI_REAL8, pid+1, jj, MPI_COMM_TASK, mpi_err)
ENDDO
DO jj=1,nd,nb
CALL MPI_RECV(r1, nb, MPI_REAL8, pid+1, nd+jj, MPI_COMM_TASK, mpi_status, mpi_err)
i=n
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r1(j-jj+1)/v(i)
ENDDO
DO i=n-1,1,-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r(j,i+1)/v(i)
ENDDO
ENDDO
r0 = r(jj:jj+nb-1,1)
CALL MPI_SEND(r0, nb, MPI_REAL8, pid-1, nd+jj, MPI_COMM_TASK, mpi_err)
ENDDO
endif
END SUBROUTINE ctdslv
SUBROUTINE par_ptdslv(r,la,wa,nd,n,nall)
INTEGER,PARAMETER :: nb = 64
INTEGER,INTENT(IN) :: n,nd,nall
REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r
REAL*8,INTENT(IN),DIMENSION(nall) :: la,wa
REAL*8,DIMENSION(n) :: l,w
INTEGER i,j
INTEGER ii,jj
REAL*8, DIMENSION(nd) :: sum
INTEGER npart, nbase, nlow, nupp
INTEGER :: pid, np
REAL*8, DIMENSION(nb) :: r0, r1, sum0
REAL*8, DIMENSION(nb,2) :: buf
pid = myid
np = numprocs
npart = nall / np
nbase = pid * npart
nlow = pid * npart + 1
nupp = (pid + 1) * npart
l = la(nlow:nupp)
w = wa(nlow:nupp)
if (npart.lt.4) then
! assertion fail
endif
call MPI_BARRIER(MPI_COMM_WORLD, mpi_err)
! first process
if (myid.eq.0) then
DO jj=1,nd,nb
DO j=jj,jj+nb-1
sum(j)=w(1)*r(j,1)
r(j,1)=r(j,1)*l(1)
ENDDO
DO i=2,n
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r(j,i-1)
sum(j)=sum(j)+w(i)*r(j,i)
r(j,i)=r(j,i)*l(i)
ENDDO
ENDDO
buf(:,1) = r(jj:jj+nb-1,n)
buf(:,2) = sum(jj:jj+nb-1)
CALL MPI_SEND(buf, 2*nb, MPI_REAL8, pid+1, 1, MPI_COMM_TASK, mpi_err)
ENDDO
DO jj=1,nd,nb
CALL MPI_RECV(buf, 2*nb, MPI_REAL8, pid+1, 2, MPI_COMM_TASK, mpi_status, mpi_err)
r0 = buf(:,1)
r1 = buf(:,2)
i=n
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-l(i)*r0(j-jj+1)-w(i)*r1(j-jj+1)
ENDDO
DO i=n-1,1,-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-l(i)*r(j,i+1)-w(i)*r1(j-jj+1)
ENDDO
ENDDO
ENDDO
! last process
elseif (pid.eq.(np-1)) then
DO jj=1,nd,nb
CALL MPI_RECV(buf, 2*nb, MPI_REAL8, pid-1, 1, MPI_COMM_TASK, mpi_status, mpi_err)
r0 = buf(:,1)
sum0 = buf(:,2)
i=1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r0(j-jj+1)
sum(j)=sum0(j-jj+1)+w(i)*r(j,i)
r(j,i)=r(j,i)*l(i)
ENDDO
DO i=2,n-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r(j,i-1)
sum(j)=sum(j)+w(i)*r(j,i)
r(j,i)=r(j,i)*l(i)
ENDDO
ENDDO
DO j=jj,jj+nb-1
r(j,n)=l(n)*(r(j,n)-sum(j))
ENDDO
ENDDO
DO jj=1,nd,nb
DO j=jj,jj+nb-1
r(j,n-1)=r(j,n-1)-w(n-1)*r(j,n)
ENDDO
DO i=n-2,1,-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-l(i)*r(j,i+1)-w(i)*r(j,n)
ENDDO
ENDDO
buf(:,1) = r(jj:jj+nb-1,1)
buf(:,2) = r(jj:jj+nb-1,n)
CALL MPI_SEND(buf, 2*nb, MPI_REAL8, pid-1, 2, MPI_COMM_TASK, mpi_err)
ENDDO
! intermediate process
else
DO jj=1,nd,nb
CALL MPI_RECV(buf, 2*nb, MPI_REAL8, pid-1, 1, MPI_COMM_TASK, mpi_status, mpi_err)
r0 = buf(:,1)
sum0 = buf(:,2)
i=1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r0(j-jj+1)
sum(j)=sum0(j-jj+1)+w(i)*r(j,i)
r(j,i)=r(j,i)*l(i)
ENDDO
DO i=2,n
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-r(j,i-1)
sum(j)=sum(j)+w(i)*r(j,i)
r(j,i)=r(j,i)*l(i)
ENDDO
ENDDO
buf(:,1) = r(jj:jj+nb-1,n)
buf(:,2) = sum(jj:jj+nb-1)
CALL MPI_SEND(buf, 2*nb, MPI_REAL8, pid+1, 1, MPI_COMM_TASK, mpi_err)
ENDDO
DO jj=1,nd,nb
CALL MPI_RECV(buf, 2*nb, MPI_REAL8, pid+1, 2, MPI_COMM_TASK, mpi_status, mpi_err)
r0 = buf(:,1)
r1 = buf(:,2)
i=n
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-l(i)*r0(j-jj+1)-w(i)*r1(j-jj+1)
ENDDO
DO i=n-1,1,-1
DO j=jj,jj+nb-1
r(j,i)=r(j,i)-l(i)*r(j,i+1)-w(i)*r1(j-jj+1)
ENDDO
ENDDO
buf(:,1) = r(jj:jj+nb-1,1)
!buf(:,2) = r1
CALL MPI_SEND(buf, 2*nb, MPI_REAL8, pid-1, 2, MPI_COMM_TASK, mpi_err)
ENDDO
endif
END SUBROUTINE par_ptdslv
SUBROUTINE d2fp(n,h,x,dx,nd,dir) SUBROUTINE d2fp(n,h,x,dx,nd,dir)
INTEGER,INTENT(IN) :: n,nd,dir INTEGER,INTENT(IN) :: n,nd,dir
REAL*8,INTENT(IN) :: h REAL*8,INTENT(IN) :: h
@ -260,30 +991,22 @@
DO j=1,nd DO j=1,nd
t1 = (x(j,n)-2.*x(j,n-1)+x(j,n-2)) t1 = (x(j,n)-2.*x(j,n-1)+x(j,n-2))
t2 = (x(j,1)-2.*x(j,n-1)+x(j,n-3)) t2 = (x(j,1)-2.*x(j,n-1)+x(j,n-3))
IF (x(j,n).eq.x(j,n-1).and.x(j,n-1).eq.x(j,n-2)) t1=0.
IF (x(j,1).eq.x(j,n-1).and.x(j,n-1).eq.x(j,n-3)) t2=0.
dx(j,n-1)=(r1*t1+r2*t2) dx(j,n-1)=(r1*t1+r2*t2)
t1 = (x(j,1)-2.*x(j,n)+x(j,n-1)) t1 = (x(j,1)-2.*x(j,n)+x(j,n-1))
t2 = (x(j,2)-2.*x(j,n)+x(j,n-2)) t2 = (x(j,2)-2.*x(j,n)+x(j,n-2))
IF (x(j,1).eq.x(j,n).and.x(j,n).eq.x(j,n-1)) t1=0.
IF (x(j,2).eq.x(j,n).and.x(j,n).eq.x(j,n-2)) t2=0.
! dx(j,n)=(r1*(x(j,1)-2.*x(j,n)+x(j,n-1)) & ! dx(j,n)=(r1*(x(j,1)-2.*x(j,n)+x(j,n-1)) &
! +r2*(x(j,2)-2.*x(j,n)+x(j,n-2))) ! +r2*(x(j,2)-2.*x(j,n)+x(j,n-2)))
dx(j,n)=(r1*t1+r2*t2) dx(j,n)=(r1*t1+r2*t2)
t1 = (x(j,2)-2.*x(j,1)+x(j,n)) t1 = (x(j,2)-2.*x(j,1)+x(j,n))
t2 = (x(j,3)-2.*x(j,1)+x(j,n-1)) t2 = (x(j,3)-2.*x(j,1)+x(j,n-1))
IF (x(j,2).eq.x(j,1).and.x(j,1).eq.x(j,n)) t1=0.
IF (x(j,3).eq.x(j,1).and.x(j,1).eq.x(j,n-1)) t2=0.
! dx(j,1)=(r1*(x(j,2)-2.*x(j,1)+x(j,n)) & ! dx(j,1)=(r1*(x(j,2)-2.*x(j,1)+x(j,n)) &
! +r2*(x(j,3)-2.*x(j,1)+x(j,n-1))) ! +r2*(x(j,3)-2.*x(j,1)+x(j,n-1)))
dx(j,1)=(r1*t1+r2*t2) dx(j,1)=(r1*t1+r2*t2)
t1 = (x(j,3)-2.*x(j,2)+x(j,1)) t1 = (x(j,3)-2.*x(j,2)+x(j,1))
t2 = (x(j,4)-2.*x(j,2)+x(j,n)) t2 = (x(j,4)-2.*x(j,2)+x(j,n))
IF (x(j,3).eq.x(j,2).and.x(j,2).eq.x(j,1)) t1=0.
IF (x(j,4).eq.x(j,2).and.x(j,2).eq.x(j,n)) t2=0.
! dx(j,2)=(r1*(x(j,3)-2.*x(j,2)+x(j,1)) & ! dx(j,2)=(r1*(x(j,3)-2.*x(j,2)+x(j,1)) &
! +r2*(x(j,4)-2.*x(j,2)+x(j,n))) ! +r2*(x(j,4)-2.*x(j,2)+x(j,n)))
dx(j,2)=(r1*t1+r2*t2) dx(j,2)=(r1*t1+r2*t2)
@ -297,8 +1020,6 @@
DO j=1,nd DO j=1,nd
t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1)) t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1))
t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2)) t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2))
IF (x(j,i+1).eq.x(j,i).and.x(j,i).eq.x(j,i-1)) t1=0.
IF (x(j,i+2).eq.x(j,i).and.x(j,i).eq.x(j,i-2)) t2=0.
! dx(j,i)=(r1*(x(j,i+1)-2.*x(j,i)+x(j,i-1)) & ! dx(j,i)=(r1*(x(j,i+1)-2.*x(j,i)+x(j,i-1)) &
! +r2*(x(j,i+2)-2.*x(j,i)+x(j,i-2))) ! +r2*(x(j,i+2)-2.*x(j,i)+x(j,i-2)))
dx(j,i)=(r1*t1+r2*t2) dx(j,i)=(r1*t1+r2*t2)
@ -310,6 +1031,109 @@
IF (dir.eq.3) CALL ptdslv(dx,n,lzs,wzs,nd) ! z-direction IF (dir.eq.3) CALL ptdslv(dx,n,lzs,wzs,nd) ! z-direction
END SUBROUTINE d2fp END SUBROUTINE d2fp
SUBROUTINE par_d2fp(x,xu,xl,dx,h,nd,n,nall,dir)
INTEGER,INTENT(IN) :: nall,n,nd,dir
REAL*8,INTENT(IN) :: h
REAL*8,INTENT(IN),DIMENSION(nd,n) :: x
REAL*8,INTENT(OUT),DIMENSION(nd,n) :: dx
INTEGER :: i,j
REAL*8 :: h2,r1,r2,t1,t2
REAL*8,DIMENSION(nd,2) :: xu, xl
h2=1./(h*h)
r1=6.
r2=3./8.
!if (myid.eq.0) write(*,*) "parallel d2fp commuication"
if (myid.eq.master) then
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, numprocs-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, myid+1, myid+1, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, myid+1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, numprocs-1, numprocs-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
elseif (myid.eq.numprocs-1) then
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, myid-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, 0, 0, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, 0, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, myid-1, myid-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
else
call MPI_ISEND (x(1,1), 2*nd, MPI_REAL8, myid-1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xu, 2*nd, MPI_REAL8, myid+1, myid+1, &
MPI_COMM_TASK, mpi_status, mpi_err)
call MPI_ISEND (x(1,n-1), 2*nd, MPI_REAL8, myid+1, myid, &
MPI_COMM_TASK, mpi_request, mpi_err)
call MPI_RECV (xl, 2*nd, MPI_REAL8, myid-1, myid-1, &
MPI_COMM_TASK, mpi_status, mpi_err)
endif
DO j=1,nd
t1 = ( x(j,n)-2.*x(j,n-1)+x(j,n-2))
t2 = (xu(j,1)-2.*x(j,n-1)+x(j,n-3))
dx(j,n-1)=(r1*t1+r2*t2)*h2
t1 = (xu(j,1)-2.*x(j,n)+x(j,n-1))
t2 = (xu(j,2)-2.*x(j,n)+x(j,n-2))
! dx(j,n)=(r1*(x(j,1)-2.*x(j,n)+x(j,n-1)) &
! +r2*(x(j,2)-2.*x(j,n)+x(j,n-2)))
dx(j,n)=(r1*t1+r2*t2)*h2
t1 = (x(j,2)-2.*x(j,1)+xl(j,2))
t2 = (x(j,3)-2.*x(j,1)+xl(j,1))
! dx(j,1)=(r1*(x(j,2)-2.*x(j,1)+x(j,n)) &
! +r2*(x(j,3)-2.*x(j,1)+x(j,n-1)))
dx(j,1)=(r1*t1+r2*t2)*h2
t1 = (x(j,3)-2.*x(j,2)+ x(j,1))
t2 = (x(j,4)-2.*x(j,2)+xl(j,2))
! dx(j,2)=(r1*(x(j,3)-2.*x(j,2)+x(j,1)) &
! +r2*(x(j,4)-2.*x(j,2)+x(j,n)))
dx(j,2)=(r1*t1+r2*t2)*h2
ENDDO
DO i=3,n-2
DO j=1,nd
t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1))
t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2))
! dx(j,i)=(r1*(x(j,i+1)-2.*x(j,i)+x(j,i-1)) &
! +r2*(x(j,i+2)-2.*x(j,i)+x(j,i-2)))
dx(j,i)=(r1*t1+r2*t2)*h2
ENDDO
ENDDO
IF (dir.eq.1) CALL par_ptdslv(dx,lxs,wxs,nd,n,nall) ! x-direction
IF (dir.eq.2) CALL par_ptdslv(dx,lys,wys,nd,n,nall) ! y-direction
IF (dir.eq.3) CALL par_ptdslv(dx,lzs,wzs,nd,n,nall) ! z-direction
END SUBROUTINE par_d2fp
SUBROUTINE tdslv(r,n,l,nd) SUBROUTINE tdslv(r,n,l,nd)
INTEGER,INTENT(IN) :: n,nd INTEGER,INTENT(IN) :: n,nd
REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r REAL*8,INTENT(INOUT),DIMENSION(nd,n) :: r
@ -354,10 +1178,6 @@
dx(j,2)=(x(j,3)-2.*x(j,2)+x(j,1)) dx(j,2)=(x(j,3)-2.*x(j,2)+x(j,1))
dx(j,n-1)=(x(j,n)-2.*x(j,n-1)+x(j,n-2)) dx(j,n-1)=(x(j,n)-2.*x(j,n-1)+x(j,n-2))
dx(j,n)=(a*x(j,n)+b*x(j,n-1)+c*x(j,n-2)+e*x(j,n-3)) dx(j,n)=(a*x(j,n)+b*x(j,n-1)+c*x(j,n-2)+e*x(j,n-3))
IF (x(j,1).eq.x(j,2).and.x(j,2).eq.x(j,3).and.x(j,3).eq.x(j,4)) dx(j,1)=0.
IF (x(j,3).eq.x(j,2).and.x(j,2).eq.x(j,1)) dx(j,2)=0.
IF (x(j,n).eq.x(j,n-1).and.x(j,n-1).eq.x(j,n-2).and.x(j,n-2).eq.x(j,n-3)) dx(j,n)=0.
IF (x(j,n).eq.x(j,n-1).and.x(j,n-1).eq.x(j,n-2)) dx(j,n-1)=0.
dx(j,1)=dx(j,1)*h2 dx(j,1)=dx(j,1)*h2
dx(j,2)=dx(j,2)*h2*r3 dx(j,2)=dx(j,2)*h2*r3
dx(j,n-1)=dx(j,n-1)*h2*r3 dx(j,n-1)=dx(j,n-1)*h2*r3
@ -367,8 +1187,6 @@
DO j=1,nd DO j=1,nd
t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1)) t1 = (x(j,i+1)-2.*x(j,i)+x(j,i-1))
t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2)) t2 = (x(j,i+2)-2.*x(j,i)+x(j,i-2))
IF (x(j,i+1).eq.x(j,i).and.x(j,i).eq.x(j,i-1)) t1=0.
IF (x(j,i+2).eq.x(j,i).and.x(j,i).eq.x(j,i-2)) t2=0.
! dx(j,i)=(r1*(x(j,i+1)-2.*x(j,i)+x(j,i-1)) & ! dx(j,i)=(r1*(x(j,i+1)-2.*x(j,i)+x(j,i-1)) &
! +r2*(x(j,i+2)-2.*x(j,i)+x(j,i-2))) ! +r2*(x(j,i+2)-2.*x(j,i)+x(j,i-2)))

1026
m_fdm_calc.f90
View file

File diff suppressed because it is too large Load diff

3
m_mpi.f90
View file

@ -64,7 +64,8 @@ contains
! initializing MPI environment ! initializing MPI environment
call MPI_INIT_THREAD(MPI_THREAD_SERIALIZED, mpi_provide, mpi_err) !call MPI_INIT_THREAD(MPI_THREAD_SERIALIZED, mpi_provide, mpi_err)
call MPI_INIT(mpi_err)
call MPI_Comm_size(MPI_COMM_WORLD,numprocs_world,mpi_err) call MPI_Comm_size(MPI_COMM_WORLD,numprocs_world,mpi_err)
call MPI_Comm_rank(MPI_COMM_WORLD,myid_world,mpi_err) call MPI_Comm_rank(MPI_COMM_WORLD,myid_world,mpi_err)

3
m_openmpi.f90
View file

@ -64,7 +64,8 @@ contains
! initializing MPI environment ! initializing MPI environment
call MPI_INIT_THREAD(MPI_THREAD_SERIALIZED, mpi_provide, mpi_err) !call MPI_INIT_THREAD(MPI_THREAD_SERIALIZED, mpi_provide, mpi_err)
call MPI_INIT(mpi_err)
call MPI_Comm_size(MPI_COMM_WORLD,numprocs_world,mpi_err) call MPI_Comm_size(MPI_COMM_WORLD,numprocs_world,mpi_err)
call MPI_Comm_rank(MPI_COMM_WORLD,myid_world,mpi_err) call MPI_Comm_rank(MPI_COMM_WORLD,myid_world,mpi_err)

4
m_timing.f90
View file

@ -19,8 +19,8 @@ contains
! make the default job runlimit to be 6 months ! make the default job runlimit to be 6 months
job_runlimit = 6 * 30 * 24 * 60 job_runlimit = 6 * 30 * 24 * 60
! make the default job runlimit to be 12 hours ! make the default job runlimit to be 12 hours -> one month
job_runlimit = 12 * 60 job_runlimit = 12 * 60 * 30 * 2
write(out,'(a30,3x,i20)') 'job_runlimit (default):',job_runlimit write(out,'(a30,3x,i20)') 'job_runlimit (default):',job_runlimit

41
main.f90
View file

@ -67,10 +67,7 @@ program x_code
! Initializing the LES stuff ! Initializing the LES stuff
if (les) call m_les_begin if (les) call m_les_begin
if (task.eq.'hydro' .and. myid.eq.master) then call prepare_fdm
call prepare_fdm
end if
call MPI_BARRIER(MPI_COMM_TASK, mpi_err)
call stat_main call stat_main
@ -132,22 +129,10 @@ program x_code
end if end if
end if end if
if (fdm_sw.ne.0) then ! Reacting ! reacting scalars
if (fdm_sw.ne.0) then
call gather_4 call gather_4
if (myid.eq.master) then call fdm_exe
call fdm_exe
do id_to=1,numprocs-1
call MPI_Isend (dummy, 1, MPI_REAL, id_to, 0, MPI_COMM_TASK, mpi_request, mpi_err)
end do
else
call MPI_Irecv (dummy, 1, MPI_REAL, master, 0, MPI_COMM_TASK, mpi_request, mpi_err)
do
call MPI_Test(mpi_request, flag, mpi_status, mpi_err)
if (flag) exit
call SLEEPQQ (200)
end do
end if
call MPI_BARRIER(MPI_COMM_TASK, mpi_err)
endif endif
! RHS for scalars ! RHS for scalars
@ -202,18 +187,18 @@ program x_code
! write the restart file if it's the time ! write the restart file if it's the time
if (mod(itime,IPRINT2).eq.0) then if (mod(itime,IPRINT2).eq.0) then
call restart_write_parallel call restart_write_parallel
!remove hit restart file
call system ('rm -f '//run_name//'.64.'//ex_file_ext)
if (fdm_sw.ne.0) then ! Reacting if (fdm_sw.ne.0) then ! Reacting
call gather_4 call gather_4
if (myid.eq.master) then if (TIME.ge.bgn_save) then
!write itime, savenum pair call fdm_restart_write
if(TIME.ge.bgn_save) call fdm_restart_write if (myid.eq.master) then
write (restart_pair, *) file_ext, fullsavenum-1 write (restart_pair, *) file_ext, fullsavenum-1, '\n'
call write_tp_field !call write_tp_field
end if
end if end if
call MPI_BARRIER(MPI_COMM_TASK, mpi_err)
endif endif
!remove HIT3D restart file except the latest
call system ('rm -f '//run_name//'.64.'//ex_file_ext)
ex_file_ext = file_ext ex_file_ext = file_ext
endif endif
@ -266,7 +251,7 @@ program x_code
if (task_split) call fields_to_stats if (task_split) call fields_to_stats
if (mod(itime,iprint1).eq.0) call stat_main if (mod(itime,iprint1).eq.0) call stat_main
if (fdm_sw.eq.0) then ! non-reacting if (fdm_sw.eq.0) then ! non-reacting
if (mod(itime,iprint2).eq.0) call io_write_4 !if (mod(itime,iprint2).eq.0) call io_write_4
endif endif
end if end if
end if stats end if stats

4
write_stats.f90
View file

@ -31,6 +31,7 @@ subroutine write_tp_field
do i=1,nx,spx do i=1,nx,spx
write(tp_field,'(6e18.9)') dx*real(i),dy*real(j),dz*real(k),u_(i,j,k),v_(i,j,k),w_(i,j,k) ! u',v',w' write(tp_field,'(6e18.9)') dx*real(i),dy*real(j),dz*real(k),u_(i,j,k),v_(i,j,k),w_(i,j,k) ! u',v',w'
enddo enddo
call flush(tp_field)
enddo enddo
enddo enddo
@ -49,13 +50,12 @@ subroutine write_tp_field
(1.-y1(2,i,j,ii)/y1(1,i,j,ii)),u_(i,j,ii), & (1.-y1(2,i,j,ii)/y1(1,i,j,ii)),u_(i,j,ii), &
v_(i,j,ii),w_(i,j,ii) v_(i,j,ii),w_(i,j,ii)
enddo enddo
call flush(tp_field)
enddo enddo
enddo enddo
endif endif
call flush(tp_field)
897 format('ZONE T="time= ',f10.5,'" I= ',i4,' J= ',i4,' K= ',i4,' F=POINT') 897 format('ZONE T="time= ',f10.5,'" I= ',i4,' J= ',i4,' K= ',i4,' F=POINT')
end subroutine write_tp_field end subroutine write_tp_field

2
x_fftw.f90
View file

@ -298,7 +298,7 @@ CONTAINS
! Definition of the array ialias. ! Definition of the array ialias.
! The array ialias is just the number of wavenumbers at (i,j,k) that have ! The array ialias is just the number of wavenumbers at (i,j,k) that have
! their magnitude higher than nx/3. This is needed in dealiasing procedures. ! their magnitude higher than nx/3. This is needed in dealiasing procedures.
rnx3 = real(nx/3, 8) rnx3 = real(2*nx, 8) / real(3*lx, 8)
do k = 1,nz do k = 1,nz
if (abs(aky(k)) .gt. rnx3) ialias(:,:,k) = 1 if (abs(aky(k)) .gt. rnx3) ialias(:,:,k) = 1
do j = 1,ny do j = 1,ny