fdm restart for mpi
This commit is contained in:
ignis
parent
c3345ba9c2
commit
9854238fb3
2 changed files with 173 additions and 55 deletions
214
m_fdm_calc.f90
214
m_fdm_calc.f90
|
|
@ -1,6 +1,7 @@
|
|||
module m_fdm_calc
|
||||
|
||||
use m_parameters
|
||||
use m_io
|
||||
use m_compact
|
||||
|
||||
implicit none
|
||||
|
|
@ -33,18 +34,171 @@ module m_fdm_calc
|
|||
contains
|
||||
|
||||
|
||||
subroutine fdm_restart_read
|
||||
integer(kind=MPI_INTEGER_KIND) :: fh
|
||||
integer(kind=MPI_OFFSET_KIND) :: offset
|
||||
|
||||
write(fname,"('fort.',i4)") restartnum
|
||||
|
||||
if (myid.eq.master) then
|
||||
write (6,*) '********************************************************'
|
||||
write (6,*) ' FDM field is being initialized for restarting.'
|
||||
write (6,457) restartnum
|
||||
457 format(' Restart file number : ',i6)
|
||||
|
||||
OPEN (restartnum,file=fname,form='unformatted',status='unknown')
|
||||
read (restartnum) fdmtime,tt1,tt2,tt3,oldsumc,time_int
|
||||
read (restartnum) fdmcyc,fdmdt,dummyu_
|
||||
read (restartnum) tt1,tt2 !dt_fdmsave,dt_fullsave
|
||||
read (restartnum) t_fdmsave,t_fullsave
|
||||
read (restartnum) in_yr,out_yr
|
||||
CLOSE (restartnum)
|
||||
|
||||
if(fdmtime.ge.t_fdmsave) t_fdmsave=t_fdmsave+dt_fdmsave
|
||||
if(fdmtime.ge.t_fullsave) t_fullsave=t_fullsave+dt_fullsave
|
||||
|
||||
write (6,456) fdmtime,fdmcyc
|
||||
456 format(' Restart time : ',f10.5,' / FDM cycle : ',i6)
|
||||
write(*,*)
|
||||
write(*,*) ' Save Times : FDM FULL '
|
||||
write(*,454) t_fdmsave,t_fullsave
|
||||
454 format(' ',f7.3,' ',f7.3)
|
||||
write(*,*)' Save Intervals : FDM FULL','from input'
|
||||
write(*,453) dt_fdmsave,dt_fullsave
|
||||
453 format(' ',f7.3,' ',f7.3)
|
||||
write(*,*)
|
||||
|
||||
if (ABS(fdmtime-TIME).le.1.0e-10) then
|
||||
write(*,*) ' Spectral and FDM times are consistent.'
|
||||
else
|
||||
write(*,*)' !! Warning : Spectral and FDM times are different !!'
|
||||
write(*,455) TIME,fdmtime
|
||||
455 format(' !! Spectral Time : ',f10.5,' / FDM Time : ',f10.5)
|
||||
endif
|
||||
|
||||
end if
|
||||
|
||||
count = 1
|
||||
|
||||
call MPI_BCAST(fdmtime, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(tt1, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(tt2, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(tt3, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(oldsumc, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(time_int, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
|
||||
call MPI_BCAST(fdmcyc, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(fdmdt, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(dummyu_, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
|
||||
!call MPI_BCAST(dt_fdmsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
!call MPI_BCAST(dt_fullsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
|
||||
!call MPI_BCAST(t_fdmsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
!call MPI_BCAST(t_fullsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
|
||||
call MPI_BCAST(in_yr, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
call MPI_BCAST(out_yr, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err)
|
||||
|
||||
|
||||
! opening the file
|
||||
call MPI_INFO_CREATE(mpi_info, mpi_err)
|
||||
call MPI_FILE_OPEN(MPI_COMM_TASK,fname,MPI_MODE_RDONLY,mpi_info,fh,mpi_err)
|
||||
|
||||
count = nx * ny * nz
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 0*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_READ_AT_ALL(fh, offset, u_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 1*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_READ_AT_ALL(fh, offset, v_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 2*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_READ_AT_ALL(fh, offset, w_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
count = 2 * nx * ny * nz
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 3*nx*ny*nz_all*8 + myid*2*nx*ny*nz*8
|
||||
call MPI_FILE_READ_AT_ALL(fh, offset, y1, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_CLOSE(fh, mpi_err)
|
||||
call MPI_INFO_FREE(mpi_info, mpi_err)
|
||||
|
||||
end subroutine fdm_restart_read
|
||||
|
||||
subroutine fdm_restart_write
|
||||
write(*,*) '======================================================='
|
||||
write(*,*) 'Full results are being written',fullsavenum
|
||||
OPEN (fullsavenum,form='unformatted',status='unknown')
|
||||
write (fullsavenum) fdmtime,nx,ny,nz_all,oldsumc,time_int
|
||||
write (fullsavenum) fdmcyc,DT,dummyu_
|
||||
write (fullsavenum) dt_fdmsave,dt_fullsave
|
||||
write (fullsavenum) t_fdmsave,t_fullsave
|
||||
write (fullsavenum) in_yr,out_yr
|
||||
write (fullsavenum) u_+dummyu_,v_,w_,y1
|
||||
CLOSE (fullsavenum)
|
||||
write(*,*) '======================================================='
|
||||
integer(kind=MPI_INTEGER_KIND) :: fh
|
||||
integer(kind=MPI_OFFSET_KIND) :: offset
|
||||
! --------------- writing process ------------------
|
||||
|
||||
if (myid.eq.0) write(*,*) '======================================================='
|
||||
if (myid.eq.0) write(*,*) 'Full results are being written',fullsavenum
|
||||
|
||||
!fname = 'fort.'//savenum
|
||||
write(fname,"('fort.',i4)") fullsavenum
|
||||
|
||||
! opening the file
|
||||
call MPI_INFO_CREATE(mpi_info, mpi_err)
|
||||
call MPI_FILE_OPEN(MPI_COMM_TASK,fname,MPI_MODE_WRONLY+MPI_MODE_CREATE,mpi_info,fh,mpi_err)
|
||||
|
||||
! the master node writes the header with parameters
|
||||
if (myid.eq.0) then
|
||||
count = 1
|
||||
call MPI_FILE_WRITE(fh, 8*6, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, fdmtime, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, nx, count, MPI_INTEGER8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, ny, count, MPI_INTEGER8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, nz_all, count, MPI_INTEGER8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, oldsumc, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, time_int, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, 8*6, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_WRITE(fh, 8*3, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, fdmcyc, count, MPI_INTEGER8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, DT, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, dummyu_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, 8*3, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, dt_fdmsave, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, dt_fullsave, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, t_fdmsave, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, t_fullsave, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, in_yr, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, out_yr, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
call MPI_FILE_WRITE(fh, 8*nx*ny*nz_all*5, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 5*nx*ny*nz_all*8
|
||||
call MPI_FILE_WRITE_AT(fh, offset, 8*nx*ny*nz_all*5, count, MPI_INTEGER4, mpi_status, mpi_err)
|
||||
|
||||
end if
|
||||
|
||||
|
||||
count = nx * ny * nz
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 0*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_WRITE_AT(fh, offset, u_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 1*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_WRITE_AT(fh, offset, v_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 2*nx*ny*nz_all*8 + myid*nx*ny*nz*8
|
||||
call MPI_FILE_WRITE_AT(fh, offset, w_, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
count = 2 * nx * ny * nz
|
||||
offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 3*nx*ny*nz_all*8 + myid*2*nx*ny*nz*8
|
||||
call MPI_FILE_WRITE_AT(fh, offset, y1, count, MPI_REAL8, mpi_status, mpi_err)
|
||||
|
||||
|
||||
call MPI_FILE_CLOSE(fh, mpi_err)
|
||||
call MPI_INFO_FREE(mpi_info, mpi_err)
|
||||
|
||||
if (myid.eq.0) write(*,*) '======================================================='
|
||||
fullsavenum=fullsavenum+1
|
||||
end subroutine fdm_restart_write
|
||||
|
||||
|
|
@ -187,44 +341,8 @@ module m_fdm_calc
|
|||
|
||||
! Restart setup ================================================================
|
||||
else
|
||||
return
|
||||
write (6,*) '********************************************************'
|
||||
write (6,*) ' FDM field is being initialized for restarting.'
|
||||
write (6,457) restartnum
|
||||
457 format(' Restart file number : ',i6)
|
||||
|
||||
OPEN (restartnum,form='unformatted',status='unknown')
|
||||
read (restartnum) fdmtime,tt1,tt2,tt3,oldsumc,time_int
|
||||
read (restartnum) fdmcyc,tt4,dummyu_
|
||||
read (restartnum) tt1,tt2 !dt_fdmsave,dt_fullsave
|
||||
read (restartnum) t_fdmsave,t_fullsave
|
||||
read (restartnum) in_yr,out_yr
|
||||
read (restartnum) u_,v_,w_,y1
|
||||
CLOSE (restartnum)
|
||||
|
||||
|
||||
if(fdmtime.ge.t_fdmsave) t_fdmsave=t_fdmsave+dt_fdmsave
|
||||
if(fdmtime.ge.t_fullsave) t_fullsave=t_fullsave+dt_fullsave
|
||||
|
||||
write (6,456) fdmtime,fdmcyc
|
||||
456 format(' Restart time : ',f10.5,' / FDM cycle : ',i6)
|
||||
write(*,*)
|
||||
write(*,*) ' Save Times : FDM FULL '
|
||||
write(*,454) t_fdmsave,t_fullsave
|
||||
454 format(' ',f7.3,' ',f7.3)
|
||||
write(*,*)' Save Intervals : FDM FULL','from input'
|
||||
write(*,453) dt_fdmsave,dt_fullsave
|
||||
453 format(' ',f7.3,' ',f7.3)
|
||||
write(*,*)
|
||||
|
||||
if (ABS(fdmtime-TIME).le.1.0e-10) then
|
||||
write(*,*) ' Spectral and FDM times are consistent.'
|
||||
else
|
||||
write(*,*)' !! Warning : Spectral and FDM times are different !!'
|
||||
write(*,455) TIME,fdmtime
|
||||
455 format(' !! Spectral Time : ',f10.5,' / FDM Time : ',f10.5)
|
||||
endif
|
||||
|
||||
CALL fdm_restart_read
|
||||
fullsavenum=restartnum+1
|
||||
|
||||
endif
|
||||
|
|
|
|||
14
main.f90
14
main.f90
|
|
@ -187,14 +187,14 @@ program x_code
|
|||
if (mod(itime,IPRINT2).eq.0) then
|
||||
call restart_write_parallel
|
||||
if (fdm_sw.ne.0) then ! Reacting
|
||||
!call gather_4
|
||||
if (myid.eq.master) then
|
||||
!write itime, savenum pair
|
||||
!if(TIME.ge.bgn_save) call fdm_restart_write
|
||||
!write (restart_pair, *) file_ext, fullsavenum-1
|
||||
!call write_tp_field
|
||||
call gather_4
|
||||
if (TIME.ge.bgn_save) then
|
||||
call fdm_restart_write
|
||||
if (myid.eq.master) then
|
||||
write (restart_pair, *) file_ext, fullsavenum-1, '\n'
|
||||
!call write_tp_field
|
||||
end if
|
||||
end if
|
||||
call MPI_BARRIER(MPI_COMM_TASK, mpi_err)
|
||||
endif
|
||||
!remove HIT3D restart file except the latest
|
||||
call system ('rm -f '//run_name//'.64.'//ex_file_ext)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue