From 9854238fb38b7bba8bc173f3e80482cb4fa40abb Mon Sep 17 00:00:00 2001 From: ignis Date: Fri, 22 Aug 2014 14:19:00 +0900 Subject: [PATCH] fdm restart for mpi --- m_fdm_calc.f90 | 214 ++++++++++++++++++++++++++++++++++++++----------- main.f90 | 14 ++-- 2 files changed, 173 insertions(+), 55 deletions(-) diff --git a/m_fdm_calc.f90 b/m_fdm_calc.f90 index dfc9709..c3ded7d 100644 --- a/m_fdm_calc.f90 +++ b/m_fdm_calc.f90 @@ -1,6 +1,7 @@ module m_fdm_calc use m_parameters + use m_io use m_compact implicit none @@ -33,18 +34,171 @@ module m_fdm_calc contains + subroutine fdm_restart_read + integer(kind=MPI_INTEGER_KIND) :: fh + integer(kind=MPI_OFFSET_KIND) :: offset + + write(fname,"('fort.',i4)") restartnum + + if (myid.eq.master) then + write (6,*) '********************************************************' + write (6,*) ' FDM field is being initialized for restarting.' + write (6,457) restartnum + 457 format(' Restart file number : ',i6) + + OPEN (restartnum,file=fname,form='unformatted',status='unknown') + read (restartnum) fdmtime,tt1,tt2,tt3,oldsumc,time_int + read (restartnum) fdmcyc,fdmdt,dummyu_ + read (restartnum) tt1,tt2 !dt_fdmsave,dt_fullsave + read (restartnum) t_fdmsave,t_fullsave + read (restartnum) in_yr,out_yr + CLOSE (restartnum) + + if(fdmtime.ge.t_fdmsave) t_fdmsave=t_fdmsave+dt_fdmsave + if(fdmtime.ge.t_fullsave) t_fullsave=t_fullsave+dt_fullsave + + write (6,456) fdmtime,fdmcyc + 456 format(' Restart time : ',f10.5,' / FDM cycle : ',i6) + write(*,*) + write(*,*) ' Save Times : FDM FULL ' + write(*,454) t_fdmsave,t_fullsave + 454 format(' ',f7.3,' ',f7.3) + write(*,*)' Save Intervals : FDM FULL','from input' + write(*,453) dt_fdmsave,dt_fullsave + 453 format(' ',f7.3,' ',f7.3) + write(*,*) + + if (ABS(fdmtime-TIME).le.1.0e-10) then + write(*,*) ' Spectral and FDM times are consistent.' + else + write(*,*)' !! Warning : Spectral and FDM times are different !!' + write(*,455) TIME,fdmtime + 455 format(' !! Spectral Time : ',f10.5,' / FDM Time : ',f10.5) + endif + + end if + + count = 1 + + call MPI_BCAST(fdmtime, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(tt1, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(tt2, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(tt3, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(oldsumc, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(time_int, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + + call MPI_BCAST(fdmcyc, count, MPI_INTEGER8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(fdmdt, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(dummyu_, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + + !call MPI_BCAST(dt_fdmsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + !call MPI_BCAST(dt_fullsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + + !call MPI_BCAST(t_fdmsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + !call MPI_BCAST(t_fullsave, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + + call MPI_BCAST(in_yr, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + call MPI_BCAST(out_yr, count, MPI_REAL8, 0, MPI_COMM_TASK, mpi_err) + + + ! opening the file + call MPI_INFO_CREATE(mpi_info, mpi_err) + call MPI_FILE_OPEN(MPI_COMM_TASK,fname,MPI_MODE_RDONLY,mpi_info,fh,mpi_err) + + count = nx * ny * nz + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 0*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_READ_AT_ALL(fh, offset, u_, count, MPI_REAL8, mpi_status, mpi_err) + + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 1*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_READ_AT_ALL(fh, offset, v_, count, MPI_REAL8, mpi_status, mpi_err) + + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 2*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_READ_AT_ALL(fh, offset, w_, count, MPI_REAL8, mpi_status, mpi_err) + + count = 2 * nx * ny * nz + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 3*nx*ny*nz_all*8 + myid*2*nx*ny*nz*8 + call MPI_FILE_READ_AT_ALL(fh, offset, y1, count, MPI_REAL8, mpi_status, mpi_err) + + call MPI_FILE_CLOSE(fh, mpi_err) + call MPI_INFO_FREE(mpi_info, mpi_err) + + end subroutine fdm_restart_read + subroutine fdm_restart_write - write(*,*) '=======================================================' - write(*,*) 'Full results are being written',fullsavenum - OPEN (fullsavenum,form='unformatted',status='unknown') - write (fullsavenum) fdmtime,nx,ny,nz_all,oldsumc,time_int - write (fullsavenum) fdmcyc,DT,dummyu_ - write (fullsavenum) dt_fdmsave,dt_fullsave - write (fullsavenum) t_fdmsave,t_fullsave - write (fullsavenum) in_yr,out_yr - write (fullsavenum) u_+dummyu_,v_,w_,y1 - CLOSE (fullsavenum) - write(*,*) '=======================================================' + integer(kind=MPI_INTEGER_KIND) :: fh + integer(kind=MPI_OFFSET_KIND) :: offset + ! --------------- writing process ------------------ + + if (myid.eq.0) write(*,*) '=======================================================' + if (myid.eq.0) write(*,*) 'Full results are being written',fullsavenum + + !fname = 'fort.'//savenum + write(fname,"('fort.',i4)") fullsavenum + + ! opening the file + call MPI_INFO_CREATE(mpi_info, mpi_err) + call MPI_FILE_OPEN(MPI_COMM_TASK,fname,MPI_MODE_WRONLY+MPI_MODE_CREATE,mpi_info,fh,mpi_err) + + ! the master node writes the header with parameters + if (myid.eq.0) then + count = 1 + call MPI_FILE_WRITE(fh, 8*6, count, MPI_INTEGER4, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, fdmtime, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, nx, count, MPI_INTEGER8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, ny, count, MPI_INTEGER8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, nz_all, count, MPI_INTEGER8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, oldsumc, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, time_int, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, 8*6, count, MPI_INTEGER4, mpi_status, mpi_err) + + call MPI_FILE_WRITE(fh, 8*3, count, MPI_INTEGER4, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, fdmcyc, count, MPI_INTEGER8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, DT, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, dummyu_, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, 8*3, count, MPI_INTEGER4, mpi_status, mpi_err) + + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, dt_fdmsave, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, dt_fullsave, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, t_fdmsave, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, t_fullsave, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, in_yr, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, out_yr, count, MPI_REAL8, mpi_status, mpi_err) + call MPI_FILE_WRITE(fh, 8*2, count, MPI_INTEGER4, mpi_status, mpi_err) + + call MPI_FILE_WRITE(fh, 8*nx*ny*nz_all*5, count, MPI_INTEGER4, mpi_status, mpi_err) + + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 5*nx*ny*nz_all*8 + call MPI_FILE_WRITE_AT(fh, offset, 8*nx*ny*nz_all*5, count, MPI_INTEGER4, mpi_status, mpi_err) + + end if + + + count = nx * ny * nz + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 0*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_WRITE_AT(fh, offset, u_, count, MPI_REAL8, mpi_status, mpi_err) + + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 1*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_WRITE_AT(fh, offset, v_, count, MPI_REAL8, mpi_status, mpi_err) + + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 2*nx*ny*nz_all*8 + myid*nx*ny*nz*8 + call MPI_FILE_WRITE_AT(fh, offset, w_, count, MPI_REAL8, mpi_status, mpi_err) + + count = 2 * nx * ny * nz + offset = 2*4+6*8 + 2*4+3*8 + 3*(2*4+2*8) + 4 + 3*nx*ny*nz_all*8 + myid*2*nx*ny*nz*8 + call MPI_FILE_WRITE_AT(fh, offset, y1, count, MPI_REAL8, mpi_status, mpi_err) + + + call MPI_FILE_CLOSE(fh, mpi_err) + call MPI_INFO_FREE(mpi_info, mpi_err) + + if (myid.eq.0) write(*,*) '=======================================================' fullsavenum=fullsavenum+1 end subroutine fdm_restart_write @@ -187,44 +341,8 @@ module m_fdm_calc ! Restart setup ================================================================ else - return - write (6,*) '********************************************************' - write (6,*) ' FDM field is being initialized for restarting.' - write (6,457) restartnum - 457 format(' Restart file number : ',i6) - - OPEN (restartnum,form='unformatted',status='unknown') - read (restartnum) fdmtime,tt1,tt2,tt3,oldsumc,time_int - read (restartnum) fdmcyc,tt4,dummyu_ - read (restartnum) tt1,tt2 !dt_fdmsave,dt_fullsave - read (restartnum) t_fdmsave,t_fullsave - read (restartnum) in_yr,out_yr - read (restartnum) u_,v_,w_,y1 - CLOSE (restartnum) - - - if(fdmtime.ge.t_fdmsave) t_fdmsave=t_fdmsave+dt_fdmsave - if(fdmtime.ge.t_fullsave) t_fullsave=t_fullsave+dt_fullsave - - write (6,456) fdmtime,fdmcyc - 456 format(' Restart time : ',f10.5,' / FDM cycle : ',i6) - write(*,*) - write(*,*) ' Save Times : FDM FULL ' - write(*,454) t_fdmsave,t_fullsave - 454 format(' ',f7.3,' ',f7.3) - write(*,*)' Save Intervals : FDM FULL','from input' - write(*,453) dt_fdmsave,dt_fullsave - 453 format(' ',f7.3,' ',f7.3) - write(*,*) - - if (ABS(fdmtime-TIME).le.1.0e-10) then - write(*,*) ' Spectral and FDM times are consistent.' - else - write(*,*)' !! Warning : Spectral and FDM times are different !!' - write(*,455) TIME,fdmtime - 455 format(' !! Spectral Time : ',f10.5,' / FDM Time : ',f10.5) - endif + CALL fdm_restart_read fullsavenum=restartnum+1 endif diff --git a/main.f90 b/main.f90 index a6abb28..ded720d 100644 --- a/main.f90 +++ b/main.f90 @@ -187,14 +187,14 @@ program x_code if (mod(itime,IPRINT2).eq.0) then call restart_write_parallel if (fdm_sw.ne.0) then ! Reacting - !call gather_4 - if (myid.eq.master) then - !write itime, savenum pair - !if(TIME.ge.bgn_save) call fdm_restart_write - !write (restart_pair, *) file_ext, fullsavenum-1 - !call write_tp_field + call gather_4 + if (TIME.ge.bgn_save) then + call fdm_restart_write + if (myid.eq.master) then + write (restart_pair, *) file_ext, fullsavenum-1, '\n' + !call write_tp_field + end if end if - call MPI_BARRIER(MPI_COMM_TASK, mpi_err) endif !remove HIT3D restart file except the latest call system ('rm -f '//run_name//'.64.'//ex_file_ext)