tem_stencil_communicate Subroutine

private subroutine tem_stencil_communicate(send, recv, elem, computeStencil, proc, commPattern, iStencil)

Communicate the complete stencil

Currently, this assumes same stencils for all participating elements

Arguments

Type IntentOptional Attributes Name
type(tem_communication_type), intent(inout) :: send

send and recv communication buffers
type(tem_communication_type), intent(inout) :: recv

send and recv communication buffers
type(tem_element_type), intent(inout) :: elem

levelDesc element list
type(tem_stencilHeader_type), intent(in) :: computeStencil

array of all stencils used in the simulation
type(tem_comm_env_type), intent(in) :: proc

Process description to use.
type(tem_commPattern_type), intent(in) :: commPattern

communication pattern
integer, intent(in) :: iStencil

amount of values to communicate

Calls

proc~~tem_stencil_communicate~~CallsGraph proc~tem_stencil_communicate tem_stencil_communicate interface~append~11 append proc~tem_stencil_communicate->interface~append~11 interface~init~24 init proc~tem_stencil_communicate->interface~init~24 proc~tem_stencil_getheaderpos tem_stencil_getHeaderPos proc~tem_stencil_communicate->proc~tem_stencil_getheaderpos proc~append_ga_dynlong append_ga_dynlong interface~append~11->proc~append_ga_dynlong proc~append_ga_dynlong_vec append_ga_dynlong_vec interface~append~11->proc~append_ga_dynlong_vec proc~init_ga2d_real init_ga2d_real interface~init~24->proc~init_ga2d_real interface~expand~9 expand proc~append_ga_dynlong->interface~expand~9 proc~append_ga_dynlong_vec->interface~expand~9 proc~expand_ga_dynlong expand_ga_dynlong interface~expand~9->proc~expand_ga_dynlong

Called by

proc~~tem_stencil_communicate~~CalledByGraph proc~tem_stencil_communicate tem_stencil_communicate proc~redefine_halos redefine_halos proc~redefine_halos->proc~tem_stencil_communicate proc~communicate_elements communicate_elements proc~communicate_elements->proc~redefine_halos proc~tem_find_allelements tem_find_allElements proc~tem_find_allelements->proc~communicate_elements proc~tem_create_leveldesc tem_create_levelDesc proc~tem_create_leveldesc->proc~tem_find_allelements

Source Code

  subroutine tem_stencil_communicate( send, recv, elem, computeStencil,        &
    &                                 proc, commPattern, iStencil )
    ! ---------------------------------------------------------------------------
    !> send and recv communication buffers
    type( tem_communication_type ), intent(inout) :: send, recv
    !> communication pattern
    type(tem_commPattern_type), intent(in)     :: commPattern
    !> levelDesc element list
    type(tem_element_type), intent(inout)  :: elem
    !> Process description to use.
    type(tem_comm_env_type), intent(in) :: proc
    !> array of all stencils used in the simulation
    type(tem_stencilHeader_type), intent(in) :: computeStencil
    !> amount of values to communicate
    integer, intent(in) :: iStencil
    ! ---------------------------------------------------------------------------
    integer :: iStencilElem, iElem, nElems
    integer(kind=long_k), allocatable :: buffer(:)
    type(tem_stencilElement_type ) :: tStencil
    integer :: addedPos, iProc, elemPos, neighPos, stencilPos
    logical :: wasAdded
    ! ---------------------------------------------------------------------------
    ! Number of local elements in element list
    nElems = elem%tID%nVals
    write(dbgUnit(10),*)'in stencil communicate, nElems: ', nElems

    ! buffer which sends stencil neighbors to remove process
    allocate( buffer( nElems ))

    ! Set the temporary (empty) stencil
    call init( me        = tStencil,                                           &
      &        QQN       = computeStencil%QQN ,                                &
      &        headerPos = iStencil )

    ! add stencil headerPos for all halo elements depends on requested stencil
    ! communication
    do iProc = 1, recv%nProcs
      do iElem = 1, recv%nElemsProc( iProc )
        elemPos = recv%elemPos( iProc )%val( iElem )
        if( elem%stencil%val( elemPos )%nVals > 0 ) then
          ! @todo: SZ: this does not make sense since all elements have a stencil
          !            elem%stencil%val( elemPos )%nVals .gt. 0 for all elements
          ! Find the stencil corresponding to the current one ( iStencil )
          stencilPos = tem_stencil_getHeaderPos(                               &
            &                               me  = elem%stencil%val( elemPos ), &
            &                               val = iStencil )
          ! if none was found, use the most appropriate one (headerPos = 0)
          ! this assigns the first stencil to the halo element
          ! why not assign it right away to be 1?
          if( stencilPos <= 0 )then
            stencilPos = tem_stencil_getHeaderPos(                             &
              &                             me  = elem%stencil%val( elemPos ), &
              &                             val = 0 )
            elem%stencil%val( elemPos )%val( stencilPos )%headerPos = iStencil
          end if
        else
          ! if stencil is not initialized for this element append tStencil
          call append( me = elem%stencil%val( elemPos ), val = tStencil )
        end if
      end do
    end do

    ! --------------------------------------------------------------------------
    ! Do the communication for each stencil tID entry
    do iStencilElem = 1, computeStencil%QQN
      ! Fill the send buffer
      do iProc = 1, send%nProcs
        do iElem = 1, send%nElemsProc( iProc )
          elemPos = send%elemPos( iProc )%val( iElem )
          neighPos = elem%stencil%val( elemPos )%val( iStencil )%              &
            &                                            tIDpos( iStencilElem )
          buffer( elemPos ) = elem%neighID%val( elemPos )%val( neighPos )
        end do ! iElem send
      end do ! iProc send

      ! Do the exchange between all participating processes
      call commPattern%exchange_long( send         = send,                     &
        &                             recv         = recv,                     &
        &                             state        = buffer,                   &
        &                             message_flag = 0,                        &
        &                             comm         = proc%comm )

      ! Read from the buffer
      do iProc = 1, recv%nProcs
        do iElem = 1, recv%nElemsProc( iProc )
          elemPos = recv%elemPos( iProc )%val( iElem )
          ! Find the stencil corresponding to the current one (iStencil )
          stencilPos = tem_stencil_getHeaderPos(                               &
            &                               me  = elem%stencil%val( elemPos ), &
            &                               val = iStencil )

          if( stencilPos > 0 ) then
            ! Only append elements, if there is only one stencil for the element
            ! Append the treeID to the element's neighIDs
            call append( me       = elem%neighID%val( elemPos ),               &
              &          val      = buffer( elemPos),                          &
              &          pos      = addedPos,                                  &
              &          wasAdded = wasAdded )
            ! ... and store the appended position to the stencil
            elem%stencil%val( elemPos )%val( stencilPos )%                     &
              &                               tIDpos( iStencilElem ) = addedPos
          end if
        end do ! iElem recv
      end do ! iProc recv

    end do ! iStencilElem

  end subroutine tem_stencil_communicate