=== If no file names and line numbers are shown below, one can run addr2line -Cpfie my_exefile my_line_address to convert `my_line_address` (e.g., 0x4a6b) into file name and line number. Or one can use amrex/Tools/Backtrace/parse_bt.py. === Please note that the line number reported by addr2line may not be accurate. One can use readelf -wl my_exefile | grep my_line_address' to find out the offset for that line. 0: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x599816] amrex::BLBackTrace::print_backtrace_info(_IO_FILE*) /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_BLBackTrace.cpp:179:39 1: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x59b98d] amrex::BLBackTrace::handler(int) /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_BLBackTrace.cpp:85:15 2: /lib64/libc.so.6(+0x3da60) [0x7f9ca483aa60] __restore_rt ??:0 3: /lib64/libstdc++.so.6(_ZSt29_Rb_tree_insert_and_rebalancebPSt18_Rb_tree_node_baseS0_RS_+0x9a) [0x7f9ca4c0f20a] std::_Rb_tree_insert_and_rebalance(bool, std::_Rb_tree_node_base*, std::_Rb_tree_node_base*, std::_Rb_tree_node_base&) ??:0 4: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x59f8c8] std::_Rb_tree, std::allocator >, std::pair, std::allocator > const, amrex::TinyProfiler::Stats>, std::_Select1st, std::allocator > const, amrex::TinyProfiler::Stats> >, std::less, std::allocator > >, std::allocator, std::allocator > const, amrex::TinyProfiler::Stats> > >::_M_insert_node(std::_Rb_tree_node_base*, std::_Rb_tree_node_base*, std::_Rb_tree_node, std::allocator > const, amrex::TinyProfiler::Stats> >*) inlined at /usr/include/c++/10/bits/stl_tree.h:2468:29 in amrex::TinyProfiler::start() /usr/include/c++/10/bits/stl_tree.h:2369:7 std::_Rb_tree_iterator, std::allocator > const, amrex::TinyProfiler::Stats> > std::_Rb_tree, std::allocator >, std::pair, std::allocator > const, amrex::TinyProfiler::Stats>, std::_Select1st, std::allocator > const, amrex::TinyProfiler::Stats> >, std::less, std::allocator > >, std::allocator, std::allocator > const, amrex::TinyProfiler::Stats> > >::_M_emplace_hint_unique, std::allocator > const&>, std::tuple<> >(std::_Rb_tree_const_iterator, std::allocator > const, amrex::TinyProfiler::Stats> >, std::piecewise_construct_t const&, std::tuple, std::allocator > const&>&&, std::tuple<>&&) /usr/include/c++/10/bits/stl_tree.h:2468:29 std::map, std::allocator >, amrex::TinyProfiler::Stats, std::less, std::allocator > >, std::allocator, std::allocator > const, amrex::TinyProfiler::Stats> > >::operator[](std::__cxx11::basic_string, std::allocator > const&) /usr/include/c++/10/bits/stl_map.h:501:37 amrex::TinyProfiler::start() /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_TinyProfiler.cpp:104:47 5: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x4f9d95] std::__uniq_ptr_impl, std::default_delete > >::_M_ptr() const inlined at /usr/include/c++/10/bits/unique_ptr.h:422:27 in amrex::FabArray::ParallelCopy_finish() /usr/include/c++/10/bits/unique_ptr.h:173:58 std::unique_ptr, std::default_delete > >::get() const /usr/include/c++/10/bits/unique_ptr.h:422:27 std::unique_ptr, std::default_delete > >::operator bool() const /usr/include/c++/10/bits/unique_ptr.h:436:19 amrex::FabArray::ParallelCopy_finish() /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_FabArrayCommI.H:478:9 6: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x61986b] amrex::FabArray::ParallelCopy(amrex::FabArray const&, int, int, int, amrex::IntVect const&, amrex::IntVect const&, amrex::Periodicity const&, amrex::FabArrayBase::CpOp, amrex::FabArrayBase::CPC const*) inlined at /raid/testing/castro-gfortran/amrex//Src/AmrCore/AMReX_FillPatchUtil_I.H:69:28 in std::enable_if::value, void>::type amrex::FillPatchSingleLevel(amrex::MultiFab&, amrex::IntVect const&, double, amrex::Vector > const&, amrex::Vector > const&, int, int, int, amrex::Geometry const&, amrex::StateDataPhysBCFunct&, int) /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_FabArrayCommI.H:257:5 std::enable_if::value, void>::type amrex::FillPatchSingleLevel(amrex::MultiFab&, amrex::IntVect const&, double, amrex::Vector > const&, amrex::Vector > const&, int, int, int, amrex::Geometry const&, amrex::StateDataPhysBCFunct&, int) /raid/testing/castro-gfortran/amrex//Src/AmrCore/AMReX_FillPatchUtil_I.H:69:28 7: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x60ca82] std::vector >::~vector() inlined at /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_Vector.H:25:7 in amrex::FillPatchIterator::FillFromLevel0(double, int, int, int, int) /usr/include/c++/10/bits/stl_vector.h:680:15 amrex::Vector >::~Vector() /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_Vector.H:25:7 amrex::FillPatchIterator::FillFromLevel0(double, int, int, int, int) /raid/testing/castro-gfortran/amrex//Src/Amr/AMReX_AmrLevel.cpp:1100:18 8: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x60d8d2] amrex::FillPatchIterator::Initialize(int, double, int, int, int) /raid/testing/castro-gfortran/amrex//Src/Amr/AMReX_AmrLevel.cpp:1012:27 9: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x60e751] amrex::AmrLevel::FillPatch(amrex::AmrLevel&, amrex::MultiFab&, int, double, int, int, int, int) /raid/testing/castro-gfortran/amrex//Src/Amr/AMReX_AmrLevel.cpp:2214:19 10: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x42b170] Castro::expand_state(amrex::MultiFab&, double, int) /raid/testing/castro-gfortran/Castro//Source/driver/Castro.cpp:4594:22 11: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x44531e] Castro::do_advance_sdc(double, double, int, int) /raid/testing/castro-gfortran/Castro//Source/driver/Castro_advance_sdc.cpp:87:11 12: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x4412a2] Castro::advance(double, double, int, int) /raid/testing/castro-gfortran/Castro//Source/driver/Castro_advance.cpp:62:42 13: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x5ff849] std::vector >::operator[](unsigned long) inlined at /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_Vector.H:37:59 in amrex::Amr::timeStep(int, double, int, int, double) /usr/include/c++/10/bits/stl_vector.h:1046:39 amrex::Vector >::operator[](unsigned long) /raid/testing/castro-gfortran/amrex//Src/Base/AMReX_Vector.H:37:59 amrex::Amr::timeStep(int, double, int, int, double) /raid/testing/castro-gfortran/amrex//Src/Amr/AMReX_Amr.cpp:2018:68 14: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x5f5b70] amrex::Amr::coarseTimeStep(double) /raid/testing/castro-gfortran/amrex//Src/Amr/AMReX_Amr.cpp:2125:13 15: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x42135d] main /raid/testing/castro-gfortran/Castro//Source/driver/main.cpp:136:33 16: /lib64/libc.so.6(__libc_start_main+0xf2) [0x7f9ca48251e2] __libc_start_main ??:0 17: ./Castro2d.gnu.TEST.TPROF.MPI.TRUESDC.ex() [0x427b4e] _start ??:0 ===== TinyProfilers ====== main() Amr::coarseTimeStep() Amr::timeStep() Castro::advance() Castro::do_advance_sdc() Castro::expand_state() FillPatchIterator::Initialize FillPatchSingleLevel FabArray::ParallelCopy() FabArray::ParallelCopy_finish()