=== If no file names and line numbers are shown below, one can run addr2line -Cpfie my_exefile my_line_address to convert `my_line_address` (e.g., 0x4a6b) into file name and line number. Or one can use amrex/Tools/Backtrace/parse_bt.py. === Please note that the line number reported by addr2line may not be accurate. One can use readelf -wl my_exefile | grep my_line_address' to find out the offset for that line. 0: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7292d6] amrex::BLBackTrace::print_backtrace_info(_IO_FILE*) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_BLBackTrace.cpp:174 1: ./Castro2d.pgi.TPROF.CUDA.ex() [0x72aea2] amrex::BLBackTrace::handler(int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_BLBackTrace.cpp:80 2: ./Castro2d.pgi.TPROF.CUDA.ex() [0x65f808] amrex::Gpu::Device::streamSynchronize() tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_GpuDevice.cpp:621 3: ./Castro2d.pgi.TPROF.CUDA.ex() [0x56202a] clean_bc inlined at tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro_bc_fill_nd.cpp:294 in ca_statefill(amrex::Box const&, amrex::FArrayBox&, int, int, amrex::Geometry const&, double, amrex::Vector > const&, int, int) /raid/testing/castro-gpu/Castro//Source/driver/Castro_generic_fill.H:25 ca_statefill(amrex::Box const&, amrex::FArrayBox&, int, int, amrex::Geometry const&, double, amrex::Vector > const&, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro_bc_fill_nd.cpp:294 4: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7be22b] amrex::StateData::FillBoundary(amrex::Box const&, amrex::FArrayBox&, double, amrex::Geometry const&, int, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_StateData.cpp:556 5: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7c0d45] amrex::StateDataPhysBCFunct::operator()(amrex::MultiFab&, int, int, amrex::IntVect const&, double, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_StateData.cpp:914 6: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7b67e4] std::enable_if::value, void>::type amrex::FillPatchSingleLevel(amrex::MultiFab&, amrex::IntVect const&, double, amrex::Vector > const&, amrex::Vector > const&, int, int, int, amrex::Geometry const&, amrex::StateDataPhysBCFunct&, int) /raid/testing/castro-gpu/amrex//Src/AmrCore/AMReX_FillPatchUtil_I.H:53 7: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7ad48d] std::vector >::~vector() inlined at /raid/testing/castro-gpu/amrex//Src/Base/AMReX_Vector.H:22 in amrex::FillPatchIterator::FillFromLevel0(double, int, int, int, int) /opt/gcc/gcc/7.3/include/c++/7.3.0/bits/stl_vector.h:434 amrex::Vector >::~Vector() /raid/testing/castro-gpu/amrex//Src/Base/AMReX_Vector.H:22 amrex::FillPatchIterator::FillFromLevel0(double, int, int, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_AmrLevel.cpp:1101 8: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7adc12] amrex::FillPatchIterator::Initialize(int, double, int, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_AmrLevel.cpp:1013 9: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7ae2e3] amrex::AmrLevel::FillPatch(amrex::AmrLevel&, amrex::MultiFab&, int, double, int, int, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_AmrLevel.cpp:2214 10: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7ae6b1] amrex::AmrLevel::derive(std::__cxx11::basic_string, std::allocator > const&, double, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_AmrLevel.cpp:1663 11: ./Castro2d.pgi.TPROF.CUDA.ex() [0x43bdee] Castro::derive(std::__cxx11::basic_string, std::allocator > const&, double, int) inlined at tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro.cpp:3513 in Castro::apply_tagging_func(amrex::TagBoxArray&, double, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro.cpp:3748 Castro::apply_tagging_func(amrex::TagBoxArray&, double, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro.cpp:3513 12: ./Castro2d.pgi.TPROF.CUDA.ex() [0x43dfdb] Castro::errorEst(amrex::TagBoxArray&, int, int, double, int, int) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/Castro.cpp:3311 13: ./Castro2d.pgi.TPROF.CUDA.ex() [0x78d366] amrex::AmrMesh::MakeNewGrids(int, double, int&, amrex::Vector >&) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_AmrMesh.cpp:574 14: ./Castro2d.pgi.TPROF.CUDA.ex() [0x79931d] amrex::Amr::grid_places(int, double, int&, amrex::Vector >&) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:2924 15: ./Castro2d.pgi.TPROF.CUDA.ex() [0x79a5bd] amrex::Amr::bldFineLevels(double) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:2973 16: ./Castro2d.pgi.TPROF.CUDA.ex() [0x79add6] amrex::Amr::FinalizeInit(double, double) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:1301 17: ./Castro2d.pgi.TPROF.CUDA.ex() [0x7a03aa] amrex::Amr::initialInit(double, double, amrex::BoxArray const*, amrex::Vector > const*) inlined at tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:1116 in amrex::Amr::init(double, double) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:1235 amrex::Amr::init(double, double) tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/AMReX_Amr.cpp:1116 18: ./Castro2d.pgi.TPROF.CUDA.ex() [0x40d6d9] main tmp_build_dir/s/2d.pgi.TPROF.CUDA.EXE/main.cpp:131 19: /lib64/libc.so.6(__libc_start_main+0xf3) [0x7f68f253c1a3] __libc_start_main ??:0 20: ./Castro2d.pgi.TPROF.CUDA.ex() [0x41db2e] _start ??:0 ===== TinyProfilers ====== main() Amr::init() Amr::initialInit() Amr::FinalizeInit() Amr::bldFineLevels() Amr::grid_places() AmrMesh::MakeNewGrids() Castro::errorEst() Castro::apply_tagging_func() Castro::derive() FillPatchIterator::Initialize FillPatchSingleLevel StateDataPhysBCFunct::() StateData::FillBoundary(geom)