Improve instrumentation phases (#645)

* Added instrumentation phases in multiple places for `gap-v-transfer`, `nrn_spike_exchange` and `handle-forward-skip`
* Moved `timestep` phase to different places to cover all possible code execution flows
* Created different phases for `net_receive` and `net_buf_receive` functions of different mod files
* Created distinct phases for cpu2gpu and gpu2cpu transfers related to net receive and net send structs

Author: iomaganaris

CMakeLists.txt

@@ -390,6 +390,8 @@ if(CORENRN_ENABLE_CALIPER_PROFILING)
   include_directories(${caliper_INCLUDE_DIR})
   add_definitions("-DCORENEURON_CALIPER")
   set(CALIPER_LIB "caliper")
+  set_property(GLOBAL APPEND_STRING PROPERTY CORENEURON_LIB_LINK_FLAGS
+                                             " -L${caliper_LIB_DIR} -l${CALIPER_LIB}")
 endif()
 
 if(CORENRN_ENABLE_LIKWID_PROFILING)

coreneuron/apps/main1.cpp

@@ -598,6 +598,7 @@ extern "C" int run_solve_core(int argc, char** argv) {
 
         // handle forwardskip
         if (corenrn_param.forwardskip > 0.0) {
+            Instrumentor::phase p("handle-forward-skip");
             handle_forward_skip(corenrn_param.forwardskip, corenrn_param.prcellgid);
         }
 

coreneuron/gpu/nrn_acc_manager.cpp

@@ -554,6 +554,7 @@ struct comp {
 };
 
 static void net_receive_buffer_order(NetReceiveBuffer_t* nrb) {
+    Instrumentor::phase p_net_receive_buffer_order("net-receive-buf-order");
     if (nrb->_cnt == 0) {
         nrb->_displ_cnt = 0;
         return;
@@ -593,6 +594,7 @@ static void net_receive_buffer_order(NetReceiveBuffer_t* nrb) {
  * functional version.
  */
 void update_net_receive_buffer(NrnThread* nt) {
+    Instrumentor::phase p_update_net_receive_buffer("update-net-receive-buf");
     for (auto tml = nt->tml; tml; tml = tml->next) {
         // net_receive buffer to copy
         NetReceiveBuffer_t* nrb = tml->ml->_net_receive_buffer;
@@ -604,6 +606,7 @@ void update_net_receive_buffer(NrnThread* nt) {
 
 #ifdef _OPENACC
             if (nt->compute_gpu) {
+                Instrumentor::phase p_net_receive_buffer_order("net-receive-buf-cpu2gpu");
                 // note that dont update nrb otherwise we lose pointers
 
                 /* update scalar elements */
@@ -636,6 +639,7 @@ void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb) {
     }
 
     if (nsb->_cnt) {
+        Instrumentor::phase p_net_receive_buffer_order("net-send-buf-gpu2cpu");
         acc_update_self(nsb->_sendtype, sizeof(int) * nsb->_cnt);
         acc_update_self(nsb->_vdata_index, sizeof(int) * nsb->_cnt);
         acc_update_self(nsb->_pnt_index, sizeof(int) * nsb->_cnt);

coreneuron/network/netcvode.cpp

@@ -16,6 +16,7 @@
 #include "coreneuron/network/netcvode.hpp"
 #include "coreneuron/network/netpar.hpp"
 #include "coreneuron/utils/ivocvect.hpp"
+#include "coreneuron/utils/profile/profiler_interface.h"
 #include "coreneuron/nrniv/nrniv_decl.h"
 #include "coreneuron/io/output_spikes.hpp"
 #include "coreneuron/utils/nrn_assert.h"
@@ -385,6 +386,9 @@ void NetCon::deliver(double tt, NetCvode* /* ns */, NrnThread* nt) {
     nt->_t = tt;
 
     // printf("NetCon::deliver t=%g tt=%g %s\n", t, tt, pnt_name(target_));
+    std::string ss("net-receive-");
+    ss += nrn_get_mechname(typ);
+    Instrumentor::phase p_get_pnt_receive(ss.c_str());
     (*corenrn.get_pnt_receive()[typ])(target_, u.weight_index_, 0);
 #ifdef DEBUG
     if (errno && nrn_errno_check(typ))
@@ -518,6 +522,7 @@ double PreSyn::value(NrnThread* nt) {
 }
 
 void NetCvode::check_thresh(NrnThread* nt) {  // for default method
+    Instrumentor::phase p("check-threshold");
     double teps = 1e-10;
 
     nt->_net_send_buffer_cnt = 0;
@@ -697,6 +702,9 @@ void NetCvode::deliver_net_events(NrnThread* nt) {  // for default method
     update_net_receive_buffer(nt);
 
     for (auto& net_buf_receive: corenrn.get_net_buf_receive()) {
+        std::string ss("net-buf-receive-");
+        ss += nrn_get_mechname(net_buf_receive.second);
+        Instrumentor::phase p_net_buf_receive(ss.c_str());
         (*net_buf_receive.first)(nt);
     }
 }

coreneuron/network/netpar.cpp

@@ -24,6 +24,7 @@
 #include "coreneuron/network/multisend.hpp"
 #include "coreneuron/utils/nrn_assert.h"
 #include "coreneuron/utils/nrnoc_aux.hpp"
+#include "coreneuron/utils/profile/profiler_interface.h"
 #include "coreneuron/utils/utils.hpp"
 
 #if NRNMPI
@@ -301,6 +302,7 @@ void nrn_spike_exchange_init() {
 
 #if NRNMPI
 void nrn_spike_exchange(NrnThread* nt) {
+    Instrumentor::phase p_spike_exchange("spike-exchange");
     if (!active_) {
         return;
     }

coreneuron/sim/fadvance_core.cpp

@@ -88,6 +88,7 @@ void dt2thread(double adt) { /* copied from nrnoc/fadvance.c */
 }
 
 void nrn_fixed_step_minimal() { /* not so minimal anymore with gap junctions */
+    Instrumentor::phase p_timestep("timestep");
     if (t != nrn_threads->_t) {
         dt2thread(-1.);
     } else {
@@ -96,7 +97,10 @@ void nrn_fixed_step_minimal() { /* not so minimal anymore with gap junctions */
     nrn_thread_table_check();
     nrn_multithread_job(nrn_fixed_step_thread);
     if (nrn_have_gaps) {
-        nrnmpi_v_transfer();
+        {
+            Instrumentor::phase p_gap("gap-v-transfer");
+            nrnmpi_v_transfer();
+        }
         nrn_multithread_job(nrn_fixed_step_lastpart);
     }
 #if NRNMPI
@@ -148,7 +152,6 @@ void nrn_fixed_step_group_minimal(int total_sim_steps) {
     int step_group_end = 0;
 
     ProgressBar progress_bar(step_group_n);
-
     while (step_group_end < step_group_n) {
         nrn_multithread_job(nrn_fixed_step_group_thread,
                             step_group_n,
@@ -179,6 +182,7 @@ static void nrn_fixed_step_group_thread(NrnThread* nth,
                                         int& step_group_end) {
     nth->_stop_stepping = 0;
     for (int i = step_group_begin; i < step_group_max; ++i) {
+        Instrumentor::phase p_timestep("timestep");
         nrn_fixed_step_thread(nth);
         if (nth->_stop_stepping) {
             if (nth->id == 0) {
@@ -339,10 +343,8 @@ void nrncore2nrn_send_values(NrnThread* nth) {
 static void* nrn_fixed_step_thread(NrnThread* nth) {
     /* check thresholds and deliver all (including binqueue)
        events up to t+dt/2 */
-    Instrumentor::phase_begin("timestep");
-
     {
-        Instrumentor::phase p("deliver_events");
+        Instrumentor::phase p("deliver-events");
         deliver_net_events(nth);
     }
 
@@ -361,7 +363,7 @@ static void* nrn_fixed_step_thread(NrnThread* nth) {
         fixed_play_continuous(nth);
 
         {
-            Instrumentor::phase p("setup_tree_matrix");
+            Instrumentor::phase p("setup-tree-matrix");
             setup_tree_matrix_minimal(nth);
         }
 
@@ -371,7 +373,7 @@ static void* nrn_fixed_step_thread(NrnThread* nth) {
         }
 
         {
-            Instrumentor::phase p("second_order_cur");
+            Instrumentor::phase p("second-order-cur");
             second_order_cur(nth, secondorder);
         }
 
@@ -383,7 +385,6 @@ static void* nrn_fixed_step_thread(NrnThread* nth) {
     if (!nrn_have_gaps) {
         nrn_fixed_step_lastpart(nth);
     }
-    Instrumentor::phase_end("timestep");
     return nullptr;
 }
 
@@ -408,7 +409,7 @@ void* nrn_fixed_step_lastpart(NrnThread* nth) {
     }
 
     {
-        Instrumentor::phase p("deliver_events");
+        Instrumentor::phase p("deliver-events");
         nrn_deliver_events(nth); /* up to but not past texit */
     }
 

coreneuron/sim/finitialize.cpp

@@ -66,6 +66,7 @@ void nrn_finitialize(int setv, double v) {
     }
 
     if (nrn_have_gaps) {
+        Instrumentor::phase p("gap-v-transfer");
         nrnmpi_v_transfer();
         for (int i = 0; i < nrn_nthread; ++i) {
             nrnthread_v_transfer(nrn_threads + i);