Reading Profile files in profile.* NODE 0;CONTEXT 0;THREAD 0: --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 4 30,889 1 2 30889894 .TAU application 100.0 30,885 30,885 1 0 30885243 .TAU application => caflaunch$MAIN$blk 100.0 30,885 30,885 1 0 30885243 caflaunch$MAIN$blk 0.0 0.045 0.045 1 0 45 .TAU application => CI0000$pgmio_ 0.0 0.045 0.045 1 0 45 CI0000$pgmio_ NODE 1;CONTEXT 0;THREAD 0: --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 12 30,692 1 63 30692491 .TAU application 99.7 20,301 30,591 1 339997 30591845 .TAU application => coback2 99.7 20,301 30,591 1 339997 30591845 coback2 30.6 9,380 9,380 40008 0 234 MPI_Barrier() 30.6 9,379 9,379 40005 0 234 .TAU application => coback2 => MPI_Barrier() 1.6 487 487 100001 0 5 MPI_Win_unlock() [THROTTLED] 1.6 486 486 99992 0 5 .TAU application => coback2 => MPI_Win_unlock() 0.8 237 237 1 0 237400 .TAU application => coback2 => pgmread 0.8 237 237 1 0 237400 pgmread 0.3 81 81 100001 0 1 MPI_Get() [THROTTLED] 0.3 81 81 99997 0 1 .TAU application => coback2 => MPI_Get() 0.2 68 68 100001 0 1 MPI_Win_lock() [THROTTLED] 0.2 68 68 99992 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 51 51 1 0 51348 .TAU application => MPI_Init() 0.2 51 51 1 0 51348 MPI_Init() 0.1 35 35 1 0 35096 .TAU application => MPI_Finalize() 0.1 35 35 1 0 35096 MPI_Finalize() 0.1 34 34 1 0 34453 .TAU application => coback2 => pgmwrite 0.1 34 34 1 0 34453 pgmwrite 0.0 1 1 10 0 147 MPI_Win_create() 0.0 0.988 0.988 4 0 247 .TAU application => coback2 => MPI_Win_create() 0.0 0.486 0.486 6 0 81 .TAU application => MPI_Win_create() 0.0 0.343 0.343 3 0 114 .TAU application => MPI_Barrier() 0.0 0.269 0.269 9 0 30 .TAU application => MPI_Win_free() 0.0 0.269 0.269 9 0 30 MPI_Win_free() 0.0 0.158 0.158 1 0 158 .TAU application => coback2 => pgmsize 0.0 0.158 0.158 1 0 158 pgmsize 0.0 0.125 0.125 9 0 14 .TAU application => MPI_Win_unlock() 0.0 0.065 0.065 1 0 65 .TAU application => CI0000$pgmio_ 0.0 0.065 0.065 1 0 65 CI0000$pgmio_ 0.0 0.062 0.062 1 0 62 .TAU application => MPI_Comm_dup() 0.0 0.062 0.062 1 0 62 MPI_Comm_dup() 0.0 0.048 0.048 9 0 5 MPI_Win_fence() 0.0 0.025 0.025 5 0 5 .TAU application => MPI_Win_fence() 0.0 0.023 0.023 7 0 3 .TAU application => MPI_Put() 0.0 0.023 0.023 4 0 6 .TAU application => coback2 => MPI_Win_fence() 0.0 0.023 0.023 7 0 3 MPI_Put() 0.0 0.02 0.02 9 0 2 .TAU application => MPI_Win_lock() 0.0 0.019 0.019 1 0 19 .TAU application => MPI_Comm_rank() 0.0 0.019 0.019 1 0 19 MPI_Comm_rank() 0.0 0.018 0.018 1 0 18 .TAU application => caflaunch$MAIN$blk 0.0 0.018 0.018 1 0 18 caflaunch$MAIN$blk 0.0 0.016 0.016 4 0 4 .TAU application => MPI_Get() 0.0 0.009 0.009 1 0 9 .TAU application => MPI_Alloc_mem() 0.0 0.009 0.009 1 0 9 .TAU application => MPI_Finalized() 0.0 0.009 0.009 1 0 9 MPI_Alloc_mem() 0.0 0.009 0.009 1 0 9 MPI_Finalized() 0.0 0.007 0.007 1 0 7 .TAU application => MPI_Comm_size() 0.0 0.007 0.007 1 0 7 MPI_Comm_size() 0.0 0.005 0.005 1 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.005 0.005 1 0 5 CI0000$coback2_$BLK NODE 2;CONTEXT 0;THREAD 0: --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 1,534 30,696 1 70 30696705 .TAU application 94.7 18,801 29,070 1 339994 29070279 .TAU application => coback2 94.7 18,801 29,070 1 339994 29070279 coback2 30.3 9,298 9,298 40008 0 232 MPI_Barrier() 30.3 9,297 9,297 40005 0 232 .TAU application => coback2 => MPI_Barrier() 1.9 580 580 100001 0 6 MPI_Win_unlock() [THROTTLED] 1.9 580 580 99992 0 6 .TAU application => coback2 => MPI_Win_unlock() 0.8 239 239 10 0 23946 MPI_Win_create() 0.8 238 238 4 0 59736 .TAU application => coback2 => MPI_Win_create() 0.3 90 90 100001 0 1 MPI_Win_lock() [THROTTLED] 0.3 90 90 99992 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 61 61 100001 0 1 MPI_Get() [THROTTLED] 0.2 61 61 99997 0 1 .TAU application => coback2 => MPI_Get() 0.2 51 51 1 0 51370 .TAU application => MPI_Init() 0.2 51 51 1 0 51370 MPI_Init() 0.1 39 39 1 0 39302 .TAU application => MPI_Finalize() 0.1 39 39 1 0 39302 MPI_Finalize() 0.0 0.515 0.515 6 0 86 .TAU application => MPI_Win_create() 0.0 0.281 0.281 9 0 31 .TAU application => MPI_Win_free() 0.0 0.281 0.281 9 0 31 MPI_Win_free() 0.0 0.193 0.193 3 0 64 .TAU application => MPI_Barrier() 0.0 0.159 0.159 9 0 18 .TAU application => MPI_Win_unlock() 0.0 0.071 0.071 1 0 71 .TAU application => MPI_Comm_dup() 0.0 0.071 0.071 1 0 71 MPI_Comm_dup() 0.0 0.046 0.046 1 0 46 .TAU application => CI0000$pgmio_ 0.0 0.046 0.046 1 0 46 CI0000$pgmio_ 0.0 0.037 0.037 9 0 4 MPI_Win_fence() 0.0 0.03 0.03 14 0 2 .TAU application => MPI_Put() 0.0 0.03 0.03 14 0 2 MPI_Put() 0.0 0.026 0.026 5 0 5 .TAU application => MPI_Win_fence() 0.0 0.021 0.021 9 0 2 .TAU application => MPI_Win_lock() 0.0 0.013 0.013 1 0 13 .TAU application => caflaunch$MAIN$blk 0.0 0.013 0.013 1 0 13 caflaunch$MAIN$blk 0.0 0.012 0.012 4 0 3 .TAU application => MPI_Get() 0.0 0.011 0.011 4 0 3 .TAU application => coback2 => MPI_Win_fence() 0.0 0.01 0.01 1 0 10 .TAU application => MPI_Alloc_mem() 0.0 0.01 0.01 1 0 10 .TAU application => MPI_Comm_rank() 0.0 0.01 0.01 1 0 10 MPI_Alloc_mem() 0.0 0.01 0.01 1 0 10 MPI_Comm_rank() 0.0 0.009 0.009 1 0 9 .TAU application => MPI_Finalized() 0.0 0.009 0.009 1 0 9 MPI_Finalized() 0.0 0.006 0.006 1 0 6 .TAU application => MPI_Comm_size() 0.0 0.006 0.006 1 0 6 MPI_Comm_size() 0.0 0.005 0.005 1 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.005 0.005 1 0 5 CI0000$coback2_$BLK NODE 3;CONTEXT 0;THREAD 0: --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 1,533 30,695 1 73 30695823 .TAU application 94.7 26,496 29,070 1 339994 29070281 .TAU application => coback2 94.7 26,496 29,070 1 339994 29070281 coback2 4.8 1,474 1,474 40008 0 37 MPI_Barrier() 4.8 1,474 1,474 40005 0 37 .TAU application => coback2 => MPI_Barrier() 2.3 698 698 100001 0 7 MPI_Win_unlock() [THROTTLED] 2.3 697 697 99992 0 7 .TAU application => coback2 => MPI_Win_unlock() 0.8 239 239 10 0 23941 MPI_Win_create() 0.8 238 238 4 0 59726 .TAU application => coback2 => MPI_Win_create() 0.3 97 97 100001 0 1 MPI_Win_lock() [THROTTLED] 0.3 97 97 99992 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 65 65 100001 0 1 MPI_Get() [THROTTLED] 0.2 65 65 99997 0 1 .TAU application => coback2 => MPI_Get() 0.2 51 51 1 0 51372 .TAU application => MPI_Init() 0.2 51 51 1 0 51372 MPI_Init() 0.1 39 39 1 0 39341 .TAU application => MPI_Finalize() 0.1 39 39 1 0 39341 MPI_Finalize() 0.0 0.504 0.504 6 0 84 .TAU application => MPI_Win_create() 0.0 0.278 0.278 9 0 31 .TAU application => MPI_Win_free() 0.0 0.278 0.278 9 0 31 MPI_Win_free() 0.0 0.223 0.223 3 0 74 .TAU application => MPI_Barrier() 0.0 0.153 0.153 9 0 17 .TAU application => MPI_Win_unlock() 0.0 0.094 0.094 1 0 94 .TAU application => CI0000$pgmio_ 0.0 0.094 0.094 1 0 94 CI0000$pgmio_ 0.0 0.065 0.065 1 0 65 .TAU application => MPI_Comm_dup() 0.0 0.065 0.065 1 0 65 MPI_Comm_dup() 0.0 0.033 0.033 17 0 2 .TAU application => MPI_Put() 0.0 0.033 0.033 17 0 2 MPI_Put() 0.0 0.028 0.028 9 0 3 MPI_Win_fence() 0.0 0.026 0.026 9 0 3 .TAU application => MPI_Win_lock() 0.0 0.02 0.02 1 0 20 .TAU application => MPI_Comm_rank() 0.0 0.02 0.02 1 0 20 MPI_Comm_rank() 0.0 0.015 0.015 5 0 3 .TAU application => MPI_Win_fence() 0.0 0.013 0.013 1 0 13 .TAU application => caflaunch$MAIN$blk 0.0 0.013 0.013 4 0 3 .TAU application => coback2 => MPI_Win_fence() 0.0 0.013 0.013 1 0 13 caflaunch$MAIN$blk 0.0 0.012 0.012 4 0 3 .TAU application => MPI_Get() 0.0 0.01 0.01 1 0 10 .TAU application => MPI_Finalized() 0.0 0.01 0.01 1 0 10 MPI_Finalized() 0.0 0.009 0.009 1 0 9 .TAU application => MPI_Alloc_mem() 0.0 0.009 0.009 1 0 9 MPI_Alloc_mem() 0.0 0.005 0.005 1 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.005 0.005 1 0 5 CI0000$coback2_$BLK 0.0 0.004 0.004 1 0 4 .TAU application => MPI_Comm_size() 0.0 0.004 0.004 1 0 4 MPI_Comm_size() NODE 4;CONTEXT 0;THREAD 0: --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 1,533 30,695 1 69 30695977 .TAU application 94.7 26,572 29,070 1 339994 29070271 .TAU application => coback2 94.7 26,572 29,070 1 339994 29070271 coback2 4.6 1,400 1,400 40008 0 35 MPI_Barrier() 4.6 1,400 1,400 40005 0 35 .TAU application => coback2 => MPI_Barrier() 2.3 698 698 100001 0 7 MPI_Win_unlock() [THROTTLED] 2.3 698 698 99992 0 7 .TAU application => coback2 => MPI_Win_unlock() 0.8 239 239 10 0 23933 MPI_Win_create() 0.8 238 238 4 0 59718 .TAU application => coback2 => MPI_Win_create() 0.3 95 95 100001 0 1 MPI_Win_lock() [THROTTLED] 0.3 95 95 99992 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 65 65 100001 0 1 MPI_Get() [THROTTLED] 0.2 65 65 99997 0 1 .TAU application => coback2 => MPI_Get() 0.2 51 51 1 0 51367 .TAU application => MPI_Init() 0.2 51 51 1 0 51367 MPI_Init() 0.1 39 39 1 0 39494 .TAU application => MPI_Finalize() 0.1 39 39 1 0 39494 MPI_Finalize() 0.0 0.459 0.459 6 0 76 .TAU application => MPI_Win_create() 0.0 0.278 0.278 9 0 31 .TAU application => MPI_Win_free() 0.0 0.278 0.278 9 0 31 MPI_Win_free() 0.0 0.183 0.183 9 0 20 .TAU application => MPI_Win_unlock() 0.0 0.076 0.076 3 0 25 .TAU application => MPI_Barrier() 0.0 0.07 0.07 1 0 70 .TAU application => MPI_Comm_dup() 0.0 0.07 0.07 1 0 70 MPI_Comm_dup() 0.0 0.068 0.068 1 0 68 .TAU application => CI0000$pgmio_ 0.0 0.068 0.068 1 0 68 CI0000$pgmio_ 0.0 0.028 0.028 13 0 2 .TAU application => MPI_Put() 0.0 0.028 0.028 13 0 2 MPI_Put() 0.0 0.026 0.026 9 0 3 MPI_Win_fence() 0.0 0.02 0.02 1 0 20 .TAU application => MPI_Comm_rank() 0.0 0.02 0.02 1 0 20 MPI_Comm_rank() 0.0 0.019 0.019 9 0 2 .TAU application => MPI_Win_lock() 0.0 0.015 0.015 5 0 3 .TAU application => MPI_Win_fence() 0.0 0.013 0.013 4 0 3 .TAU application => MPI_Get() 0.0 0.013 0.013 1 0 13 .TAU application => caflaunch$MAIN$blk 0.0 0.013 0.013 1 0 13 caflaunch$MAIN$blk 0.0 0.012 0.012 1 0 12 .TAU application => MPI_Alloc_mem() 0.0 0.012 0.012 1 0 12 MPI_Alloc_mem() 0.0 0.011 0.011 4 0 3 .TAU application => coback2 => MPI_Win_fence() 0.0 0.009 0.009 1 0 9 .TAU application => MPI_Finalized() 0.0 0.009 0.009 1 0 9 MPI_Finalized() 0.0 0.005 0.005 1 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.005 0.005 1 0 5 CI0000$coback2_$BLK 0.0 0.004 0.004 1 0 4 .TAU application => MPI_Comm_size() 0.0 0.004 0.004 1 0 4 MPI_Comm_size() FUNCTION SUMMARY (total): --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 4,618 2:33.670 5 277 30734178 .TAU application 76.7 1:32.172 1:57.802 4 1.35998E+06 29450669 .TAU application => coback2 76.7 1:32.172 1:57.802 4 1.35998E+06 29450669 coback2 20.1 30,885 30,885 5 0 6177060 .TAU application => caflaunch$MAIN$blk 20.1 30,885 30,885 5 0 6177060 caflaunch$MAIN$blk 14.0 21,553 21,553 160032 0 135 MPI_Barrier() 14.0 21,552 21,552 160020 0 135 .TAU application => coback2 => MPI_Barrier() 1.6 2,464 2,464 400004 0 6 MPI_Win_unlock() [THROTTLED] 1.6 2,463 2,463 399968 0 6 .TAU application => coback2 => MPI_Win_unlock() 0.5 719 719 40 0 17992 MPI_Win_create() 0.5 717 717 16 0 44857 .TAU application => coback2 => MPI_Win_create() 0.2 351 351 400004 0 1 MPI_Win_lock() [THROTTLED] 0.2 351 351 399968 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 273 273 400004 0 1 MPI_Get() [THROTTLED] 0.2 273 273 399988 0 1 .TAU application => coback2 => MPI_Get() 0.2 237 237 1 0 237400 .TAU application => coback2 => pgmread 0.2 237 237 1 0 237400 pgmread 0.1 205 205 4 0 51364 .TAU application => MPI_Init() 0.1 205 205 4 0 51364 MPI_Init() 0.1 153 153 4 0 38308 .TAU application => MPI_Finalize() 0.1 153 153 4 0 38308 MPI_Finalize() 0.0 34 34 1 0 34453 .TAU application => coback2 => pgmwrite 0.0 34 34 1 0 34453 pgmwrite 0.0 1 1 24 0 82 .TAU application => MPI_Win_create() 0.0 1 1 36 0 31 .TAU application => MPI_Win_free() 0.0 1 1 36 0 31 MPI_Win_free() 0.0 0.835 0.835 12 0 70 .TAU application => MPI_Barrier() 0.0 0.62 0.62 36 0 17 .TAU application => MPI_Win_unlock() 0.0 0.318 0.318 5 0 64 .TAU application => CI0000$pgmio_ 0.0 0.318 0.318 5 0 64 CI0000$pgmio_ 0.0 0.268 0.268 4 0 67 .TAU application => MPI_Comm_dup() 0.0 0.268 0.268 4 0 67 MPI_Comm_dup() 0.0 0.158 0.158 1 0 158 .TAU application => coback2 => pgmsize 0.0 0.158 0.158 1 0 158 pgmsize 0.0 0.139 0.139 36 0 4 MPI_Win_fence() 0.0 0.114 0.114 51 0 2 .TAU application => MPI_Put() 0.0 0.114 0.114 51 0 2 MPI_Put() 0.0 0.086 0.086 36 0 2 .TAU application => MPI_Win_lock() 0.0 0.081 0.081 20 0 4 .TAU application => MPI_Win_fence() 0.0 0.069 0.069 4 0 17 .TAU application => MPI_Comm_rank() 0.0 0.069 0.069 4 0 17 MPI_Comm_rank() 0.0 0.058 0.058 16 0 4 .TAU application => coback2 => MPI_Win_fence() 0.0 0.053 0.053 16 0 3 .TAU application => MPI_Get() 0.0 0.04 0.04 4 0 10 .TAU application => MPI_Alloc_mem() 0.0 0.04 0.04 4 0 10 MPI_Alloc_mem() 0.0 0.037 0.037 4 0 9 .TAU application => MPI_Finalized() 0.0 0.037 0.037 4 0 9 MPI_Finalized() 0.0 0.021 0.021 4 0 5 .TAU application => MPI_Comm_size() 0.0 0.021 0.021 4 0 5 MPI_Comm_size() 0.0 0.02 0.02 4 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.02 0.02 4 0 5 CI0000$coback2_$BLK FUNCTION SUMMARY (mean): --------------------------------------------------------------------------------------- %Time Exclusive Inclusive #Call #Subrs Inclusive Name msec total msec usec/call --------------------------------------------------------------------------------------- 100.0 923 30,734 1 55.4 30734178 .TAU application 76.7 18,434 23,560 0.8 271996 29450669 .TAU application => coback2 76.7 18,434 23,560 0.8 271996 29450669 coback2 20.1 6,177 6,177 1 0 6177060 .TAU application => caflaunch$MAIN$blk 20.1 6,177 6,177 1 0 6177060 caflaunch$MAIN$blk 14.0 4,310 4,310 32006.4 0 135 MPI_Barrier() 14.0 4,310 4,310 32004 0 135 .TAU application => coback2 => MPI_Barrier() 1.6 492 492 80000.8 0 6 MPI_Win_unlock() [THROTTLED] 1.6 492 492 79993.6 0 6 .TAU application => coback2 => MPI_Win_unlock() 0.5 143 143 8 0 17992 MPI_Win_create() 0.5 143 143 3.2 0 44857 .TAU application => coback2 => MPI_Win_create() 0.2 70 70 80000.8 0 1 MPI_Win_lock() [THROTTLED] 0.2 70 70 79993.6 0 1 .TAU application => coback2 => MPI_Win_lock() 0.2 54 54 80000.8 0 1 MPI_Get() [THROTTLED] 0.2 54 54 79997.6 0 1 .TAU application => coback2 => MPI_Get() 0.2 47 47 0.2 0 237400 .TAU application => coback2 => pgmread 0.2 47 47 0.2 0 237400 pgmread 0.1 41 41 0.8 0 51364 .TAU application => MPI_Init() 0.1 41 41 0.8 0 51364 MPI_Init() 0.1 30 30 0.8 0 38308 .TAU application => MPI_Finalize() 0.1 30 30 0.8 0 38308 MPI_Finalize() 0.0 6 6 0.2 0 34453 .TAU application => coback2 => pgmwrite 0.0 6 6 0.2 0 34453 pgmwrite 0.0 0.393 0.393 4.8 0 82 .TAU application => MPI_Win_create() 0.0 0.221 0.221 7.2 0 31 .TAU application => MPI_Win_free() 0.0 0.221 0.221 7.2 0 31 MPI_Win_free() 0.0 0.167 0.167 2.4 0 70 .TAU application => MPI_Barrier() 0.0 0.124 0.124 7.2 0 17 .TAU application => MPI_Win_unlock() 0.0 0.0636 0.0636 1 0 64 .TAU application => CI0000$pgmio_ 0.0 0.0636 0.0636 1 0 64 CI0000$pgmio_ 0.0 0.0536 0.0536 0.8 0 67 .TAU application => MPI_Comm_dup() 0.0 0.0536 0.0536 0.8 0 67 MPI_Comm_dup() 0.0 0.0316 0.0316 0.2 0 158 .TAU application => coback2 => pgmsize 0.0 0.0316 0.0316 0.2 0 158 pgmsize 0.0 0.0278 0.0278 7.2 0 4 MPI_Win_fence() 0.0 0.0228 0.0228 10.2 0 2 .TAU application => MPI_Put() 0.0 0.0228 0.0228 10.2 0 2 MPI_Put() 0.0 0.0172 0.0172 7.2 0 2 .TAU application => MPI_Win_lock() 0.0 0.0162 0.0162 4 0 4 .TAU application => MPI_Win_fence() 0.0 0.0138 0.0138 0.8 0 17 .TAU application => MPI_Comm_rank() 0.0 0.0138 0.0138 0.8 0 17 MPI_Comm_rank() 0.0 0.0116 0.0116 3.2 0 4 .TAU application => coback2 => MPI_Win_fence() 0.0 0.0106 0.0106 3.2 0 3 .TAU application => MPI_Get() 0.0 0.008 0.008 0.8 0 10 .TAU application => MPI_Alloc_mem() 0.0 0.008 0.008 0.8 0 10 MPI_Alloc_mem() 0.0 0.0074 0.0074 0.8 0 9 .TAU application => MPI_Finalized() 0.0 0.0074 0.0074 0.8 0 9 MPI_Finalized() 0.0 0.0042 0.0042 0.8 0 5 .TAU application => MPI_Comm_size() 0.0 0.0042 0.0042 0.8 0 5 MPI_Comm_size() 0.0 0.004 0.004 0.8 0 5 .TAU application => CI0000$coback2_$BLK 0.0 0.004 0.004 0.8 0 5 CI0000$coback2_$BLK