Programs linked against root libraries *sometimes* hang on startup

From: Bruce O'Neel <bruce.oneel_at_obs.unige.ch>
Date: Fri, 05 Aug 2005 12:14:09 +0200 (MEST)


Hi,

Sorry the subject is so long.

Our software package builds a lot of program which link against root libraries. On Solaris 5.8, sometimes, with root 4.03/02, the programs start and then hang with a thread deadlock. A pstack of a few different cases is below. A line of === separates the different cases.

One of the things I find interesting is that, with the exception of the first case, none of the programs seem to even have gotten to root or cint code when they hang. They all seem to be hanging in the C++ startup.

Any ideas anyone? Does anyone else see this? I'm about to start trying other versions of root, but, I'd be happier if I knew what the problem was rather than just have it disapear by accident.

Thanks very much in advance.

cheers

bruce


                 12147 sh -c 

/home/isdc/isdc_cms/work/tc_cases/isdcmath-SunOS/bin/ibis_gti swgDOL="./s

8040: /isdc/integration/osa_int/current/bin/ibis_correction swgDOL=./scw/010
----------------- lwp# 1 / thread# 3 --------------------
 fcf9bd34 lwp_sema_wait (fcb0de30)

 fd9c9ab0 _park    (fcb0de30, fd9ee000, 0, fcb0dd70, 24d98, 0) + 114
 fd9c9778 _swtch   (fcb0dd70, fcb0dd70, fd9ee000, 5, 1000, 1) + 424
 fd9cddb4 _reap_wait (fd9f29e8, 20528, 0, fd9ee000, 0, 0) + 38
 fd9cdb0c _reaper  (fd9eee38, fd9f4748, fd9f29e8, fd9eee10, 1, fe400000) + 38
 fd9db6e0 _thread_start (0, 0, 0, 0, 0, 0) + 40

----------------- lwp# 2 / thread# 2 --------------------
 fcf9b684 signotifywait ()
 fd9ced4c _dynamiclwps (fd9ee000, 59, 0, 0, ffbeeac4, 4) + 1c  fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
----------------- lwp# 3 / thread# 1 --------------------
 fcf9bc70 lwp_mutex_lock (fd9f5560)
 fd9cd004 _sched_lock (fd9ee000, 16cac, fe2610e4, 0, ff3e2660, fde2841e) + 24  fd9d7394 _tdb_sync_obj_deregister (ffbee2e8, fd9ee000, 3, ff3e2660, 29360, ff3c384c) + 44  fd9cadf8 mutex_destroy (ffbee2e8, 0, ffffffff, ffffffff, fe56b4e8, 2b) + 1c
 fde31728 __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___I__getRep6MII_pnH__rwstdM__string_ref4Ccn0B_n0C____ (ab9910, 1, 1, 294, ab9938, 0) + 12c
 fde2f658 __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___Hreplace6MIIpkcIII_pc_ (ab8ec8, 0, 0, fd774e42, 0, fde508e0) + 3f0
 fde2dcbc __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___2G6Mpkc_r1_ (aacfe0, fd774e42, ab8ec8, ab8eb8, fd78f000, fde508e0) + 1ac
 fd713194 __1cDstdGlocaleEinit6F_v_ (10, 670, fd78fa60, 800, fd7884d0, 1) + 74
 fd708db0 __1cDstdNbasic_istream4Ccn0ALchar_traits4Cc___2t6Mn0AIios_baseJEmptyCtor__v_ (fd78e388, 0, 7f790, fd78ab84, fd78ab90, fd78e3a8) + 74
 fd70a41c __1cU__STATIC_CONSTRUCTOR6F_v_ (400, 7e0f4, fd78dc00, 1, fd7884d0, fd78e3f0) + 44
 fd7237ec _init    (0, ab87e8, 1, ff3e2660, 29360, ff3bad78) + 1e0
 ff3bad80 call_init (fe261998, 3, ff3e21f0, ff3e2660, 400000, fe26199c) + 180
 ff3c3844 elf_bndr (fd615308, 1b9, fe7d0390, 0, ff3e2660, fd64b9f6) + 3bc
 ff3b2980 elf_rtbndr (fd709fe0, 0, 0, 0, 0, 0) + 1c
 fd78909c ???????? (0, 0, 0, 0, 0, 0)
 fd709fe0 __SUNW_init_iostreams (fd78e9c4, fd709fdc, ffffffff, fd78e9c4, fe56fdf8, fe56bca4) + 4
 fe5573a0 __1cH__CimplKcplus_init6F_v_ (0, fcfb801c, fde3c210, fde3c298, fffed3f0, 0) + 10c  fde3c2cc _init (0, ab87e8, 1, ff3e2660, 29360, ff3bad78) + c8  ff3bad80 call_init (fe261798, 1, ff3e21f0, ff3e2660, 400000, fe2617ac) + 180
 ff3ba9f8 setup    (ff3a0018, ff3e2000, ff3e20d0, ff3e3c30, ff3e2030, ff3e2660) + 730
 ff3c4d58 _setup   (d4, ff3e2660, ff3a0018, 5, 100d4, 0) + 8a8
 ff3b2938 _rt_boot (0, 0, 0, 0, 0, 0) + 88
 00000000 ???????? (0, 0, 0, 0, 0, 0)

----------------- lwp# 4 --------------------------------
 fcf9bc70 lwp_mutex_lock (fd9f5560)
 fd9cd004 _sched_lock (fd9ee000, 177e0, 0, 0, 0, 0) + 24
 fd9d6830 _sc_add  (fdc75d70, fd9ee000, fdc75cac, 0, 0, 0) + 14
 fd9d6944 _sc_setup (fdc75d70, fd9ef690, fd9ef6a8, 3, fd9ee000, 1) + bc
 fd9ca720 _lwp_start (fdc75d70, 0, 6000, fde15b74, 0, 0) + c
 fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
----------------- lwp# 5 --------------------------------
 fcf9bc70 lwp_mutex_lock (fd9f5560)
 fd9cd004 _sched_lock (fd9ee000, 0, 0, 0, 0, 0) + 24
 fd9cb070 _mutex_adaptive_lock (fd9f98ec, 4c00, 1000, fffeffff, 1, 4d58) + 90
 fd9cae78 _cmutex_lock (fe56fde0, fd9ee000, 8, fe555174, 74657200, 74657200) + 70
 fe555174 _ex_register (fe56fde0, 178e0, 16388, fd9ee000, fffedd88, fffed7ac) + 2c
 fe5591ac _init    (0, fdc63d70, 1, ff3e2660, 29360, ff3bad78) + 94
 ff3bad80 call_init (fe261950, 3, ff3e21f0, ff3e2660, 400000, fe261954) + 180  ff3c3844 elf_bndr (fe5513e4, 191, fe7d0780, 0, ff3e2660, 7eeaf) + 3bc  ff3b2980 elf_rtbndr (fcf9c508, 1, 8, fd9ee000, 5f72656d, 5f5f6c77) + 1c  00494f54 ???????? (fcfba5ac, 178e0, 0, fd9ee000, 293b0, 0)  fcf9c508 _init (0, fdc63d70, 1, ff3e2660, 29360, ff3bad78) + 2c  ff3bad80 call_init (fe261878, 3, ff3e21f0, ff3e2660, 400000, fe26187c) + 180  ff3c3844 elf_bndr (fcf06b58, 4, fe7d0dac, 0, ff3e2660, fd9c3df6) + 3bc  ff3b2980 elf_rtbndr (fd9c9098, fd9f9d00, 0, fd9cb2b4, 0, 0) + 1c  fd9ee434 ???????? (fd9f5550, fd9f5560, fd9eedb8, 0, 0, 0)  fd9c9098 _age (fdc63d70, fd9eeda4, fd9ee000, 3, fd9ee000, 1) + 74  fd9ca72c _lwp_start (fdc63d70, 0, 4000, fcc0fc2c, 0, 0) + 18  fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
----------------- lwp# 6 --------------------------------
 fcf9bc70 lwp_mutex_lock (fd9f5560)
 fd9cd004 _sched_lock (fd9ee000, 177e0, 0, 0, 0, 0) + 24
 fd9d6830 _sc_add  (fd9b3d70, fd9ee000, fd9b3cac, 0, 0, 0) + 14
 fd9d6944 _sc_setup (fd9b3d70, fd9ef690, fd9ef6a8, 3, fd9ee000, 1) + bc
 fd9ca720 _lwp_start (fd9b3d70, 0, 4000, fcc0fc2c, 0, 0) + c
 fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
  1. /isdc/isdc_lib/CHECK_DELIVERIES/002/bin/o_cor_science swgDOL=scw/01020
    ----------------- lwp# 1 / thread# 3 --------------------
    fcf9bd34 lwp_sema_wait (fcb0de30) fd9c9ab0 _park (fcb0de30, fd9ee000, 0, fcb0dd70, 24d98, 0) + 114 fd9c9778 _swtch (fcb0dd70, fcb0dd70, fd9ee000, 5, 1000, 1) + 424 fd9cddb4 _reap_wait (fd9f29e8, 20528, 0, fd9ee000, 0, 0) + 38 fd9cdb0c _reaper (fd9eee38, fd9f4748, fd9f29e8, fd9eee10, 1, fe400000) + 38 fd9db6e0 _thread_start (0, 0, 0, 0, 0, 0) + 40
    ----------------- lwp# 2 / thread# 2 --------------------
    fcf9b684 signotifywait () fd9ced4c _dynamiclwps (fd9ee000, 59, 0, 0, 0, 0) + 1c fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
    ----------------- lwp# 3 --------------------------------
    fcf9bc70 lwp_mutex_lock (fd9f5560) fd9cd004 _sched_lock (fd9ee000, 0, 0, 0, 0, 0) + 24 fd9cb070 _mutex_adaptive_lock (fd9f98ec, 4c00, 1000, fffeffff, 1, 4d58) + 90 fd9cae78 _cmutex_lock (fe56fde0, fd9ee000, 0, fe555174, 0, 0) + 70 fe555174 _ex_register (fe56fde0, 178e0, 16388, fd9ee000, fffedd88, fffed7ac) + 2c fe5591ac _init (0, fde15d70, 1, ff3e2660, 29360, ff3bad78) + 94 ff3bad80 call_init (fe261c7c, 3, ff3e21f0, ff3e2660, 400000, fe261c80) + 180 ff3c3844 elf_bndr (fe5513e4, fe, fe7d0aec, 0, ff3e2660, 6ac48) + 3bc ff3b2980 elf_rtbndr (fcf9c508, 1, 0, 0, 0, 0) + 1c 0046e8cc ???????? (fcfba5ac, 178e0, 0, fd9ee000, 293b0, 1) fcf9c508 _init (0, fde15d70, 1, ff3e2660, 29360, ff3bad78) + 2c ff3bad80 call_init (fe261ba4, 3, ff3e21f0, ff3e2660, 400000, fe261ba8) + 180 ff3c3844 elf_bndr (fcf06b58, 4, fe7d1118, 0, ff3e2660, fd9c3df6) + 3bc ff3b2980 elf_rtbndr (fd9c9098, fd9f9d00, 0, fd9ee000, c0, 0) + 1c fd9ee434 ???????? (fd9f5550, fd9f5560, fd9eedb8, fd9ee000, 0, 0) fd9c9098 _age (fde15d70, fd9eeda4, fd9ee000, 0, 0, 4) + 74 fcf99284 _door_return (fde15cb0, fd9ca72c, 0, 0, 0, 0) + 68
    ----------------- lwp# 4 / thread# 1 --------------------
    fcf9bc70 lwp_mutex_lock (fd9f5560) fd9cd004 _sched_lock (fd9ee000, 16cac, fe261410, 0, ff3e2660, fde2841e) + 24 fd9d7394 _tdb_sync_obj_deregister (ffbec6a0, fd9ee000, 3, ff3e2660, 29360, ff3c384c) + 44 fd9cadf8 mutex_destroy (ffbec6a0, 0, ffffffff, ffffffff, fe56b4e8, 2b) + 1c fde31728 __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___I__getRep6MII_pnH__rwstdM__string_ref4Ccn0B_n0C____ (a96cb8, 1, 1, 294, a96ce0, 0) + 12c fde2f658 __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___Hreplace6MIIpkcIII_pc_ (a96270, 0, 0, fd774e42, 0, fde508e0) + 3f0 fde2dcbc __1cDstdMbasic_string4Ccn0ALchar_traits4Cc__n0AJallocator4Cc___2G6Mpkc_r1_ (a83bc0, fd774e42, a96270, a96260, fd78f000, fde508e0) + 1ac fd713194 __1cDstdGlocaleEinit6F_v_ (10, 670, fd78fa60, 800, fd7884d0, 1) + 74 fd708db0 __1cDstdNbasic_istream4Ccn0ALchar_traits4Cc___2t6Mn0AIios_baseJEmptyCtor__v_ (fd78e388, 0, 7f790, fd78ab84, fd78ab90, fd78e3a8) + 74 fd70a41c __1cU__STATIC_CONSTRUCTOR6F_v_ (400, 7e0f4, fd78dc00, 1, fd7884d0, fd78e3f0) + 44 fd7237ec _init (0, a95b90, 1, ff3e2660, 29360, ff3bad78) + 1e0 ff3bad80 call_init (fe261cc4, 3, ff3e21f0, ff3e2660, 400000, fe261cc8) + 180 ff3c3844 elf_bndr (fd615308, 1b9, fe7d06fc, 0, ff3e2660, fd64b9f6) + 3bc ff3b2980 elf_rtbndr (fd709fe0, 0, 0, 0, 0, 0) + 1c fd78909c ???????? (0, 0, 0, 0, 0, 0) fd709fe0 __SUNW_init_iostreams (fd78e9c4, fd709fdc, ffffffff, fd78e9c4, fe56fdf8, fe56bca4) + 4 fe5573a0 __1cH__CimplKcplus_init6F_v_ (0, fcfb801c, fde3c210, fde3c298, fffed3f0, 0) + 10c fde3c2cc _init (0, a95b90, 1, ff3e2660, 29360, ff3bad78) + c8 ff3bad80 call_init (fe261ac4, 1, ff3e21f0, ff3e2660, 400000, fe261ad8) + 180 ff3ba9f8 setup (ff3a0018, ff3e2000, ff3e20d0, ff3e3c30, ff3e2030, ff3e2660) + 730 ff3c4d58 _setup (d4, ff3e2660, ff3a0018, 5, 100d4, 0) + 8a8 ff3b2938 _rt_boot (0, 0, 0, 0, 0, 0) + 88 00000000 ???????? (0, 0, 0, 0, 0, 0)
    ----------------- lwp# 5 --------------------------------
    fcf9bc70 lwp_mutex_lock (fd9f5560) fd9cd004 _sched_lock (fd9ee000, 177e0, 0, 0, 0, 0) + 24 fd9d6830 _sc_add (fdc63d70, fd9ee000, fdc63cac, 0, 0, 0) + 14 fd9d6944 _sc_setup (fdc63d70, fd9ef690, fd9ef6a8, 3, fd9ee000, 1) + bc fd9ca720 _lwp_start (fdc63d70, 0, 4000, fcc0fc2c, 0, 0) + c fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
    ----------------- lwp# 6 --------------------------------
    fcf9bc70 lwp_mutex_lock (fd9f5560) fd9cd004 _sched_lock (fd9ee000, 177e0, 0, 0, 0, 0) + 24 fd9d6830 _sc_add (fd9b5d70, fd9ee000, fd9b5cac, 0, 0, 0) + 14 fd9d6944 _sc_setup (fd9b5d70, fd9ef690, fd9ef6a8, 3, fd9ee000, 1) + bc fd9ca720 _lwp_start (fd9b5d70, 0, 6000, fde15b74, 0, 0) + c fd9d2028 thr_yield (0, 0, 0, 0, 0, 0) + 8c
-- 
"We didn't just track down that bug, we left evidence of its extermination as 
a warning to other bugs" - Dan Lyke - flutterby

Bruce O'Neel                       phone:  +41 22 379 21 57
INTEGRAL Science Data Centre               +41 22 379 21 00 (switchb.)
Chemin d'Ecogia 16                 fax:    +41 22 379 21 35
CH-1290 VERSOIX                    e-mail: Bruce.Oneel_at_obs.unige.ch
Switzerland                        WWW:    http://isdc.unige.ch/
Received on Fri Aug 05 2005 - 12:14:43 MEST

This archive was generated by hypermail 2.2.0 : Tue Jan 02 2007 - 14:45:11 MET