This is a useful script for crawling the Weblogic domain logs and printing threads that have been stuck, as well as the duration they were stuck. It doesn’t include the initial ten minute (default) window before a log entry is written indicating the thread is stuck.
import glob, time
from datetime import timedelta
stuck = dict()
for f in sorted(glob.glob("./RPMDomain/servers/rpm_server1/logs/rpm_server1.log*"),reverse=True):
for line in open(f,"r"):
tmp = line.split()
if len(tmp) > 27 and tmp[23].find("STUCK") > -1:
stuck[tmp[25]] = tmp[0].replace("####<","") + " " + tmp[1] + " " + tmp[2] + " " + tmp[3] + " " + tmp[4]
elif len(tmp) > 32 and tmp[32].find("unstuck") > -1 and tmp[25] in stuck:
unstuck = tmp[0].replace("####<","") + " " + tmp[1] + " " + tmp[2] + " " + tmp[3] + " " + tmp[4]
l=time.strptime(stuck[tmp[25]],"%b %d, %Y %H:%M:%S %p")
l2=time.strptime(unstuck,"%b %d, %Y %H:%M:%S %p")
diff = time.mktime(l2) - time.mktime(l)
if diff >= 0:
m, s = divmod(diff, 60)
h, m = divmod(m, 60)
tm = "%d:%02d:%02d" % (h, m, s)
print tmp[25],"stuck at", stuck[tmp[25]],", unstuck at",unstuck,", stuck for",tm
This will result in output such as…
'25' stuck at Sep 14, 2016 9:39:29 AM, unstuck at Sep 14, 2016 9:44:42 AM , stuck for 0:05:13 '46' stuck at Sep 14, 2016 9:40:29 AM, unstuck at Sep 14, 2016 9:46:41 AM , stuck for 0:06:12 '17' stuck at Sep 14, 2016 9:38:29 AM, unstuck at Sep 14, 2016 9:48:09 AM , stuck for 0:09:40 '11' stuck at Sep 14, 2016 9:47:29 AM, unstuck at Sep 14, 2016 9:49:07 AM , stuck for 0:01:38 '29' stuck at Sep 14, 2016 9:39:29 AM, unstuck at Sep 14, 2016 9:56:02 AM , stuck for 0:16:33 '3' stuck at Sep 14, 2016 9:39:29 AM, unstuck at Sep 14, 2016 9:56:21 AM , stuck for 0:16:52 '16' stuck at Sep 14, 2016 9:38:29 AM, unstuck at Sep 14, 2016 10:00:50 AM , stuck for 0:22:21 '0' stuck at Sep 14, 2016 9:46:29 AM, unstuck at Sep 14, 2016 10:24:46 AM , stuck for 0:38:17