If your microservice stops responding from time to time, and they only way out is to kill it with SIGINT or SIGTERM then adding a shutdown hook might be the way to go. Do note that this will not work if you kill the process with SIGKILL (-9), cause that will result in an unclean shutdown.

Some of this code is heavily influenced by Print all of the thread’s information and stack traces : Exception « Development « Java Tutorial. But has been translated into Scala, and cleaned up a little.

import java.io.PrintWriter
import java.lang.management.ManagementFactory

object TryOutShutdownHook {


  def doStuff() = {
    (1 to 1000).foreach( _ => {
      print(".")
      Thread.sleep(1000)
    })
  }


  def main(args: Array[String]): Unit = {
    sys.addShutdownHook({
      println("Shutdown hook!")
      ThreadUtils.printThreadInfo(new PrintWriter(System.out),"Threads during shutdown")
    })

    doStuff()

  }
}



object ThreadUtils {

  private val threadBean = ManagementFactory.getThreadMXBean

  def setContentionTracing(on: Boolean): Unit = {
    threadBean.setThreadContentionMonitoringEnabled( on )
  }

  private def getTaskName(id: Long, name: String): String = {
    if (name == null) id.toString
    else id + " (" + name + ")"
  }

  /**
   * Print all of the thread's information and stack traces.
   *
   * @param stream
   * the stream to
   * @param title
   * a string title for the stack trace
   */
  def printThreadInfo(stream: PrintWriter, title: String): Unit = {
    val STACK_DEPTH = 20
    val contention = threadBean.isThreadContentionMonitoringEnabled
    val threadIds = threadBean.getAllThreadIds
    stream.println("Process Thread Dump: " + title)
    stream.println(threadIds.length + " active threads")
    for (tid <- threadIds) {
      val info = threadBean.getThreadInfo(tid, STACK_DEPTH)
      if (info == null) {
        stream.println("  Inactive")
      } else {
        stream.println("Thread " + getTaskName(info.getThreadId, info.getThreadName) + ":")
        val state = info.getThreadState
        stream.println("  State: " + state)
        stream.println("  Blocked count: " + info.getBlockedCount)
        stream.println("  Waited count: " + info.getWaitedCount)
        if (contention) {
          stream.println("  Blocked time: " + info.getBlockedTime)
          stream.println("  Waited time: " + info.getWaitedTime)
        }
        if (state eq Thread.State.WAITING) stream.println("  Waiting on " + info.getLockName)
        else if (state eq Thread.State.BLOCKED) {
          stream.println("  Blocked on " + info.getLockName)
          stream.println("  Blocked by " + getTaskName(info.getLockOwnerId, info.getLockOwnerName))
        }
        stream.println("  Stack:")
        for (frame <- info.getStackTrace) {
          stream.println("    " + frame.toString)
        }
      }
    }
    stream.flush()
  }
}

import java.io.PrintWriter

import java.lang.management.ManagementFactory

object TryOutShutdownHook {

def doStuff() = {

(1 to 1000).foreach( _ => {

print(".")

Thread.sleep(1000)

})

}

def main(args: Array[String]): Unit = {

sys.addShutdownHook({

println("Shutdown hook!")

ThreadUtils.printThreadInfo(new PrintWriter(System.out),"Threads during shutdown")

})

doStuff()

}

object ThreadUtils {

private val threadBean = ManagementFactory.getThreadMXBean

def setContentionTracing(on: Boolean): Unit = {

threadBean.setThreadContentionMonitoringEnabled( on )

}

private def getTaskName(id: Long, name: String): String = {

if (name == null) id.toString

else id + " (" + name + ")"

}

/**

* Print all of the thread's information and stack traces.

* @param stream

* the stream to

* @param title

* a string title for the stack trace

def printThreadInfo(stream: PrintWriter, title: String): Unit = {

val STACK_DEPTH = 20

val contention = threadBean.isThreadContentionMonitoringEnabled

val threadIds = threadBean.getAllThreadIds

stream.println("Process Thread Dump: " + title)

stream.println(threadIds.length + " active threads")

for (tid <- threadIds) {

val info = threadBean.getThreadInfo(tid, STACK_DEPTH)

if (info == null) {

stream.println(" Inactive")

} else {

stream.println("Thread " + getTaskName(info.getThreadId, info.getThreadName) + ":")

val state = info.getThreadState

stream.println(" State: " + state)

stream.println(" Blocked count: " + info.getBlockedCount)

stream.println(" Waited count: " + info.getWaitedCount)

if (contention) {

stream.println(" Blocked time: " + info.getBlockedTime)

stream.println(" Waited time: " + info.getWaitedTime)

}

if (state eq Thread.State.WAITING) stream.println(" Waiting on " + info.getLockName)

else if (state eq Thread.State.BLOCKED) {

stream.println(" Blocked on " + info.getLockName)

stream.println(" Blocked by " + getTaskName(info.getLockOwnerId, info.getLockOwnerName))

}

stream.println(" Stack:")

for (frame <- info.getStackTrace) {

stream.println(" " + frame.toString)

}

stream.flush()

}

The output would look something like this

...........................Shutdown hook!
Process Thread Dump: Threads during shutdown
8 active threads
Thread 11 (shutdownHook1):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    sun.management.ThreadImpl.getThreadInfo1(Native Method)
    sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:178)
    sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139)
    ThreadUtils$.$anonfun$printThreadInfo$1(TryOutShutdownHook.scala:56)
    ThreadUtils$$$Lambda$7/1354382162.apply$mcVJ$sp(Unknown Source)
    scala.runtime.java8.JFunction1$mcVJ$sp.apply(JFunction1$mcVJ$sp.java:23)
    scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
    scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
    scala.collection.mutable.ArrayOps$ofLong.foreach(ArrayOps.scala:258)
    ThreadUtils$.printThreadInfo(TryOutShutdownHook.scala:55)
    TryOutShutdownHook$.$anonfun$main$1(TryOutShutdownHook.scala:18)
    TryOutShutdownHook$$$Lambda$1/728890494.apply$mcV$sp(Unknown Source)
    scala.sys.ShutdownHookThread$$anon$1.run(ShutdownHookThread.scala:37)
Thread 13 (SIGTERM handler):
  State: WAITING
  Blocked count: 0
  Waited count: 1
  Waiting on scala.sys.ShutdownHookThread$$anon$1@506c589e
  Stack:
    java.lang.Object.wait(Native Method)
    java.lang.Thread.join(Thread.java:1252)
    java.lang.Thread.join(Thread.java:1326)
    java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:107)
    java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46)
    java.lang.Shutdown.runHooks(Shutdown.java:123)
    java.lang.Shutdown.sequence(Shutdown.java:167)
    java.lang.Shutdown.exit(Shutdown.java:212)
    java.lang.Terminator$1.handle(Terminator.java:52)
    sun.misc.Signal$1.run(Signal.java:212)
    java.lang.Thread.run(Thread.java:748)
Thread 12 (Attach Listener):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
Thread 5 (Monitor Ctrl-Break):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
    java.net.SocketInputStream.socketRead0(Native Method)
    java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
    java.net.SocketInputStream.read(SocketInputStream.java:171)
    java.net.SocketInputStream.read(SocketInputStream.java:141)
    sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
    sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
    sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
    java.io.InputStreamReader.read(InputStreamReader.java:184)
    java.io.BufferedReader.fill(BufferedReader.java:161)
    java.io.BufferedReader.readLine(BufferedReader.java:324)
    java.io.BufferedReader.readLine(BufferedReader.java:389)
    com.intellij.rt.execution.application.AppMainV2$1.run(AppMainV2.java:64)
Thread 4 (Signal Dispatcher):
  State: RUNNABLE
  Blocked count: 0
  Waited count: 0
  Stack:
Thread 3 (Finalizer):
  State: WAITING
  Blocked count: 1
  Waited count: 2
  Waiting on java.lang.ref.ReferenceQueue$Lock@79c01878
  Stack:
    java.lang.Object.wait(Native Method)
    java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
    java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
    java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216)
Thread 2 (Reference Handler):
  State: WAITING
  Blocked count: 1
  Waited count: 1
  Waiting on java.lang.ref.Reference$Lock@2b1227e6
  Stack:
    java.lang.Object.wait(Native Method)
    java.lang.Object.wait(Object.java:502)
    java.lang.ref.Reference.tryHandlePending(Reference.java:191)
    java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
Thread 1 (main):
  State: TIMED_WAITING
  Blocked count: 0
  Waited count: 27
  Stack:
    java.lang.Thread.sleep(Native Method)
    TryOutShutdownHook$.$anonfun$doStuff$1(TryOutShutdownHook.scala:10)
    TryOutShutdownHook$$$Lambda$6/2047526627.apply$mcVI$sp(Unknown Source)
    scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:158)
    TryOutShutdownHook$.doStuff(TryOutShutdownHook.scala:8)
    TryOutShutdownHook$.main(TryOutShutdownHook.scala:21)
    TryOutShutdownHook.main(TryOutShutdownHook.scala)

Process finished with exit code 143 (interrupted by signal 15: SIGTERM)

...........................Shutdown hook!

Process Thread Dump: Threads during shutdown

8 active threads

Thread 11 (shutdownHook1):

State: RUNNABLE

Blocked count: 0

Waited count: 0

Stack:

sun.management.ThreadImpl.getThreadInfo1(Native Method)

sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:178)

sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139)

ThreadUtils$.$anonfun$printThreadInfo$1(TryOutShutdownHook.scala:56)

ThreadUtils$$$Lambda$7/1354382162.apply$mcVJ$sp(Unknown Source)

scala.runtime.java8.JFunction1$mcVJ$sp.apply(JFunction1$mcVJ$sp.java:23)

scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)

scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)

scala.collection.mutable.ArrayOps$ofLong.foreach(ArrayOps.scala:258)

ThreadUtils$.printThreadInfo(TryOutShutdownHook.scala:55)

TryOutShutdownHook$.$anonfun$main$1(TryOutShutdownHook.scala:18)

TryOutShutdownHook$$$Lambda$1/728890494.apply$mcV$sp(Unknown Source)

scala.sys.ShutdownHookThread$$anon$1.run(ShutdownHookThread.scala:37)

Thread 13 (SIGTERM handler):

State: WAITING

Blocked count: 0

Waited count: 1

Waiting on scala.sys.ShutdownHookThread$$anon$1@506c589e

Stack:

java.lang.Object.wait(Native Method)

java.lang.Thread.join(Thread.java:1252)

java.lang.Thread.join(Thread.java:1326)

java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:107)

java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46)

java.lang.Shutdown.runHooks(Shutdown.java:123)

java.lang.Shutdown.sequence(Shutdown.java:167)

java.lang.Shutdown.exit(Shutdown.java:212)

java.lang.Terminator$1.handle(Terminator.java:52)

sun.misc.Signal$1.run(Signal.java:212)

java.lang.Thread.run(Thread.java:748)

Thread 12 (Attach Listener):

State: RUNNABLE

Blocked count: 0

Waited count: 0

Stack:

Thread 5 (Monitor Ctrl-Break):

State: RUNNABLE

Blocked count: 0

Waited count: 0

Stack:

java.net.SocketInputStream.socketRead0(Native Method)

java.net.SocketInputStream.socketRead(SocketInputStream.java:116)

java.net.SocketInputStream.read(SocketInputStream.java:171)

java.net.SocketInputStream.read(SocketInputStream.java:141)

sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)

sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)

sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)

java.io.InputStreamReader.read(InputStreamReader.java:184)

java.io.BufferedReader.fill(BufferedReader.java:161)

java.io.BufferedReader.readLine(BufferedReader.java:324)

java.io.BufferedReader.readLine(BufferedReader.java:389)

com.intellij.rt.execution.application.AppMainV2$1.run(AppMainV2.java:64)

Thread 4 (Signal Dispatcher):

State: RUNNABLE

Blocked count: 0

Waited count: 0

Stack:

Thread 3 (Finalizer):

State: WAITING

Blocked count: 1

Waited count: 2

Waiting on java.lang.ref.ReferenceQueue$Lock@79c01878

Stack:

java.lang.Object.wait(Native Method)

java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)

java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)

java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216)

Thread 2 (Reference Handler):

State: WAITING

Blocked count: 1

Waited count: 1

Waiting on java.lang.ref.Reference$Lock@2b1227e6

Stack:

java.lang.Object.wait(Native Method)

java.lang.Object.wait(Object.java:502)

java.lang.ref.Reference.tryHandlePending(Reference.java:191)

java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)

Thread 1 (main):

State: TIMED_WAITING

Blocked count: 0

Waited count: 27

Stack:

java.lang.Thread.sleep(Native Method)

TryOutShutdownHook$.$anonfun$doStuff$1(TryOutShutdownHook.scala:10)

TryOutShutdownHook$$$Lambda$6/2047526627.apply$mcVI$sp(Unknown Source)

scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:158)

TryOutShutdownHook$.doStuff(TryOutShutdownHook.scala:8)

TryOutShutdownHook$.main(TryOutShutdownHook.scala:21)

TryOutShutdownHook.main(TryOutShutdownHook.scala)

Process finished with exit code 143 (interrupted by signal 15: SIGTERM)

Zeppelin has become one of my favourite tools in my toolbox. I am heavily designing stuff for Cassandra and in Scala, and even though I love Cassandra there are times when things just gets so complicated with the CQL command line, and creating a small project in IntelliJ just seems like too much hazel. Then using Zeppelin to try out is just perfect. So this page is a How-To with some useful Cookbook recipes.

Setting Up Zeppelin

I use Docker where things are so much easier, and I pick v0.8.0 cause I never got 0.8.2 to work for some reason.

Download and Start Cassandra

docker pull cassandra

1	docker pull cassandra

docker run --name Cassandra3 -p 9042:9042 cassandra:3.11

1	docker run --name Cassandra3 -p 9042:9042 cassandra:3.11

Download and Start Zeppelin

Download Zeppelin image

docker pull apache/zeppelin:0.8.0

1	docker pull apache/zeppelin:0.8.0

Start Zeppelin on port 8080

docker run -p 8080:8080 --name zeppelin apache/zeppelin:0.8.0

1	docker run -p 8080:8080 --name zeppelin apache/zeppelin:0.8.0

-p hp:cp
hp = Host Port, the port on your local machine
cp = Container Port, the port inside the docker which is what Zeppelin is exposing

Go to localhost:8080 in your web browser and you should see something like this

Setup Zeppelin

Find out the IP address of Cassandra in you Docker network, as you can see of the inspect, the IP address is 172.17.0.3.

QSWEM078:~ teriksson$ docker network inspect bridge
[
    {
        "Name": "bridge",
        "Id": "355be8072aafa87bafa8de19d00af597746039000d27e9245e2464fa54bf81a8",
        "Created": "2020-04-03T14:23:57.446760383Z",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": null,
            "Config": [
                {
                    "Subnet": "172.17.0.0/16",
                    "Gateway": "172.17.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Ingress": false,
        "ConfigFrom": {
            "Network": ""
        },
        "ConfigOnly": false,
        "Containers": {
            "ceda1cebea87ee7244f00d5e88292ff76fc46142627ed4064e0b98cd92f728a3": {
                "Name": "zeppelin",
                "EndpointID": "2cc39278d16db811bc593945adcc4a7ae2d0e5409a98c1ddf0d548bcf0b7052a",
                "MacAddress": "02:42:ac:11:00:02",
                "IPv4Address": "172.17.0.2/16",
                "IPv6Address": ""
            },
            "f772b8c66fe3729bd00e2bd9d2e50472ec40b1e8047796f8f69db6ecee6a77ae": {
                "Name": "<strong>cassandra3</strong>",
                "EndpointID": "23fde4a184ca9456ddec164616c4603f6ee8f3c310e21cb7c4409d350d7c3fd6",
                "MacAddress": "02:42:ac:11:00:03",
                "IPv4Address": "<strong>172.17.0.3</strong>/16",
                "IPv6Address": ""
            }
        },
        "Options": {
            "com.docker.network.bridge.default_bridge": "true",
            "com.docker.network.bridge.enable_icc": "true",
            "com.docker.network.bridge.enable_ip_masquerade": "true",
            "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
            "com.docker.network.bridge.name": "docker0",
            "com.docker.network.driver.mtu": "1500"
        },
        "Labels": {}
    }
]

QSWEM078:~ teriksson$ docker network inspect bridge

[

{

"Name": "bridge",

"Id": "355be8072aafa87bafa8de19d00af597746039000d27e9245e2464fa54bf81a8",

"Created": "2020-04-03T14:23:57.446760383Z",

"Scope": "local",

"Driver": "bridge",

"EnableIPv6": false,

"IPAM": {

"Driver": "default",

"Options": null,

"Config": [

{

"Subnet": "172.17.0.0/16",

"Gateway": "172.17.0.1"

}

]

"Internal": false,

"Attachable": false,

"Ingress": false,

"ConfigFrom": {

"Network": ""

"ConfigOnly": false,

"Containers": {

"ceda1cebea87ee7244f00d5e88292ff76fc46142627ed4064e0b98cd92f728a3": {

"Name": "zeppelin",

"EndpointID": "2cc39278d16db811bc593945adcc4a7ae2d0e5409a98c1ddf0d548bcf0b7052a",

"MacAddress": "02:42:ac:11:00:02",

"IPv4Address": "172.17.0.2/16",

"IPv6Address": ""

"f772b8c66fe3729bd00e2bd9d2e50472ec40b1e8047796f8f69db6ecee6a77ae": {

"Name": "<strong>cassandra3</strong>",

"EndpointID": "23fde4a184ca9456ddec164616c4603f6ee8f3c310e21cb7c4409d350d7c3fd6",

"MacAddress": "02:42:ac:11:00:03",

"IPv4Address": "<strong>172.17.0.3</strong>/16",

"IPv6Address": ""

}

"Options": {

"com.docker.network.bridge.default_bridge": "true",

"com.docker.network.bridge.enable_icc": "true",

"com.docker.network.bridge.enable_ip_masquerade": "true",

"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",

"com.docker.network.bridge.name": "docker0",

"com.docker.network.driver.mtu": "1500"

"Labels": {}

}

]

Set up IP address for Cassandra in the Spark Interpreter

Go to the section on “Spark”

Now add a row that says

spark.cassandra.connection.host : <span class="ng-scope ng-binding editable">172.17.0.3 </span>

1	spark.cassandra.connection.host : <span class="ng-scope ng-binding editable">172.17.0.3 </span>

Now also edit the Dependencies

You can do this in many ways, either you specify the MAVEN repo with version OR you download the JAR file(s) to disk and copy them into the Docker. I had to do the latter due to some issue with my network.

You need these two libraries :

Simply click on the JAR file and download the file, then copy it into the docker with

docker cp spark-cassandra-connector_2.11-2.0.12.jar zeppelin:/zeppelin/interpreter/spark/dep/spark-cassandra-connector_2.11-2.0.12.jar

1	docker cp spark-cassandra-connector_2.11-2.0.12.jar zeppelin:/zeppelin/interpreter/spark/dep/spark-cassandra-connector_2.11-2.0.12.jar

docker cp jsr166e-1.1.0.jar zeppelin:/zeppelin/interpreter/spark/dep/jsr166e-1.1.0.jar

1	docker cp jsr166e-1.1.0.jar zeppelin:/zeppelin/interpreter/spark/dep/jsr166e-1.1.0.jar

Setup IP address for Cassandra in Cassandra Interpreter

cassandra.hosts : 172.17.0.3

1	cassandra.hosts : 172.17.0.3

tsoft.se

Tobias – With a Passion For Software Development

Monthly Archives: April 2020

Print stacktraces for all threads on shutdown

Apache Zeppelin, with Spark and Cassandra, the perfect tool

Setting Up Zeppelin

Download and Start Cassandra

Download and Start Zeppelin

Setup Zeppelin

Setup IP address for Cassandra in Cassandra Interpreter

Create your first Notebook