diff --git a/src/main/scala/eu/neverblink/jelly/cli/ErrorHandler.scala b/src/main/scala/eu/neverblink/jelly/cli/ErrorHandler.scala
index f3f0591..42964a7 100644
--- a/src/main/scala/eu/neverblink/jelly/cli/ErrorHandler.scala
+++ b/src/main/scala/eu/neverblink/jelly/cli/ErrorHandler.scala
@@ -9,16 +9,19 @@ object ErrorHandler:
     t match
       case e: CriticalException =>
         command.printLine(f"${e.getMessage}", toStderr = true)
+        printStackTraceIfDebug(command, t)
       case e: Throwable =>
         command.printLine("Unknown error", toStderr = true)
-    printStackTrace(command, t)
+        // Always print stack trace for unknown exceptions,
+        // as otherwise the user has no clue what happened.
+        t.printStackTrace(command.err)
     command.exit(1, t)
 
   /** Print out stack trace or debugging information
     * @param command
     * @param t
     */
-  private def printStackTrace(
+  private def printStackTraceIfDebug(
       command: JellyCommand[?],
       t: Throwable,
   ): Unit =
diff --git a/src/test/scala/eu/neverblink/jelly/cli/command/ErrorHandlerSpec.scala b/src/test/scala/eu/neverblink/jelly/cli/command/ErrorHandlerSpec.scala
new file mode 100644
index 0000000..1e4c589
--- /dev/null
+++ b/src/test/scala/eu/neverblink/jelly/cli/command/ErrorHandlerSpec.scala
@@ -0,0 +1,61 @@
+package eu.neverblink.jelly.cli.command
+
+import caseapp.core.RemainingArgs
+import eu.neverblink.jelly.cli.{CriticalException, ExitException, JellyCommand}
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+class ErrorHandlerSpec extends AnyWordSpec, Matchers:
+  def makeMockCommand(ex: Throwable): JellyCommand[VersionOptions] =
+    val command = new JellyCommand[VersionOptions] {
+      override def names: List[List[String]] = List(List("test"))
+      override def doRun(options: VersionOptions, remainingArgs: RemainingArgs): Unit =
+        throw ex
+    }
+    command.testMode(true)
+    command
+
+  "ErrorHandler" should {
+    "not print stack trace for known exceptions if not in debug mode" in {
+      val command = makeMockCommand(new CriticalException("Known error!"))
+      intercept[ExitException] {
+        command.main("test", Array())
+      }
+      val err = command.getErrString
+      err should include("Known error!")
+      err should include("Run with --debug to see the complete stack trace.")
+    }
+
+    "print stack trace for known exceptions in debug mode" in {
+      val command = makeMockCommand(new CriticalException("Known error!"))
+      intercept[ExitException] {
+        command.main("test", Array("--debug"))
+      }
+      val err = command.getErrString
+      err should include("Known error!")
+      err should include("CriticalException")
+      err should include("at eu.neverblink.jelly.cli.command.ErrorHandlerSpec")
+    }
+
+    "print stack trace for unknown exceptions if not in debug mode" in {
+      val command = makeMockCommand(new RuntimeException("Unknown error!"))
+      intercept[ExitException] {
+        command.main("test", Array())
+      }
+      val err = command.getErrString
+      err should include("Unknown error")
+      err should include("RuntimeException")
+      err should include("at eu.neverblink.jelly.cli.command.ErrorHandlerSpec")
+    }
+
+    "print stack trace for unknown exceptions in debug mode" in {
+      val command = makeMockCommand(new RuntimeException("Unknown error!"))
+      intercept[ExitException] {
+        command.main("test", Array("--debug"))
+      }
+      val err = command.getErrString
+      err should include("Unknown error")
+      err should include("RuntimeException")
+      err should include("at eu.neverblink.jelly.cli.command.ErrorHandlerSpec")
+    }
+  }