Openwhisk in a standalone runnable jar

common/scala/src/main/scala/org/apache/openwhisk/common/Config.scala

@@ -116,14 +116,14 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St
    */
   protected def getProperties(): scala.collection.mutable.Map[String, String] = {
     val required = scala.collection.mutable.Map[String, String]() ++= requiredProperties
-    Config.readPropertiesFromEnvironment(required, env)
+    Config.readPropertiesFromSystemAndEnv(required, env)
 
     // for optional value, assign them a default from the required properties list
     // to prevent loss of a default value on a required property that may not otherwise be defined
     val optional = scala.collection.mutable.Map[String, String]() ++= optionalProperties.map { k =>
       k -> required.getOrElse(k, null)
     }
-    Config.readPropertiesFromEnvironment(optional, env)
+    Config.readPropertiesFromSystemAndEnv(optional, env)
 
     required ++ optional
   }
@@ -133,13 +133,15 @@ class Config(requiredProperties: Map[String, String], optionalProperties: Set[St
  * Singleton object which provides global methods to manage configuration.
  */
 object Config {
+  val prefix = "whisk-config."
 
   /**
    * Reads a Map of key-value pairs from the environment -- store them in the
    * mutable properties object.
    */
-  def readPropertiesFromEnvironment(properties: scala.collection.mutable.Map[String, String], env: Map[String, String])(
-    implicit logging: Logging) = {
+  def readPropertiesFromSystemAndEnv(properties: scala.collection.mutable.Map[String, String],
+                                     env: Map[String, String])(implicit logging: Logging) = {
+    readPropertiesFromSystem(properties)
     for (p <- properties.keys) {
       val envp = p.replace('.', '_').toUpperCase
       val envv = env.get(envp)
@@ -150,6 +152,16 @@ object Config {
     }
   }
 
+  def readPropertiesFromSystem(properties: scala.collection.mutable.Map[String, String])(implicit logging: Logging) = {
+    for (p <- properties.keys) {
+      val sysv = Option(System.getProperty(prefix + p))
+      if (sysv.isDefined) {
+        logging.info(this, s"system set value for $p")
+        properties += p -> sysv.get.trim
+      }
+    }
+  }
+
   /**
    * Checks that the properties object defines all the required properties.
    *

common/scala/src/main/scala/org/apache/openwhisk/common/Logging.scala

@@ -95,10 +95,12 @@ class AkkaLogging(loggingAdapter: LoggingAdapter) extends Logging {
       val logmsg: String = message // generates the message
       if (logmsg.nonEmpty) { // log it only if its not empty
         val name = if (from.isInstanceOf[String]) from else Logging.getCleanSimpleClassName(from.getClass)
-        loggingAdapter.log(loglevel, s"[$id] [$name] $logmsg")
+        loggingAdapter.log(loglevel, format(id, name.toString, logmsg))
       }
     }
   }
+
+  protected def format(id: TransactionId, name: String, logmsg: String) = s"[$id] [$name] $logmsg"
 }
 
 /**

common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala

@@ -49,10 +49,14 @@ class WhiskConfig(requiredProperties: Map[String, String],
    */
   override protected def getProperties() = {
     val properties = super.getProperties()
-    WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile))
+    if (!disableReadFromFile()) {
+      WhiskConfig.readPropertiesFromFile(properties, Option(propertiesFile) getOrElse (WhiskConfig.whiskPropertiesFile))
+    }
     properties
   }
 
+  private def disableReadFromFile() = java.lang.Boolean.getBoolean(WhiskConfig.disableWhiskPropsFileRead)
+
   val servicePort = this(WhiskConfig.servicePort)
   val dockerEndpoint = this(WhiskConfig.dockerEndpoint)
   val dockerPort = this(WhiskConfig.dockerPort)
@@ -85,6 +89,7 @@ class WhiskConfig(requiredProperties: Map[String, String],
 }
 
 object WhiskConfig {
+  val disableWhiskPropsFileRead = Config.prefix + "disable.whisks.props.file.read"
 
   /**
    * Reads a key from system environment as if it was part of WhiskConfig.
@@ -170,7 +175,7 @@ object WhiskConfig {
   val kafkaHostList = "kafka.hosts"
   val zookeeperHostList = "zookeeper.hosts"
 
-  private val edgeHostApiPort = "edge.host.apiport"
+  val edgeHostApiPort = "edge.host.apiport"
 
   val invokerHostsList = "invoker.hosts"
   val dbHostsList = "db.hostsList"
@@ -217,6 +222,7 @@ object ConfigKeys {
   val docker = "whisk.docker"
   val dockerClient = s"$docker.client"
   val dockerContainerFactory = s"$docker.container-factory"
+  val standaloneDockerContainerFactory = s"$docker.standalone.container-factory"
   val runc = "whisk.runc"
   val runcTimeouts = s"$runc.timeouts"
 
@@ -255,4 +261,6 @@ object ConfigKeys {
 
   val metrics = "whisk.metrics"
   val featureFlags = "whisk.feature-flags"
+
+  val whiskConfig = "whisk.config"
 }

common/scala/src/main/scala/org/apache/openwhisk/core/containerpool/Container.scala

@@ -82,6 +82,8 @@ trait Container {
   protected var containerHttpMaxConcurrent: Int = 1
   protected var containerHttpTimeout: FiniteDuration = 60.seconds
 
+  def containerId: ContainerId = id
+
   /** Stops the container from consuming CPU cycles. NOT thread-safe - caller must synchronize. */
   def suspend()(implicit transid: TransactionId): Future[Unit] = {
     //close connection first, then close connection pool

common/scala/src/main/scala/org/apache/openwhisk/core/database/StoreUtils.scala

@@ -24,7 +24,7 @@ import akka.stream.SinkShape
 import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Sink}
 import akka.util.ByteString
 import spray.json.DefaultJsonProtocol._
-import spray.json.{JsObject, RootJsonFormat}
+import spray.json.{JsObject, JsValue, RootJsonFormat}
 import org.apache.openwhisk.common.{Logging, StartMarker, TransactionId}
 import org.apache.openwhisk.core.entity.{DocInfo, DocRevision, DocumentReader, WhiskDocument}
 
@@ -97,6 +97,21 @@ private[database] object StoreUtils {
     s"$encodedAlgoName-$digest"
   }
 
+  /**
+   * Transforms a json object by adding and removing fields
+   *
+   * @param json base json object to transform
+   * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored
+   * @param fieldsToRemove list of field names to remove
+   * @return transformed json
+   */
+  def transform(json: JsObject,
+                fieldsToAdd: Seq[(String, Option[JsValue])],
+                fieldsToRemove: Seq[String] = Seq.empty): JsObject = {
+    val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove
+    JsObject(fields)
+  }
+
   private def combineResult[T](digest: Future[String], length: Future[Long], upload: Future[T])(
     implicit ec: ExecutionContext) = {
     for {

common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBArtifactStore.scala

@@ -28,7 +28,7 @@ import com.microsoft.azure.cosmosdb.internal.Constants.Properties
 import com.microsoft.azure.cosmosdb.rx.AsyncDocumentClient
 import kamon.metric.MeasurementUnit
 import org.apache.openwhisk.common.{LogMarkerToken, Logging, LoggingMarkers, MetricEmitter, Scheduler, TransactionId}
-import org.apache.openwhisk.core.database.StoreUtils.{checkDocHasRevision, deserialize, reportFailure}
+import org.apache.openwhisk.core.database.StoreUtils._
 import org.apache.openwhisk.core.database._
 import org.apache.openwhisk.core.database.cosmosdb.CosmosDBArtifactStoreProvider.DocumentClientRef
 import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._

common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBUtil.scala

@@ -19,7 +19,8 @@ package org.apache.openwhisk.core.database.cosmosdb
 
 import com.microsoft.azure.cosmosdb.internal.Constants.Properties.{AGGREGATE, E_TAG, ID, SELF_LINK}
 import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants._
-import spray.json.{JsObject, JsString, JsValue}
+import org.apache.openwhisk.core.database.StoreUtils.transform
+import spray.json.{JsObject, JsString}
 
 import scala.collection.immutable.Iterable
 
@@ -124,21 +125,6 @@ private[cosmosdb] trait CosmosDBUtil {
     transform(stripInternalFields(js), fieldsToAdd, Seq.empty)
   }
 
-  /**
-   * Transforms a json object by adding and removing fields
-   *
-   * @param json base json object to transform
-   * @param fieldsToAdd list of fields to add. If the value provided is `None` then it would be ignored
-   * @param fieldsToRemove list of field names to remove
-   * @return transformed json
-   */
-  def transform(json: JsObject,
-                fieldsToAdd: Seq[(String, Option[JsValue])],
-                fieldsToRemove: Seq[String] = Seq.empty): JsObject = {
-    val fields = json.fields ++ fieldsToAdd.flatMap(f => f._2.map((f._1, _))) -- fieldsToRemove
-    JsObject(fields)
-  }
-
   private def stripInternalFields(js: JsObject) = {
     //Strip out all field name starting with '_' which are considered as db specific internal fields
     JsObject(js.fields.filter { case (k, _) => !k.startsWith("_") && k != cid })

common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryArtifactStore.scala

@@ -17,13 +17,13 @@
 
 package org.apache.openwhisk.core.database.memory
 
+import java.nio.charset.StandardCharsets.UTF_8
+
 import akka.actor.ActorSystem
 import akka.http.scaladsl.model.{ContentType, Uri}
 import akka.stream.ActorMaterializer
 import akka.stream.scaladsl.{Sink, Source}
 import akka.util.ByteString
-import pureconfig.loadConfigOrThrow
-import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat}
 import org.apache.openwhisk.common.{Logging, LoggingMarkers, TransactionId}
 import org.apache.openwhisk.core.ConfigKeys
 import org.apache.openwhisk.core.database.StoreUtils._
@@ -32,13 +32,16 @@ import org.apache.openwhisk.core.entity.Attachments.Attached
 import org.apache.openwhisk.core.entity._
 import org.apache.openwhisk.core.entity.size._
 import org.apache.openwhisk.http.Messages
+import pureconfig.loadConfigOrThrow
+import spray.json.{DefaultJsonProtocol, DeserializationException, JsObject, JsString, RootJsonFormat}
 
 import scala.collection.concurrent.TrieMap
 import scala.concurrent.{ExecutionContext, Future}
 import scala.reflect.ClassTag
 import scala.util.{Failure, Success, Try}
 
 object MemoryArtifactStoreProvider extends ArtifactStoreProvider {
+  private val stores = new TrieMap[String, MemoryArtifactStore[_]]()
   override def makeStore[D <: DocumentSerializer: ClassTag](useBatching: Boolean)(
     implicit jsonFormat: RootJsonFormat[D],
     docReader: DocumentReader,
@@ -58,9 +61,12 @@ object MemoryArtifactStoreProvider extends ArtifactStoreProvider {
     val classTag = implicitly[ClassTag[D]]
     val (dbName, handler, viewMapper) = handlerAndMapper(classTag)
     val inliningConfig = loadConfigOrThrow[InliningConfig](ConfigKeys.db)
-    new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore)
+    val storeFactory = () => new MemoryArtifactStore(dbName, handler, viewMapper, inliningConfig, attachmentStore)
+    stores.getOrElseUpdate(dbName, storeFactory.apply()).asInstanceOf[ArtifactStore[D]]
   }
 
+  def purgeAll(): Unit = stores.clear()
+
   private def handlerAndMapper[D](entityType: ClassTag[D])(
     implicit actorSystem: ActorSystem,
     logging: Logging,
@@ -96,6 +102,8 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
     with DocumentProvider
     with AttachmentSupport[DocumentAbstraction] {
 
+  logging.info(this, s"Created MemoryStore for [$dbName]")
+
   override protected[core] implicit val executionContext: ExecutionContext = system.dispatcher
 
   private val artifacts = new TrieMap[String, Artifact]
@@ -110,22 +118,25 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
     val id = asJson.fields(_id).convertTo[String].trim
     require(!id.isEmpty, "document id must be defined")
 
-    val rev: Int = getRevision(asJson)
-    val docinfoStr = s"id: $id, rev: $rev"
+    val (oldRev, newRev) = computeRevision(asJson)
+    val docinfoStr = s"id: $id, rev: ${oldRev.getOrElse("null")}"
     val start = transid.started(this, LoggingMarkers.DATABASE_SAVE, s"[PUT] '$dbName' saving document: '$docinfoStr'")
 
-    val existing = Artifact(id, rev, asJson)
-    val updated = existing.incrementRev()
+    val updated = Artifact(id, newRev, asJson)
     val t = Try[DocInfo] {
-      if (rev == 0) {
-        artifacts.putIfAbsent(id, updated) match {
-          case Some(_) => throw DocumentConflictException("conflict on 'put'")
-          case None    => updated.docInfo
-        }
-      } else if (artifacts.replace(id, existing, updated)) {
-        updated.docInfo
-      } else {
-        throw DocumentConflictException("conflict on 'put'")
+      oldRev match {
+        case Some(rev) =>
+          val existing = Artifact(id, rev, asJson)
+          if (artifacts.replace(id, existing, updated)) {
+            updated.docInfo
+          } else {
+            throw DocumentConflictException("conflict on 'put'")
+          }
+        case None =>
+          artifacts.putIfAbsent(id, updated) match {
+            case Some(_) => throw DocumentConflictException("conflict on 'put'")
+            case None    => updated.docInfo
+          }
       }
     }
 
@@ -285,7 +296,6 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
   }
 
   override def shutdown(): Unit = {
-    artifacts.clear()
     attachmentStore.shutdown()
   }
 
@@ -308,38 +318,34 @@ class MemoryArtifactStore[DocumentAbstraction <: DocumentSerializer](dbName: Str
     reportFailure(f, start, failure => s"[GET] '$dbName' internal error, doc: '$id', failure: '${failure.getMessage}'")
   }
 
-  private def getRevision(asJson: JsObject) = {
-    asJson.fields.get(_rev) match {
-      case Some(JsString(r)) => r.toInt
-      case _                 => 0
+  private def computeRevision(js: JsObject): (Option[String], String) = {
+    js.fields.get(_rev) match {
+      case Some(JsString(r)) => (Some(r), digest(js))
+      case _                 => (None, digest(js))
     }
   }
 
+  private def digest(js: JsObject) = {
+    val jsWithoutRev = transform(js, Seq.empty, Seq(_rev))
+    val md = emptyDigest()
+    encodeDigest(md.digest(jsWithoutRev.compactPrint.getBytes(UTF_8)))
+  }
+
   //Use curried case class to allow equals support only for id and rev
   //This allows us to implement atomic replace and remove which check
   //for id,rev equality only
-  private case class Artifact(id: String, rev: Int)(val doc: JsObject, val computed: JsObject) {
-    def incrementRev(): Artifact = {
-      val (newRev, updatedDoc) = incrementAndGet()
-      copy(rev = newRev)(updatedDoc, computed) //With Couch attachments are lost post update
-    }
-
+  private case class Artifact(id: String, rev: String)(val doc: JsObject, val computed: JsObject) {
     def docInfo = DocInfo(DocId(id), DocRevision(rev.toString))
-
-    private def incrementAndGet() = {
-      val newRev = rev + 1
-      val updatedDoc = JsObject(doc.fields + (_rev -> JsString(newRev.toString)))
-      (newRev, updatedDoc)
-    }
   }
 
   private object Artifact {
-    def apply(id: String, rev: Int, doc: JsObject): Artifact = {
-      Artifact(id, rev)(doc, documentHandler.computedFields(doc))
+    def apply(id: String, rev: String, doc: JsObject): Artifact = {
+      val docWithRev = transform(doc, Seq((_rev, Some(JsString(rev)))))
+      Artifact(id, rev)(docWithRev, documentHandler.computedFields(doc))
     }
 
     def apply(info: DocInfo): Artifact = {
-      Artifact(info.id.id, info.rev.rev.toInt)(JsObject.empty, JsObject.empty)
+      Artifact(info.id.id, info.rev.rev)(JsObject.empty, JsObject.empty)
     }
   }
 }

common/scala/src/main/scala/org/apache/openwhisk/http/BasicHttpService.scala

@@ -167,11 +167,11 @@ object BasicHttpService {
   /**
    * Starts an HTTP(S) route handler on given port and registers a shutdown hook.
    */
-  def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None)(
+  def startHttpService(route: Route, port: Int, config: Option[HttpsConfig] = None, interface: String = "0.0.0.0")(
     implicit actorSystem: ActorSystem,
     materializer: ActorMaterializer): Unit = {
     val connectionContext = config.map(Https.connectionContext(_)).getOrElse(HttpConnectionContext)
-    val httpBinding = Http().bindAndHandle(route, "0.0.0.0", port, connectionContext = connectionContext)
+    val httpBinding = Http().bindAndHandle(route, interface, port, connectionContext = connectionContext)
     addShutdownHook(httpBinding)
   }
 

core/controller/src/main/resources/reference.conf

@@ -29,5 +29,6 @@ whisk {
   }
   controller {
     protocol: http
+    interface: "0.0.0.0"
   }
 }