Add new tests for the common action proxy. Refactor tests so that standard...

Add new tests for the common action proxy. Refactor tests so that standard tests for actions not returning JSON, actions printing to stdout/stderr, and actions validating expected environment don't need to be repeated explicitly. Also tightened tests so that the runtimes conform to expected messages/errors and markers on stdout/stderr. As noted, new tests added to confirm action environment contains expected properties - namely auth key and edge host.

tests/.pydevproject

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+</pydev_project>

tests/build.gradle

@@ -24,17 +24,19 @@ test {
         showStandardStreams = true
         exceptionFormat = 'full'
     }
-    outputs.upToDateWhen { false } // force tests to run everytime
+    outputs.upToDateWhen { false } // force tests to run every time
 }
 
 // Add all images needed for local testing here
 test.dependsOn([
     ':core:nodejsAction:distDocker',
     ':core:nodejs6Action:distDocker',
+    ':core:actionProxy:distDocker',
     ':core:pythonAction:distDocker',
     ':core:javaAction:distDocker',
     ':core:swiftAction:distDocker',
-    ':core:swift3Action:distDocker'
+    ':core:swift3Action:distDocker',
+    ':sdk:docker:distDocker'
 ])
 
 dependencies {

tests/src/actionContainers/ActionContainer.scala

@@ -32,10 +32,15 @@ import scala.sys.process.stringToProcess
 import scala.util.Random
 import scala.util.Try
 
+import org.scalatest.FlatSpec
+import org.scalatest.Matchers
+
 import common.WhiskProperties
 import spray.json.JsObject
+import spray.json.JsString
 import spray.json.JsValue
 import spray.json.pimpString
+import org.apache.commons.lang3.StringUtils
 
 /**
  * For testing convenience, this interface abstracts away the REST calls to a
@@ -46,6 +51,29 @@ trait ActionContainer {
     def run(value: JsValue): (Int, Option[JsObject])
 }
 
+trait ActionProxyContainerTestUtils extends FlatSpec with Matchers {
+    import ActionContainer.{ filterSentinel, sentinel }
+
+    def initPayload(code: String) = JsObject("value" -> JsObject("code" -> JsString(code)))
+    def runPayload(args: JsValue, other: Option[JsObject] = None) = {
+        JsObject(Map("value" -> args) ++ (other map { _.fields } getOrElse Map()))
+    }
+
+    def checkStreams(out: String, err: String, additionalCheck: (String, String) => Unit, sentinelCount: Int = 1) = {
+        withClue("expected number of stdout sentinels") {
+            sentinelCount shouldBe StringUtils.countMatches(out, sentinel)
+        }
+        withClue("expected number of stderr sentinels") {
+            sentinelCount shouldBe StringUtils.countMatches(err, sentinel)
+        }
+
+        val (o, e) = (filterSentinel(out), filterSentinel(err))
+        o should not include (sentinel)
+        e should not include (sentinel)
+        additionalCheck(o, e)
+    }
+}
+
 object ActionContainer {
     private lazy val dockerBin: String = {
         List("/usr/bin/docker", "/usr/local/bin/docker").find { bin =>
@@ -85,6 +113,10 @@ object ActionContainer {
         Await.result(proc(docker(cmd)), t)
     }
 
+    // Filters out the sentinel markers inserted by the container (see relevant private code in Invoker.scala)
+    val sentinel = "XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX"
+    def filterSentinel(str: String) = str.replaceAll(sentinel, "").trim
+
     def withContainer(imageName: String, environment: Map[String, String] = Map.empty)(
         code: ActionContainer => Unit): (String, String) = {
         val rand = { val r = Random.nextInt; if (r < 0) -r else r }

tests/src/actionContainers/ActionProxyContainerTests.scala

+/*
+ * Copyright 2015-2016 IBM Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package actionContainers
+
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+import ActionContainer.withContainer
+import spray.json.JsNull
+import spray.json.JsNumber
+import spray.json.JsObject
+import spray.json.JsString
+import spray.json.JsArray
+
+@RunWith(classOf[JUnitRunner])
+class ActionProxyContainerTests extends BasicActionRunnerTests {
+
+    override def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit) = {
+        withContainer("openwhisk/dockerskeleton", env)(code)
+    }
+
+    val codeNotReturningJson = """
+           |#!/bin/sh
+           |echo not a json object
+        """.stripMargin.trim
+
+    /** Standard code samples, should print 'hello' to stdout and echo the input args. */
+    val stdCodeSamples = {
+        val bash = """
+                |#!/bin/bash
+                |echo 'hello stdout'
+                |echo 'hello stderr' 1>&2
+                |if [[ -z $1 || $1 == '{}' ]]; then
+                |   echo '{ "msg": "Hello from bash script!" }'
+                |else
+                |   echo $1 # echo the arguments back as the result
+                |fi
+            """.stripMargin.trim
+
+        val python = """
+                |#!/usr/bin/env python
+                |import sys
+                |print 'hello stdout'
+                |print >> sys.stderr, 'hello stderr'
+                |print(sys.argv[1])
+            """.stripMargin.trim
+
+        val perl = """
+                |#!/usr/bin/env perl
+                |print STDOUT "hello stdout\n";
+                |print STDERR "hello stderr\n";
+                |print $ARGV[0];
+            """.stripMargin.trim
+
+        Seq(("bash", bash), ("python", python), ("perl", perl))
+    }
+
+    /** Standard code samples, should print 'hello' to stdout and echo the input args. */
+    val stdEnvSamples = {
+        val bash = """
+                |#!/bin/bash
+                |echo "{ \"auth\": \"$AUTH_KEY\", \"edge\": \"$EDGE_HOST\" }"
+            """.stripMargin.trim
+
+        val python = """
+                |#!/usr/bin/env python
+                |import os
+                |print '{ "auth": "%s", "edge": "%s" }' % (os.environ['AUTH_KEY'], os.environ['EDGE_HOST'])
+            """.stripMargin.trim
+
+        val perl = """
+                |#!/usr/bin/env perl
+                |$a = $ENV{'AUTH_KEY'};
+                |$e = $ENV{'EDGE_HOST'};
+                |print "{ \"auth\": \"$a\", \"edge\": \"$e\" }";
+            """.stripMargin.trim
+
+        Seq(("bash", bash), ("python", python), ("perl", perl))
+    }
+
+    behavior of "openwhisk/dockerskeleton"
+
+    it should "run sample without init" in {
+        val (out, err) = withActionContainer() { c =>
+            val (runCode, out) = c.run(JsObject())
+            runCode should be(200)
+            out should be(Some(JsObject("error" -> JsString("This is a stub action. Replace it with custom logic."))))
+        }
+
+        checkStreams(out, err, {
+            case (o, _) => o should include("This is a stub action")
+        })
+    }
+
+    it should "run sample with init that does nothing" in {
+        val (out, err) = withActionContainer() { c =>
+            val (initCode, _) = c.init(JsObject())
+            initCode should be(200)
+            val (runCode, out) = c.run(JsObject())
+            runCode should be(200)
+            out should be(Some(JsObject("error" -> JsString("This is a stub action. Replace it with custom logic."))))
+        }
+
+        checkStreams(out, err, {
+            case (o, _) => o should include("This is a stub action")
+        })
+    }
+
+    it should "respond with 404 for bad run argument" in {
+        val (out, err) = withActionContainer() { c =>
+            val (runCode, out) = c.run(runPayload(JsString("A")))
+            runCode should be(404)
+        }
+
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
+    }
+
+    it should "fail to run a bad script" in {
+        val (out, err) = withActionContainer() { c =>
+            val (initCode, _) = c.init(initPayload(""))
+            initCode should be(200)
+            val (runCode, out) = c.run(JsNull)
+            runCode should be(502)
+            out should be(Some(JsObject("error" -> JsString("The action did not return a dictionary."))))
+        }
+
+        checkStreams(out, err, {
+            case (o, _) => o should include("error")
+        })
+    }
+
+    testNotReturningJson(codeNotReturningJson, checkResultInLogs = true)
+    testEcho(stdCodeSamples)
+    testEnv(stdEnvSamples)
+}
+
+trait BasicActionRunnerTests extends ActionProxyContainerTestUtils {
+    def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit): (String, String)
+
+    /**
+     * Runs tests for actions which do not return a dictionary and confirms expected error messages.
+     * @param codeNotReturningJson code to exectue, should not return a JSON object
+     * @param checkResultInLogs should be true iff the result of the action is expected to appear in stdout or stderr
+     */
+    def testNotReturningJson(codeNotReturningJson: String, checkResultInLogs: Boolean = true) = {
+        it should "run and report an error for script not returning a json object" in {
+            val (out, err) = withActionContainer() { c =>
+                val (initCode, _) = c.init(initPayload(codeNotReturningJson))
+                initCode should be(200)
+                val (runCode, out) = c.run(JsObject())
+                runCode should be(502)
+                out should be(Some(JsObject("error" -> JsString("The action did not return a dictionary."))))
+            }
+
+            checkStreams(out, err, {
+                case (o, e) =>
+                    if (checkResultInLogs) {
+                        (o + e) should include("not a json object")
+                    } else {
+                        o shouldBe empty
+                        e shouldBe empty
+                    }
+            })
+        }
+    }
+
+    /**
+     * Runs tests for code samples which are expected to echo the input arguments
+     * and print hello [stdout, stderr].
+     */
+    def testEcho(stdCodeSamples: Seq[(String, String)]) = {
+        for (s <- stdCodeSamples) {
+            it should s"run a ${s._1} script" in {
+                val argss = List(
+                    JsObject("string" -> JsString("hello")),
+                    JsObject("numbers" -> JsArray(JsNumber(42), JsNumber(1))),
+                    // JsObject("boolean" -> JsBoolean(true)), // fails with swift3 returning boolean: 1
+                    JsObject("object" -> JsObject("a" -> JsString("A"))))
+
+                val (out, err) = withActionContainer() { c =>
+                    val (initCode, _) = c.init(initPayload(s._2))
+                    initCode should be(200)
+
+                    for (args <- argss) {
+                        val (runCode, out) = c.run(runPayload(args))
+                        runCode should be(200)
+                        out should be(Some(args))
+                    }
+                }
+
+                checkStreams(out, err, {
+                    case (o, e) =>
+                        o should include("hello stdout")
+                        e should include("hello stderr")
+                }, argss.length)
+            }
+        }
+    }
+
+    /** Runs tests for code samples which are expected to return the expected standard environment {auth, edge}. */
+    def testEnv(stdEnvSamples: Seq[(String, String)], enforceEmptyOutputStream: Boolean = true) = {
+        for (s <- stdEnvSamples) {
+            it should s"run a ${s._1} script and confirm expected environment variables" in {
+                val auth = JsString("abc")
+                val edge = "xyz"
+                val env = Map("EDGE_HOST" -> edge)
+
+                val (out, err) = withActionContainer(env) { c =>
+                    val (initCode, _) = c.init(initPayload(s._2))
+                    initCode should be(200)
+
+                    val (runCode, out) = c.run(runPayload(JsObject(), Some(JsObject("authKey" -> auth))))
+                    runCode should be(200)
+                    out shouldBe defined
+                    out.get.fields("auth") shouldBe auth
+                    out.get.fields("edge") shouldBe JsString(edge)
+                }
+
+                checkStreams(out, err, {
+                    case (o, e) =>
+                        if (enforceEmptyOutputStream) o shouldBe empty
+                        e shouldBe empty
+                })
+            }
+        }
+    }
+}

tests/src/actionContainers/DockerSkeletonActionContainerTests.scala → tests/src/actionContainers/DockerExampleContainerTests.scala

@@ -17,78 +17,67 @@
 package actionContainers
 
 import org.junit.runner.RunWith
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
 import org.scalatest.junit.JUnitRunner
 
 import ActionContainer.withContainer
-import spray.json._
+import spray.json.JsNumber
+import spray.json.JsObject
+import spray.json.JsString
 
 @RunWith(classOf[JUnitRunner])
-class DockerSkeletonContainerTests extends FlatSpec with Matchers {
+class DockerExampleContainerTests extends ActionProxyContainerTestUtils {
 
-    val dockerSdkContainerImageName = "whisk/dockerskeleton"
+    def withPythonContainer(code: ActionContainer => Unit) = withContainer("openwhisk/example")(code)
 
-    // Helpers specific to dockerskeleton
-    def withDockerSdkContainer(code: ActionContainer => Unit) = {
-        withContainer(dockerSdkContainerImageName)(code)
-    }
-
-    def runPayload(args: JsValue) = JsObject("value" -> args)
-
-    behavior of "whisk/dockerskeleton"
-
-    it should "support valid flows without init" in {
-        val (out, err) = withDockerSdkContainer { c =>
-            val args = List(
-                JsObject(),
-                JsObject("numbers" -> JsArray(JsNumber(42), JsNumber(1))))
+    behavior of "openwhisk/example"
 
-            for (arg <- args) {
-                val (runCode, out) = c.run(runPayload(arg))
-
-                runCode should be(200)
-                out shouldBe defined
-                out.get shouldBe a[JsObject]
+    private def checkresponse(res: Option[JsObject], args: JsObject = JsObject()) = {
+        res shouldBe defined
+        res.get.fields("msg") shouldBe JsString("Hello from arbitrary C program!")
+        res.get.fields("args") shouldBe args
+    }
 
-                val JsObject(fields) = out.get
-                fields.contains("msg") should be(true)
-                fields("msg") should be(JsString("Hello from arbitrary C program!"))
-                fields("args") should be(arg)
-            }
+    it should "run sample without init" in {
+        val (out, err) = withPythonContainer { c =>
+            val (runCode, out) = c.run(JsObject())
+            runCode should be(200)
+            checkresponse(out)
         }
 
-        out.trim shouldBe empty
-        err.trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, _) => o should include("This is an example log message from an arbitrary C program!")
+        })
     }
 
-    it should "support valid flows with init" in {
-        val (out, err) = withDockerSdkContainer { c =>
-            val initPayload = JsObject("dummy init" -> JsString("dummy value"))
-            val (initCode, _) = c.init(initPayload)
-
+    it should "run sample with init that does nothing" in {
+        val (out, err) = withPythonContainer { c =>
+            val (initCode, _) = c.init(JsObject())
             initCode should be(200)
+            val (runCode, out) = c.run(JsObject())
+            runCode should be(200)
+            checkresponse(out)
+        }
 
-            val args = List(
-                JsObject(),
-                JsObject("numbers" -> JsArray(JsNumber(42), JsNumber(1))))
+        checkStreams(out, err, {
+            case (o, _) => o should include("This is an example log message from an arbitrary C program!")
+        })
+    }
 
-            for (arg <- args) {
-                val (runCode, out) = c.run(runPayload(arg))
+    it should "run sample with argument" in {
+        val (out, err) = withPythonContainer { c =>
+            val argss = List(
+                JsObject("a" -> JsString("A")),
+                JsObject("i" -> JsNumber(1)))
 
+            for (args <- argss) {
+                val (runCode, out) = c.run(runPayload(args))
                 runCode should be(200)
-                out shouldBe defined
-                out.get shouldBe a[JsObject]
-
-                val JsObject(fields) = out.get
-                fields.contains("msg") should be(true)
-                fields("msg") should be(JsString("Hello from arbitrary C program!"))
-                fields("args") should be(arg)
+                checkresponse(out, args)
             }
         }
 
-        out.trim shouldBe empty
-        err.trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, _) => o should include("This is an example log message from an arbitrary C program!")
+        }, 2)
     }
-
 }

tests/src/actionContainers/NodeJs6ActionContainerTests.scala

@@ -18,14 +18,16 @@ package actionContainers
 
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
-import spray.json.{JsBoolean, JsObject}
+
+import spray.json.JsBoolean
+import spray.json.JsObject
 
 @RunWith(classOf[JUnitRunner])
 class NodeJs6ActionContainerTests extends NodeJsActionContainerTests {
 
-    override val nodejsContainerImageName = "whisk/nodejs6action"
+    override lazy val nodejsContainerImageName = "whisk/nodejs6action"
 
-    behavior of "whisk/nodejs6action"
+    behavior of nodejsContainerImageName
 
     it should "support default function parameters" in {
         val (out, err) = withNodeJsContainer { c =>
@@ -45,8 +47,11 @@ class NodeJs6ActionContainerTests extends NodeJsActionContainerTests {
 
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
 }

tests/src/actionContainers/NodeJsActionContainerTests.scala

@@ -13,64 +13,49 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package actionContainers
 
 import org.junit.runner.RunWith
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
 import org.scalatest.junit.JUnitRunner
 
 import ActionContainer.withContainer
 import spray.json._
 
-
 @RunWith(classOf[JUnitRunner])
-class NodeJsActionContainerTests extends FlatSpec with Matchers {
+class NodeJsActionContainerTests extends BasicActionRunnerTests {
+
+    lazy val nodejsContainerImageName = "whisk/nodejsaction"
 
-    val nodejsContainerImageName = "whisk/nodejsaction"
+    override def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit) = {
+        withContainer(nodejsContainerImageName, env)(code)
+    }
+
+    def withNodeJsContainer(code: ActionContainer => Unit) = withActionContainer()(code)
 
-    // Helpers specific to nodejsaction
-    def withNodeJsContainer(code: ActionContainer => Unit) = withContainer(nodejsContainerImageName)(code)
-    def initPayload(code: String) = JsObject(
+    override def initPayload(code: String) = JsObject(
         "value" -> JsObject(
             "name" -> JsString("dummyAction"),
             "code" -> JsString(code),
             "main" -> JsString("main")))
-    def runPayload(args: JsValue) = JsObject("value" -> args)
-
-    // Filters out the sentinel markers inserted by the container (see relevant private code in Invoker.scala)
-    val sentinel = "XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX"
-    def filtered(str: String) = str.replaceAll(sentinel, "")
-
-    behavior of "whisk/nodejsaction"
-
-    it should "support valid flows" in {
-        val (out, err) = withNodeJsContainer { c =>
-            val code = """
-                | function main(args) {
-                |     return args;
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-
-            initCode should be(200)
-
-            val argss = List(
-                JsObject("greeting" -> JsString("hi!")),
-                JsObject("numbers" -> JsArray(JsNumber(42), JsNumber(1))))
 
-            for (args <- argss) {
-                val (runCode, out) = c.run(runPayload(args))
-                runCode should be(200)
-                out should be(Some(args))
-            }
-        }
-
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
-    }
+    behavior of nodejsContainerImageName
+
+    testNotReturningJson(
+        """
+        |function main(args) {
+        |    return "not a json object"
+        |}
+        """.stripMargin)
+
+    testEcho(Seq {
+        ("node", """
+          |function main(args) {
+          |    console.log('hello stdout')
+          |    console.error('hello stderr')
+          |    return args
+          |}
+          """.stripMargin)
+    })
 
     it should "fail to initialize with bad code" in {
         val (out, err) = withNodeJsContainer { c =>
@@ -85,9 +70,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
         }
 
         // Somewhere, the logs should mention an error occurred.
-        val combined = filtered(out) + err
-        combined.toLowerCase should include("error")
-        combined.toLowerCase should include("syntax")
+        checkStreams(out, err, {
+            case (o, e) =>
+                (o + e).toLowerCase should include("error")
+                (o + e).toLowerCase should include("syntax")
+        })
     }
 
     it should "fail to initialize with no code" in {
@@ -141,26 +128,6 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
         }
     }
 
-    it should "enforce that the user returns an object" in {
-        withNodeJsContainer { c =>
-            val code = """
-                | function main(args) {
-                |     return "rebel, rebel";
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200)
-
-            val (runCode, runRes) = c.run(runPayload(JsObject()))
-
-            runCode should be(502)
-            runRes.get.fields.get("error") shouldBe defined
-            // We'd like the error message to mention the broken type.
-            runRes.get.fields("error").toString should include("string")
-        }
-    }
-
     it should "not warn when using whisk.done" in {
         val (out, err) = withNodeJsContainer { c =>
             val code = """
@@ -176,8 +143,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             runRes should be(Some(JsObject("happy" -> JsString("penguins"))))
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "not warn when returning whisk.done" in {
@@ -195,8 +165,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             runRes should be(Some(JsObject("happy" -> JsString("penguins"))))
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "warn when using whisk.done twice" in {
@@ -217,8 +190,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             runRes should be(Some(JsObject()))
         }
 
-        val combined = filtered(out) + err
-        combined.toLowerCase should include("more than once")
+        checkStreams(out, err, {
+            case (o, e) =>
+                o should include("more than once")
+                e shouldBe empty
+        })
     }
 
     it should "support the documentation examples (1)" in {
@@ -252,8 +228,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             r3.get.fields.get("error") shouldBe defined
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        }, 3)
     }
 
     it should "support the documentation examples (2)" in {
@@ -274,8 +253,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             runRes should be(Some(JsObject("done" -> JsBoolean(true))))
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "support the documentation examples (3)" in {
@@ -305,8 +287,11 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             r2 should be(Some(JsObject("done" -> JsBoolean(true))))
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        }, 2)
     }
 
     it should "error when requiring a non-existent package" in {
@@ -326,11 +311,13 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
 
             val (runCode, out) = c.run(runPayload(JsObject()))
 
-            runCode should not be(200)
+            runCode should not be (200)
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        // Somewhere, the logs should mention an error occurred.
+        checkStreams(out, err, {
+            case (o, e) => (o + e) should include("MODULE_NOT_FOUND")
+        })
     }
 
     it should "have ws and socket.io-client packages available" in {
@@ -354,13 +341,16 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             runCode should be(200)
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "support resolved promises" in {
         val (out, err) = withNodeJsContainer { c =>
-        val code = """
+            val code = """
             | function main(args) {
             |     return new Promise(function(resolve, reject) {
             |       setTimeout(function() {
@@ -370,20 +360,23 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             | }
             """.stripMargin
 
-        c.init(initPayload(code))._1 should be(200)
+            c.init(initPayload(code))._1 should be(200)
 
-        val (runCode, runRes) = c.run(runPayload(JsObject()))
-        runCode should be(200)
-        runRes should be(Some(JsObject("done" -> JsBoolean(true))))
+            val (runCode, runRes) = c.run(runPayload(JsObject()))
+            runCode should be(200)
+            runRes should be(Some(JsObject("done" -> JsBoolean(true))))
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "support rejected promises" in {
         val (out, err) = withNodeJsContainer { c =>
-        val code = """
+            val code = """
             | function main(args) {
             |     return new Promise(function(resolve, reject) {
             |       setTimeout(function() {
@@ -393,15 +386,18 @@ class NodeJsActionContainerTests extends FlatSpec with Matchers {
             | }
             """.stripMargin
 
-        c.init(initPayload(code))._1 should be(200)
+            c.init(initPayload(code))._1 should be(200)
 
-        val (runCode, runRes) = c.run(runPayload(JsObject()))
+            val (runCode, runRes) = c.run(runPayload(JsObject()))
 
-        runCode should be(200)
-        runRes.get.fields.get("error") shouldBe defined
+            runCode should be(200)
+            runRes.get.fields.get("error") shouldBe defined
         }
 
-        filtered(out).trim shouldBe empty
-        filtered(err).trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 }

tests/src/actionContainers/PythonActionContainerTests.scala

@@ -17,83 +17,47 @@
 package actionContainers
 
 import org.junit.runner.RunWith
-import org.scalatest.BeforeAndAfter
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
 import org.scalatest.junit.JUnitRunner
-import spray.json._
 
 import ActionContainer.withContainer
+import spray.json.JsObject
+import spray.json.JsString
 
 @RunWith(classOf[JUnitRunner])
-class PythonActionContainerTests extends FlatSpec
-    with Matchers
-    with BeforeAndAfter {
-
-    // Helpers specific to pythonaction
-    def withPythonContainer(code: ActionContainer => Unit) = withContainer("whisk/pythonaction")(code)
-    def initPayload(code: String) = JsObject(
-        "value" -> JsObject(
-            "name" -> JsString("somePythonAction"),
-            "code" -> JsString(code)))
-    def runPayload(args: JsValue) = JsObject("value" -> args)
+class PythonActionContainerTests extends BasicActionRunnerTests {
 
-    behavior of "whisk/pythonaction"
-
-    it should "support valid flows" in {
-        val (out, err) = withPythonContainer { c =>
-            val code = """
-                |def main(dict):
-                |    if 'user' in dict:
-                |        print("hello " + dict['user'] + "!")
-                |        return {"user" : dict['user']}
-                |    else:
-                |        print("hello world!")
-                |        return {"user" : "world"}
-                |
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200)
-
-            val argss = List(
-                JsObject("user" -> JsString("Lulu")),
-                JsObject("user" -> JsString("Momo")))
-
-            for (args <- argss) {
-                val (runCode, out) = c.run(runPayload(args))
-                runCode should be(200)
-                out should be(Some(args))
-            }
-        }
-        err.trim shouldBe empty
+    override def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit) = {
+        withContainer("whisk/pythonaction", env)(code)
     }
 
-    it should "support valid json" in {
-        val (out, err) = withPythonContainer { c =>
-            val code = """
-                |def main(dict):
-                |    return dict
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200)
-
-            val argss = List(
-                JsObject("user" -> JsNull),
-                JsObject("user" -> JsString("Momo")))
-
-            for (args <- argss) {
-                val (runCode, out) = c.run(runPayload(args))
-                runCode should be(200)
-                out should be(Some(args))
-            }
-        }
-        err.trim shouldBe empty
-    }
+    behavior of "whisk/pythonaction"
 
-    it should "return some error on action error" in {
-        withPythonContainer { c =>
+    testNotReturningJson(
+        """
+        |def main(args):
+        |    return "not a json object"
+        """.stripMargin, checkResultInLogs = false)
+
+    testEcho(Seq {
+        ("python", """
+          |import sys
+          |def main(dict):
+          |    print 'hello stdout'
+          |    print >> sys.stderr, 'hello stderr'
+          |    return dict
+          """.stripMargin)
+    })
+
+    testEnv(Seq {
+        ("python", """
+         |import os
+         |def main(dict):
+         |    return { "auth": os.environ['AUTH_KEY'], "edge": os.environ['EDGE_HOST'] }
+         """.stripMargin.trim)
+    })
+
+    it should "return on action error when action fails" in {
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 |def div(x, y):
                 |    return x/y
@@ -111,10 +75,16 @@ class PythonActionContainerTests extends FlatSpec
             runRes shouldBe defined
             runRes.get.fields.get("error") shouldBe defined
         }
+
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e should include("Traceback")
+        })
     }
 
     it should "log compilation errors" in {
-        val (_, err) = withPythonContainer { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
               | 10 PRINT "Hello!"
               | 20 GOTO 10
@@ -127,10 +97,16 @@ class PythonActionContainerTests extends FlatSpec
             val (runCode, runRes) = c.run(runPayload(JsObject("basic" -> JsString("forever"))))
             runCode should be(502)
         }
+
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e should include("Traceback")
+        })
     }
 
     it should "support application errors" in {
-        withPythonContainer { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 |def main(args):
                 |    return { "error": "sorry" }
@@ -143,23 +119,13 @@ class PythonActionContainerTests extends FlatSpec
             runCode should be(200) // action writer returning an error is OK
 
             runRes shouldBe defined
-            runRes.get.fields.get("error") shouldBe defined
+            runRes should be(Some(JsObject("error" -> JsString("sorry"))))
         }
-    }
-
-    it should "enforce that the user returns an object" in {
-        withPythonContainer { c =>
-            val code = """
-                |def main(args):
-                |    return "rebel, rebel"
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200) // This could change if the action wrapper has strong type checks for `main`.
 
-            val (runCode, runRes) = c.run(runPayload(JsObject()))
-            runCode should be(502)
-            runRes.get.fields.get("error") shouldBe defined
-        }
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 }

tests/src/actionContainers/Swift3ActionContainerTests.scala

@@ -25,13 +25,11 @@ import spray.json.JsString
 @RunWith(classOf[JUnitRunner])
 class Swift3ActionContainerTests extends SwiftActionContainerTests {
 
-    override val checkStdOutEmpty = false
-    override val swiftContainerImageName = "whisk/swift3action"
-
-    behavior of "whisk/swift3action"
+    override val enforceEmptyOutputStream = false
+    override lazy val swiftContainerImageName = "whisk/swift3action"
 
     ignore should "properly use KituraNet and Dispatch" in {
-        val (out, err) = withSwiftContainer() { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 | import KituraNet
                 | import Foundation
@@ -98,13 +96,15 @@ class Swift3ActionContainerTests extends SwiftActionContainerTests {
         // in catch block an error has occurred, get docker logs and print
         // throw
 
-
-        if (checkStdOutEmpty) out.trim shouldBe empty
-        err.trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     ignore should "make Watson SDKs available to action authors" in {
-        val (out, err) = withSwiftContainer() { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 | import RestKit
                 | import InsightsForWeather
@@ -123,7 +123,10 @@ class Swift3ActionContainerTests extends SwiftActionContainerTests {
             runCode should be(200)
         }
 
-        if (checkStdOutEmpty) out.trim shouldBe empty
-        err.trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                o shouldBe empty
+                e shouldBe empty
+        })
     }
 }

tests/src/actionContainers/SwiftActionContainerTests.scala

@@ -17,67 +17,64 @@
 package actionContainers
 
 import org.junit.runner.RunWith
-import org.scalatest.BeforeAndAfter
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
 import org.scalatest.junit.JUnitRunner
-import spray.json._
 
 import ActionContainer.withContainer
-
-import common.WhiskProperties
+import spray.json.JsObject
+import spray.json.JsString
 
 @RunWith(classOf[JUnitRunner])
-class SwiftActionContainerTests extends FlatSpec
-    with Matchers
-    with BeforeAndAfter {
+class SwiftActionContainerTests extends BasicActionRunnerTests {
 
     // note: "out" will likely not be empty in some swift build as the compiler
     // prints status messages and there doesn't seem to be a way to quiet them
-    val checkStdOutEmpty = false
-    val swiftContainerImageName = "whisk/swiftaction"
+    val enforceEmptyOutputStream = true
+    lazy val swiftContainerImageName = "whisk/swiftaction"
 
     // Helpers specific to swiftaction
-    def withSwiftContainer(env: Map[String,String] = Map.empty)(code: ActionContainer => Unit) = withContainer(swiftContainerImageName, env)(code)
-    def initPayload(code: String) = JsObject(
-        "value" -> JsObject(
-            "name" -> JsString("someSwiftAction"),
-            "code" -> JsString(code)))
-    def runPayload(args: JsValue) = JsObject(
-            "authKey" -> JsString(WhiskProperties.readAuthKey(WhiskProperties.getAuthFileForTesting)),
-            "value" -> args)
-
-    behavior of "whisk/swiftaction"
-
-    it should "support valid flows" in {
-        val (out, err) = withSwiftContainer() { c =>
-            val code = """
-                | func main(args: [String: Any]) -> [String: Any] {
-                |     return args
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-
-            initCode should be(200)
-
-            val argss = List(
-                JsObject("greeting" -> JsString("hi!")),
-                JsObject("numbers" -> JsArray(JsNumber(42), JsNumber(1))))
-
-            for (args <- argss) {
-                val (runCode, out) = c.run(runPayload(args))
-                runCode should be(200)
-                out should be(Some(args))
-            }
-        }
-
-        if (checkStdOutEmpty) out.trim shouldBe empty
-        err.trim shouldBe empty
+    override def withActionContainer(env: Map[String, String] = Map.empty)(code: ActionContainer => Unit) = {
+        withContainer(swiftContainerImageName, env)(code)
     }
 
+    behavior of swiftContainerImageName
+
+    testNotReturningJson(
+        """
+        |func main(args: [String: Any]) -> String {
+        |    return "not a json object"
+        |}
+        """.stripMargin)
+
+    testEcho(Seq {
+        ("swift", """
+         |import Glibc
+         |func main(args: [String: Any]) -> [String: Any] {
+         |     print("hello stdout")
+         |     fputs("hello stderr", stderr)
+         |     return args
+         |}
+         """.stripMargin)
+    })
+
+    testEnv(Seq {
+        ("swift", """
+         |func main(args: [String: Any]) -> [String: Any] {
+         |     let env = NSProcessInfo.processInfo().environment
+         |     var auth = "???"
+         |     var edge = "???"
+         |     if let authKey : String = env["AUTH_KEY"] {
+         |         auth = "\(authKey)"
+         |     }
+         |     if let edgeHost : String = env["EDGE_HOST"] {
+         |         edge = "\(edgeHost)"
+         |     }
+         |     return ["auth": auth, "edge": edge]
+         |}
+         """.stripMargin)
+    }, enforceEmptyOutputStream)
+
     it should "return some error on action error" in {
-        withSwiftContainer() { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 | // You need an indirection, or swiftc detects the div/0
                 | // at compile-time. Smart.
@@ -98,29 +95,16 @@ class SwiftActionContainerTests extends FlatSpec
             runRes shouldBe defined
             runRes.get.fields.get("error") shouldBe defined
         }
-    }
 
-    it should "log stdout" in {
-        val (out, _) = withSwiftContainer() { c =>
-            val code = """
-                | func main(args: [String: Any]) -> [String: Any] {
-                |     print("Hello logs!")
-                |     return [ "lookAt": "the logs" ]
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200)
-
-            val (runCode, runRes) = c.run(runPayload(JsObject()))
-            runCode should be(200)
-        }
-
-        out should include("Hello logs!")
+        checkStreams(out, err, {
+            case (o, e) =>
+                if (enforceEmptyOutputStream) o shouldBe empty
+                e shouldBe empty
+        })
     }
 
     it should "log compilation errors" in {
-        val (_, err) = withSwiftContainer() { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
               | 10 PRINT "Hello!"
               | 20 GOTO 10
@@ -132,11 +116,16 @@ class SwiftActionContainerTests extends FlatSpec
             val (runCode, runRes) = c.run(runPayload(JsObject("basic" -> JsString("forever"))))
             runCode should be(502)
         }
-        err.toLowerCase should include("error")
+
+        checkStreams(out, err, {
+            case (o, e) =>
+                if (enforceEmptyOutputStream) o shouldBe empty
+                e.toLowerCase should include("error")
+        })
     }
 
     it should "support application errors" in {
-        withSwiftContainer() { c =>
+        val (out, err) = withActionContainer() { c =>
             val code = """
                 | func main(args: [String: Any]) -> [String: Any] {
                 |     return [ "error": "sorry" ]
@@ -150,52 +139,13 @@ class SwiftActionContainerTests extends FlatSpec
             runCode should be(200) // action writer returning an error is OK
 
             runRes shouldBe defined
-            runRes.get.fields.get("error") shouldBe defined
-        }
-    }
-
-    it should "enforce that the user returns an object" in {
-        withSwiftContainer() { c =>
-            val code = """
-                | func main(args: [String: Any]) -> String {
-                |     return "rebel, rebel"
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-            initCode should be(200) // This could change if the action wrapper has strong type checks for `main`.
-
-            val (runCode, runRes) = c.run(runPayload(JsObject()))
-            runCode should be(502)
-            runRes.get.fields.get("error") shouldBe defined
-        }
-    }
-
-    it should "ensure EDGE_HOST is available as an environment variable" in {
-        val (out, err) = withSwiftContainer(Map("EDGE_HOST" -> "realhost:80")) { c =>
-            val code = """
-                | func main(args: [String: Any]) -> [String: Any] {
-                |     let env = NSProcessInfo.processInfo().environment
-                |     var host = "fakehost:80"
-                |     if let edgeHost : String = env["EDGE_HOST"] {
-                |       host = "\(edgeHost)"
-                |     }
-                |    return ["host": host]
-                | }
-            """.stripMargin
-
-            val (initCode, _) = c.init(initPayload(code))
-
-            initCode should be(200)
-
-            val (runCode, response) = c.run(runPayload(JsObject()))
-
-            runCode should be(200)
-
-            response.get.fields.get("host") should be(Some(JsString("realhost:80")))
+            runRes should be(Some(JsObject("error" -> JsString("sorry"))))
         }
 
-        if (checkStdOutEmpty) out.trim shouldBe empty
-        err.trim shouldBe empty
+        checkStreams(out, err, {
+            case (o, e) =>
+                if (enforceEmptyOutputStream) o shouldBe empty
+                e shouldBe empty
+        })
     }
 }

tests/src/system/basic/CLIPythonTests.scala

@@ -28,6 +28,7 @@ import common.TestHelpers
 import common.WskTestHelpers
 import common.WskProps
 import common.JsHelpers
+import common.WhiskProperties
 
 @RunWith(classOf[JUnitRunner])
 class CLIPythonTests
@@ -53,6 +54,22 @@ class CLIPythonTests
             }
     }
 
+    it should "invoke an action and confirm expected environment is defined" in withAssetCleaner(wskprops) {
+        (wp, assetHelper) =>
+            val name = "stdenv"
+            assetHelper.withCleaner(wsk.action, name) {
+                (action, _) => action.create(name, Some(TestUtils.getTestActionFilename("stdenv.py")))
+            }
+
+            withActivation(wsk.activation, wsk.action.invoke(name)) {
+                activation =>
+                    val result = activation.fields("response").asJsObject.fields("result").asJsObject
+                    result.fields.get("error") shouldBe empty
+                    result.fields.get("auth") shouldBe Some(JsString(WhiskProperties.readAuthKey(WhiskProperties.getAuthFileForTesting)))
+                    result.fields.get("edge").toString should include(WhiskProperties.getEdgeHost)
+            }
+    }
+
     it should "invoke an invalid action and get error back" in withAssetCleaner(wskprops) {
         (wp, assetHelper) =>
             val name = "basicInvoke"
@@ -62,7 +79,7 @@ class CLIPythonTests
 
             withActivation(wsk.activation, wsk.action.invoke(name)) {
                 activation =>
-                    activation.getFieldPath("response", "result", "error") shouldBe Some(JsString("The action failed to compile. See logs for details."))
+                    activation.getFieldPath("response", "result", "error") shouldBe Some(JsString("The action failed to generate or locate a binary. See logs for details."))
                     activation.fields("logs").toString should { not include ("pythonaction.py") and not include ("flask") }
             }
     }