Skip to content
Snippets Groups Projects
Commit 4c19293b authored by Rodric Rabbah's avatar Rodric Rabbah
Browse files

Rebase swift(3) actions using new action proxy and override epilogue, buil and... 

Rebase swift(3) actions using new action proxy and override epilogue, buil and env method to properly build binary and serialize input as required by swift actions.
Rebase python action using new action proxy.
Rename runner files to {lang/runtime}runner.py.
parent 0ec2fca3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 133 additions and 299 deletions
core/pythonAction/Dockerfile
+ 4
− 13
View file @ 4c19293b
# Dockerfile for python action
FROM buildpack-deps:trusty
# Dockerfile for python actions, overrides and extends ActionRunner from actionProxy
FROM openwhisk/dockerskeleton
ENV DEBIAN_FRONTEND noninteractive
# Upgrade and install basic Python dependencies
RUN apt-get -y purge && \
apt-get -y update && \
apt-get -y install --fix-missing python2.7 python-distribute python-pip
# Install Python proxy support
RUN apt-get -y install --fix-missing python2.7-dev python-gevent python-flask
RUN apt-get clean
ENV FLASK_PROXY_PORT 8080
RUN mkdir -p /pythonAction
ADD pythonaction.py /pythonAction/
ADD pythonrunner.py /pythonAction/
CMD ["/bin/bash", "-c", "cd pythonAction && python -u pythonaction.py"]
CMD ["/bin/bash", "-c", "cd pythonAction && python -u pythonrunner.py"]
core/pythonAction/build.gradle
+ 1
− 0
View file @ 4c19293b
ext.dockerImageName = 'whisk/pythonaction'
apply from: '../../gradle/docker.gradle'
distDocker.dependsOn ':core:actionProxy:distDocker'
core/pythonAction/delete-build-run.sh deleted 100755 → 0
+ 0
− 15
View file @ 0ec2fca3
#!/usr/bin/env bash
# Useful for local testing.
# USE WITH CAUTION !!
# Removes all previously built instances.
docker rm $(docker ps -a -q)
docker build -t pythonbox .
echo ""
echo " ---- RUNNING ---- "
echo ""
docker run -i -t -p 8100:8080 pythonbox
core/pythonAction/pythonaction.py deleted 100644 → 0
+ 0
− 99
View file @ 0ec2fca3
#
# Copyright 2015-2016 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import os
import json
import subprocess
import codecs
import traceback
import flask
from gevent.wsgi import WSGIServer
proxy = flask.Flask(__name__)
proxy.debug = False
@proxy.route("/init", methods=['POST'])
def init():
flask.g = None
payload = flask.request.get_json(force=True,silent=True)
if not payload or not isinstance(payload, dict):
flask.abort(403)
message = payload.get("value", {})
if "code" in message:
# store the compiled code
try:
flask.g = compile(message["code"], filename = 'action', mode = 'exec')
except Exception:
flask.g = None
traceback.print_exc(file = sys.stderr, limit = 0)
sys.stderr.flush()
response = flask.jsonify({"error": "The action failed to compile. See logs for details." })
response.status_code = 502
return response
return ('OK', 200)
else:
flask.abort(403)
@proxy.route("/run", methods=['POST'])
def run():
message = flask.request.get_json(force=True,silent=True)
if not message or not isinstance(message, dict):
flask.abort(403)
if not "value" in message:
flask.abort(403)
value = message["value"]
if not isinstance(value, dict):
flask.abort(403)
if flask.g is None:
# no code to execute
response = flask.jsonify({"error": "No code to execute (compilation failed). See logs for details." })
response.status_code = 502
return response
# initialize the namespace for the execution
namespace = {}
result = None
try:
namespace['param'] = value
exec(flask.g, namespace)
exec("fun = main(param)", namespace)
result = namespace['fun']
except Exception:
traceback.print_exc(file = sys.stderr)
sys.stdout.flush()
sys.stderr.flush()
if result and isinstance(result, dict):
response = flask.jsonify(result)
response.status_code = 200
return response
else:
response = flask.jsonify({"error": "The action did not return a dictionary and returned this instead '%s'." % result })
response.status_code = 502
return response
# start server in a forever loop
if __name__ == "__main__":
PORT = int(os.getenv("FLASK_PROXY_PORT", 8080))
server = WSGIServer(('', PORT), proxy, log=None)
server.serve_forever()
core/swiftAction/init.py → core/pythonAction/pythonrunner.py
+ 63
− 0
View file @ 4c19293b
......@@ -14,16 +14,50 @@
# limitations under the License.
#
import os
import sys
import json
import requests
import subprocess
import codecs
sys.path.append('../actionProxy')
from actionproxy import ActionRunner, main, setRunner
import json
import traceback
class PythonRunner(ActionRunner):
def __init__(self):
ActionRunner.__init__(self)
self.fn = None
def init(self, message):
if 'code' in message:
try:
self.fn = compile(message["code"], filename = 'action', mode = 'exec')
except Exception:
traceback.print_exc(file = sys.stderr, limit = 0)
return self.verify()
DEST="http://localhost:8080/init"
def verify(self):
return self.fn is not None
with(codecs.open(sys.argv[1], "r", "utf-8")) as fp:
contents = fp.read()
def run(self, args, env):
# initialize the namespace for the execution
namespace = {}
result = None
try:
os.environ = env
namespace['param'] = args
exec(self.fn, namespace)
exec("fun = main(param)", namespace)
result = namespace['fun']
except Exception:
traceback.print_exc(file = sys.stderr)
r = requests.post(DEST, json.dumps({ "value" : { "code" : contents } }))
if result and isinstance(result, dict):
return (200, result)
else:
return (502, { 'error': 'The action did not return a dictionary.'})
print r.text
if __name__ == "__main__":
setRunner(PythonRunner())
main()
core/swiftAction/Dockerfile
+ 8
− 19
View file @ 4c19293b
# This Dockerfile is partially based on:
# https://github.com/swiftdocker/docker-swift/
FROM buildpack-deps:trusty
# Dockerfile for swift3 actions, overrides and extends ActionRunner from actionProxy
# This Dockerfile is partially based on: https://github.com/swiftdocker/docker-swift/
FROM openwhisk/dockerskeleton
ENV DEBIAN_FRONTEND noninteractive
# Upgrade and install Python dependencies
RUN apt-get -y purge && \
apt-get -y update && \
apt-get -y install --fix-missing python2.7 python-distribute python-pip
# Install Python proxy support
RUN apt-get -y install --fix-missing python2.7-dev python-gevent python-flask
# Upgrade and install Swift dependencies
RUN apt-get -y install build-essential wget clang libedit-dev libicu52 libxml2-dev
RUN apt-get -y install --fix-missing build-essential wget clang libedit-dev libicu52 libxml2-dev
RUN apt-get clean
# Install Swift keys
......@@ -32,15 +24,12 @@ RUN SWIFT_ARCHIVE_NAME=swift-$SWIFT_VERSION-$SWIFT_PLATFORM && \
tar -xzf $SWIFT_ARCHIVE_NAME.tar.gz --directory / --strip-components=1 && \
rm -rf $SWIFT_ARCHIVE_NAME* /tmp/* /var/tmp/*
# Copy the Flask proxy. Following the pattern in nodejsAction.
ADD . /swiftAction
RUN cd /swiftAction; rm -rf .project .settings build.xml Dockerfile
# Where the script will live.
# Copy the Flask proxy.
RUN mkdir -p /swiftAction
RUN touch /swiftAction/action.swift
ADD epilogue.swift /swiftAction
ADD swiftrunner.py /swiftAction
ENV FLASK_PROXY_PORT 8080
CMD ["/bin/bash", "-c", "cd swiftAction && python -u proxy.py"]
CMD ["/bin/bash", "-c", "cd swiftAction && python -u swiftrunner.py"]
core/swiftAction/build.gradle
+ 1
− 0
View file @ 4c19293b
ext.dockerImageName = 'whisk/swiftaction'
apply from: '../../gradle/docker.gradle'
distDocker.dependsOn ':core:actionProxy:distDocker'
core/swiftAction/delete-build-run.sh deleted 100755 → 0
+ 0
− 15
View file @ 0ec2fca3
#!/usr/bin/env bash
# Useful for local testing.
# USE WITH CAUTION !!
# Removes all previously built instances.
docker rm $(docker ps -a -q)
docker build -t swiftbox .
echo ""
echo " ---- RUNNING ---- "
echo ""
docker run -i -t -p 8080:8080 swiftbox
core/swiftAction/proxy.py deleted 100644 → 0
+ 0
− 138
View file @ 0ec2fca3
#
# Copyright 2015-2016 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys
import os
import json
import subprocess
import codecs
import flask
from gevent.wsgi import WSGIServer
proxy = flask.Flask(__name__)
proxy.debug = False
SRC_EPILOGUE_FILE = "./epilogue.swift"
DEST_SCRIPT_FILE = "/swiftAction/action.swift"
DEST_BIN_FILE = "/swiftAction/action"
BUILD_PROCESS = [ "swiftc", "-O", DEST_SCRIPT_FILE, "-o", DEST_BIN_FILE ]
# RUN_PROCESS = [ "swift", DEST_SCRIPT_FILE ]
RUN_PROCESS = [ DEST_BIN_FILE ]
@proxy.route("/init", methods=['POST'])
def init():
message = flask.request.get_json(force=True,silent=True)
if not message or not isinstance(message, dict):
flask.abort(403)
message = message.get("value", {})
if "code" in message:
with codecs.open(DEST_SCRIPT_FILE, "w", "utf-8") as fp:
fp.write(str(message["code"]))
with codecs.open(SRC_EPILOGUE_FILE, "r", "utf-8") as ep:
fp.write(ep.read())
p = subprocess.Popen(BUILD_PROCESS)
(o,e) = p.communicate()
if o is not None:
sys.stdout.write(o)
if e is not None:
sys.stderr.write(e)
if not (os.path.isfile(DEST_BIN_FILE) and os.access(DEST_BIN_FILE, os.X_OK)):
response = flask.jsonify({"error": "the action failed to compile. See logs for details." })
response.status_code = 502
return response
return ('OK', 200)
else:
flask.abort(403)
@proxy.route("/run", methods=['POST'])
def run():
message = flask.request.get_json(force=True,silent=True)
if not message or not isinstance(message, dict):
flask.abort(403)
if not "value" in message:
flask.abort(403)
value = message["value"]
if not isinstance(value, dict):
flask.abort(403)
if not (os.path.isfile(DEST_BIN_FILE) and os.access(DEST_BIN_FILE, os.X_OK)):
response = flask.jsonify({ "error": "the action failed to compile. See logs for details." })
response.status_code = 502
return response
# make sure to include all the env vars passed in by the invoker
swift_env_in = os.environ
swift_env_in["WHISK_INPUT"] = json.dumps(value)
p = subprocess.Popen(
RUN_PROCESS,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
env=swift_env_in)
# We run the Swift process, blocking until it completes.
(o,e) = p.communicate()
process_output = ""
if o is not None:
process_output_lines = o.strip().split("\n")
last_line = process_output_lines[-1]
for line in process_output_lines[:-1]:
sys.stdout.write("%s\n" % line)
if e is not None:
sys.stderr.write(e)
# Add sentinel to stdout marker
sys.stdout.write("XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n")
try:
json_output = json.loads(last_line)
if isinstance(json_output, dict):
response = flask.jsonify(json_output)
return response
else:
response = flask.jsonify({ "error": "the action did not return an object", "action_output": json_output })
response.status_code = 502
return response
except Exception as e:
# sys.stderr.write("Couldn't parse Swift script output as JSON: %s.\n" % last_line)
# sys.stderr.write("%s\n%s\n" % (str(e),repr(e)))
response = flask.jsonify({ "error": "the action did not return a valid result" })
response.status_code = 502
return response
if __name__ == "__main__":
PORT = int(os.getenv("FLASK_PROXY_PORT", 8080))
server = WSGIServer(('', PORT), proxy, log=None)
server.serve_forever()
core/swiftAction/run.py → core/swiftAction/swiftrunner.py
+ 56
− 0
View file @ 4c19293b
......@@ -14,38 +14,43 @@
# limitations under the License.
#
import os
import sys
import json
import requests
import subprocess
import codecs
import json
sys.path.append('../actionProxy')
from actionproxy import ActionRunner, main, setRunner
DEST="http://localhost:8080/run"
SRC_EPILOGUE_FILE = "./epilogue.swift"
DEST_SCRIPT_FILE = "/swiftAction/action.swift"
DEST_BIN_FILE = "/swiftAction/action"
BUILD_PROCESS = [ "swiftc", "-O", DEST_SCRIPT_FILE, "-o", DEST_BIN_FILE ]
def content_from_args(args):
if len(args) == 0:
return {}
class SwiftRunner(ActionRunner):
if len(args) == 1 and os.path.exists(args[0]):
with open(args[0]) as fp:
return json.load(fp)
def __init__(self):
ActionRunner.__init__(self, DEST_SCRIPT_FILE, DEST_BIN_FILE)
# else...
in_str = " ".join(args)
try:
d = json.loads(in_str)
if isinstance(d, dict):
return d
else:
raise "Not a dict."
except:
return { "payload" : " ".join(sys.argv[1:]) }
def epilogue(self, fp):
with codecs.open(SRC_EPILOGUE_FILE, "r", "utf-8") as ep:
fp.write(ep.read())
def build(self):
p = subprocess.Popen(BUILD_PROCESS)
(o, e) = p.communicate()
value = content_from_args(sys.argv[1:])
if o is not None:
sys.stdout.write(o)
print "Sending value: %s..." % json.dumps(value)[0:40]
if e is not None:
sys.stderr.write(e)
r = requests.post(DEST, json.dumps({ "value" : value }))
def env(self, message):
env = ActionRunner.env(self, message)
args = message.get('value', {}) if message else {}
env['WHISK_INPUT'] = json.dumps(args)
return env
print r.text
if __name__ == "__main__":
setRunner(SwiftRunner())
main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment