Docker images for hindsight-grpc, auto-generate scripts and configs for docker compose

DockerfileHindsight

+FROM hindsight_grpc_setup:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV MY_INSTALL_DIR=/root/.local
+RUN mkdir MY_INSTALL_DIR
+ENV PATH="$MY_INSTALL_DIR/bin:$PATH"
+
+COPY ./ /root/hindsight-grpc
+WORKDIR /root/hindsight-grpc
+RUN mv hindsight ../
+
+WORKDIR /root/hindsight
+RUN cd client &&\
+    make &&\
+    make install
+RUN cd agent &&\
+    go build cmd/agent2/main.go &&\
+    mv main /hindsight_agent
+
+WORKDIR /root/hindsight-grpc
+RUN mkdir build &&\
+    cd build &&\
+    cmake \
+    -DCMAKE_PREFIX_PATH=$MY_INSTALL_DIR \
+    -DCMAKE_BUILD_TYPE=Release \
+    .. &&\
+    make
+
+RUN apt-get -y update &&\
+    apt-get -y install apt-file &&\
+    apt-file update &&\
+    apt-get install -y vim &&\
+    apt-get install -y supervisor
+
+WORKDIR /root/hindsight-grpc
+RUN python3 gen_hindsight_config.py &&\
+    cp hindsight_conf/* /etc/hindsight_conf
+
+WORKDIR /root/hindsight-grpc/build
\ No newline at end of file

DockerfileSetup

+FROM golang:1.17.8-buster AS build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    cmake\
+    build-essential\
+    autoconf\ 
+    libtool\
+    pkg-config\
+    wget\
+    git-all\
+    openssh-client\
+    automake\ 
+    bison\ 
+    flex\ 
+    libboost-all-dev\
+    libevent-dev\ 
+    libssl-dev\ 
+    libtool\
+    curl\
+    libcurl4-openssl-dev\
+    && rm -rf /var/lib/apt/lists/*
+
+ENV MY_INSTALL_DIR=/root/.local
+RUN mkdir MY_INSTALL_DIR
+ENV PATH="$MY_INSTALL_DIR/bin:$PATH"
+
+RUN git clone --recurse-submodules -b v1.43.0 https://github.com/grpc/grpc /root/grpc
+WORKDIR /root/grpc
+RUN mkdir -p cmake/build &&\
+    cd cmake/build &&\
+    cmake -DgRPC_INSTALL=ON \
+    -DgRPC_BUILD_TESTS=OFF \
+    -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \
+    ../.. &&\
+    make &&\
+    make install
+
+RUN git clone https://github.com/google/googletest.git -b release-1.11.0 /root/googletest
+WORKDIR /root/googletest
+RUN mkdir build &&\
+    cd build &&\
+    cmake -DCMAKE_PREFIX_PATH=$MY_INSTALL_DIR \
+    -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \
+    .. &&\
+    make -j &&\
+    make install
+
+RUN git clone https://github.com/google/benchmark.git -b v1.6.1 /root/benchmark
+WORKDIR /root/benchmark
+RUN cmake -E make_directory "build" &&\
+    cmake -E chdir "build" \
+    cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_PREFIX_PATH=$MY_INSTALL_DIR \
+    -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \
+    -DBENCHMARK_ENABLE_GTEST_TESTS=OFF \
+    ../ &&\
+    cmake --build "build" --config Release &&\
+    cmake --build "build" --config Release --target install
+
+RUN wget -q -P /tmp \
+    https://downloads.apache.org/thrift/0.16.0/thrift-0.16.0.tar.gz &&\
+    tar -xf /tmp/thrift-0.16.0.tar.gz -C /root/
+WORKDIR /root/thrift-0.16.0
+RUN ./bootstrap.sh &&\
+    ./configure --without-java --without-python --without-go &&\
+    make &&\
+    make install
+
+RUN git clone -b v1.1.0 --recurse-submodules https://github.com/open-telemetry/opentelemetry-cpp /root/opentelemetry
+WORKDIR /root/opentelemetry
+RUN mkdir build &&\
+    cd build &&\
+    cmake -DWITH_ABSEIL=ON \
+    -DWITH_JAEGER=ON \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_PREFIX_PATH=$MY_INSTALL_DIR \
+    -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \
+    .. &&\
+    make &&\ 
+    make install

gen_docker_compose.py

+import json
+import os
+import shutil
+import argparse
+
+#parser = argparse.ArgumentParser(description='Generate hindsight config files')
+
+def space(i):
+	return " "*i
+
+if __name__ == '__main__':
+	# args = parser.parse_args()
+
+	# Load topology file
+	f = open("config/example_addresses.json")
+	data = json.load(f)
+	data = data["addresses"]
+
+	# Generate docker compose file
+	out = open("docker-compose.yml", "w")
+
+	out.write("version: \"3.7\"\n\nservices:\n")
+	volumes = []
+
+	for i in range(len(data)):
+		item = data[i]
+		name = item["name"]
+		port = item["port"]
+		if "otel" in name:
+			continue
+		print(name)
+		out.write(space(2)+name+":\n")
+		out.write(space(4)+"image: hindsight_grpc:latest\n")
+		out.write(space(4)+"build:\n")
+		out.write(space(6)+"context: .\n")
+		out.write(space(6)+"shm_size: \'2gb\'\n")
+		out.write(space(4)+"hostname: "+name+"\n")
+		out.write(space(4)+"ports: \n")
+		out.write(space(6)+"- "+port+":"+port+"\n")
+		out.write(space(4)+"restart: always\n")
+		out.write(space(4)+"command: ./server -a ../config/example_addresses.json -t ../config/example_topology.json -x hindsight "+name+"\n")
+		out.write(space(4)+"volumes:\n")
+		out.write(space(6)+"- "+name+"_volume:/dev/shm\n")
+		out.write("\n")
+
+		out.write(space(2)+name+"_agent:\n")
+		out.write(space(4)+"image: hindsight_grpc:latest\n")
+		out.write(space(4)+"build:\n")
+		out.write(space(6)+"context: .\n")
+		out.write(space(6)+"shm_size: \'2gb\'\n")
+		out.write(space(4)+"hostname: "+name+"_agent\n")
+		out.write(space(4)+"restart: always\n")
+		out.write(space(4)+"command: /bin/bash -c \'cd /root/hindsight/agent/ && go run cmd/agent2/main.go --serv "+name+"\'\n")
+		out.write(space(4)+"volumes:\n")
+		out.write(space(6)+"- "+name+"_volume:/dev/shm\n")
+		out.write("\n")
+	
+		volumes.append(name+"_volume")
+
+	out.write("volumes:\n")
+	for v in volumes:
+		out.write(space(2)+v+":\n")
+	out.close()
\ No newline at end of file

gen_hindsight_config.py

+import json
+import os
+import shutil
+import argparse
+
+parser = argparse.ArgumentParser(description='Generate hindsight config files')
+parser.add_argument('-o', "--output", metavar="STR", type=str, default="./hindsight_conf/", help='Output Dir')
+parser.add_argument('-s', "--buffer_size", metavar="NUM", type=str, default="4096", help='Buffer size')
+parser.add_argument('-c', "--buffer_count", metavar="NUM", type=str, default="25000", help='Buffer Count')
+
+if __name__ == '__main__':
+	args = parser.parse_args()
+
+	# Load topology file
+	f = open("config/example_topology.json")
+	data = json.load(f)
+	data = data["services"]
+
+	# Refresh config file output dir
+	isOutDirExists = os.path.isdir(args.output)
+	if isOutDirExists:
+		shutil.rmtree(args.output)
+	os.makedirs(args.output)
+
+	# Generate config files for each service. Assign agent ports from 5050. TODO: check potential port conflict if we have too much services
+	agent_port = 5050
+
+	for i in range(len(data)):
+		item = data[i]
+		name = item["name"]
+		print(name)
+
+		out = open(args.output+name+".conf", "w")
+		out.write("cap " + args.buffer_count + "\n")
+		out.write("buf_length " + args.buffer_size + "\n")
+		out.write("addr 0.0.0.0\n")
+		out.write("port "+str(agent_port)+"\n")
+		agent_port += 1
+		out.write("lc_addr 0.0.0.0\n")
+		out.write("lc_port 5252\n")
+		out.write("r_addr 0.0.0.0\n")
+		out.write("r_port 5253\n")
+
+		out.close()
\ No newline at end of file