Commit c4d420f3 authored by Renato Figueiro Maia's avatar Renato Figueiro Maia
Browse files

Ajustar imagem Docker para instalar com script oficial do SGA.

[SOMA-7075][SOMA-7061]
parent 5fa0bb86
Pipeline #68800 passed with stages
in 3 minutes and 58 seconds
......@@ -95,17 +95,17 @@ static_check:
- *docker_login
- docker pull $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$LAST || true
- docker pull $DOCKER_REGISTRY$DOCKER_REGISTRY_IMAGE_DIND:$LAST || true
- cd Docker/posix/
- |
docker build $BUILD_ARGS \
docker build \
--tag $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$VERSION \
--tag $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$LAST .
- sed -r "s%^FROM .*%FROM $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$VERSION%" Dockerfile_sgadocker > /tmp/sgadind
--tag $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$LAST \
-f Docker/Dockerfile .
- sed -r "s%^FROM .*%FROM $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$VERSION%" Docker/Dockerfile_withdocker > Docker/Dockerfile_dind
- |
docker build \
--tag $DOCKER_REGISTRY$DOCKER_REGISTRY_IMAGE_DIND:$VERSION \
--tag $DOCKER_REGISTRY$DOCKER_REGISTRY_IMAGE_DIND:$LAST -f /tmp/sgadind .
- rm -f /tmp/sgadind
--tag $DOCKER_REGISTRY$DOCKER_REGISTRY_IMAGE_DIND:$LAST \
-f Docker/Dockerfile_dind .
- docker push $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$VERSION
- docker push $DOCKER_REGISTRY$CI_REGISTRY_IMAGE:$LAST
- docker push $DOCKER_REGISTRY$DOCKER_REGISTRY_IMAGE_DIND:$VERSION
......@@ -122,7 +122,6 @@ docker_release:
extends: .docker_deploy
before_script:
- export LAST="latest"
- export BUILD_ARGS="--build-arg TAG=${CI_COMMIT_TAG}"
only:
- tags
......
FROM repo.tecgraf.puc-rio.br:18089/soma/sga-build-ubuntu:20.04 as build
COPY . /tmp/sgarest-daemon
RUN cd /tmp/sgarest-daemon && ./install.sh --posix /sgad
FROM repo.tecgraf.puc-rio.br:18089/soma/sga-runtime-ubuntu:20.04
ENV SGAD_HOME /sgad
COPY --from=build /sgad /sgad
WORKDIR /sgad/data
RUN mkdir algorithms logs projects
COPY collect_execution_data .
COPY flowmonitor .
COPY sgad.sh .
COPY Docker/sgad.cfg .
VOLUME /sgad/data/projects
VOLUME /sgad/data/algorithms
VOLUME /sgad/data/logs
ENV SGAD_HOME /sgad
ENTRYPOINT ["./sgad.sh"]
CMD ["sgad.cfg"]
FROM ubuntu:16.04
ENV LUA_VERSION 5.3.3
ENV LUAROCKS_VERSION 2.4.2
FROM repo.tecgraf.puc-rio.br:18082/ubuntu:20.04
RUN apt-get update && apt-get install -y \
bc \
curl \
g++ \
gcc \
git \
ksh \
liblua5.3-dev \
libreadline-dev \
libssh-dev \
lua5.3 \
make \
perl \
unzip
RUN ln -s /usr/bin/lua5.3 /usr/bin/lua && \
ln -s /usr/bin/luac5.3 /usr/bin/luac
\ No newline at end of file
FROM repo.tecgraf.puc-rio.br:18082/ubuntu:20.04
RUN apt-get update && apt-get install -y \
bc \
ksh \
libssl1.1 \
readline-common
FROM repo.tecgraf.puc-rio.br:18089/soma/sga:1.5.2
FROM SGA_IMAGE_NAME_TO_BE_REPLACED
RUN apt-get update && apt-get install -y docker.io
FROM ubuntu:16.04
ENV LUA_VERSION 5.2.4
ENV LUAROCKS_VERSION 2.4.2
RUN apt-get update && apt-get install -y \
bc \
curl \
g++ \
gcc \
git \
ksh \
liblua5.2-dev \
libssh-dev \
lua5.2 \
make \
perl \
unzip
RUN ln -sf /usr/bin/lua5.2 /usr/bin/lua && \
ln -sf /usr/bin/luac5.2 /usr/bin/luac
# RUN curl -L http://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar xzf - && \
# cd /lua-$LUA_VERSION && \
# make linux test && \
# make install && \
# cd .. && rm /lua-$LUA_VERSION -rf
RUN curl -L http://luarocks.github.io/luarocks/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar xzf - && \
cd /luarocks-$LUAROCKS_VERSION && \
./configure && \
make build && \
make install && \
cd .. && rm -rf /lua-$LUAROCKS_VERSION
RUN mkdir sgarest-daemon && \
curl -L https://git.tecgraf.puc-rio.br/csbase-dev/sgarest-daemon/-/archive/master/sgarest-daemon-master.tar.gz | tar xzf - -C sgarest-daemon --strip-components 1 && \
cd sgarest-daemon && \
luarocks install lua-schema-scm-1.rockspec && \
luarocks make sga-daemon-scm-1.rockspec && \
luarocks make sga-exec-scm-1.rockspec && \
luarocks make sga-driver-pbs-scm-1.rockspec
WORKDIR /sgad
RUN mkdir projects && mkdir algorithms && mkdir logs && mkdir -p /tmp/sgad/sandbox
RUN cp ../sgarest-daemon/collect_execution_data .
RUN cp ../sgarest-daemon/flowmonitor .
RUN rm -rf ../sgarest-daemon
COPY sgad.sh .
COPY sgad.cfg .
VOLUME /sgad/projects
VOLUME /sgad/algorithms
VOLUME /sgad/logs
ENTRYPOINT ["./sgad.sh"]
# CMD ["sgad.cfg"]
csbase_server = os.getenv("CSBASE_SERVER")
platform = os.getenv("SGAD_PLATFORM") or "Linux44_64"
sgad_host = os.getenv("SGAD_HOST")
sgad_port = tonumber(os.getenv("SGAD_PORT"))
sga_name = os.getenv("SGAD_NAME")
status_interval_s = 10
exec_polling_interval_s = 5
register_retry_s = 3
project_root_dir = "/sgad/projects"
algorithm_root_dir = "/sgad/algorithms"
runtime_data_dir = "/tmp/sgad"
sandbox_root_dir = "/tmp/sgad/sandbox"
driver = "sga.driver.pbs"
#!/bin/sh
timestamp=$(date +%Y%m%d%H%M%S)
logfile="logs/sgad_${timestamp}.log"
confgfile=${1}
if [ -z ${confgfile} ]; then
echo "Using default file configuration" > ${logfile};
confgfile=sgad.cfg
fi
eval $(luarocks path --bin)
# sgad ${confgfile} >> "${logfile}" 2>&1
sgad ${confgfile}
\ No newline at end of file
FROM repo.tecgraf.puc-rio.br:18089/soma/ubuntu:16.04-sga-posix
ENV LUA_VERSION 5.3.3
ENV LUAROCKS_VERSION 2.4.2
ARG TAG=master
# RUN curl -L http://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar xzf - && \
# cd /lua-$LUA_VERSION && \
# make linux test && \
# make install && \
# cd .. && rm /lua-$LUA_VERSION -rf
RUN curl -L http://luarocks.github.io/luarocks/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar xzf - && \
cd /luarocks-$LUAROCKS_VERSION && \
./configure && \
make build && \
make install && \
cd .. && rm -rf /lua-$LUAROCKS_VERSION
RUN mkdir sgarest-daemon && \
curl -L https://git.tecgraf.puc-rio.br/csbase-dev/sgarest-daemon/-/archive/${TAG}/sgarest-daemon-${TAG}.tar.gz | tar xzf - -C sgarest-daemon --strip-components 1 && \
cd sgarest-daemon && \
luarocks install lua-schema-*.rockspec && \
luarocks make sga-daemon-*.rockspec && \
luarocks make sga-driver-posix-*.rockspec
WORKDIR /sgad
RUN mkdir projects && mkdir algorithms && mkdir logs
RUN cp ../sgarest-daemon/collect_execution_data .
RUN cp ../sgarest-daemon/flowmonitor .
RUN rm -rf ../sgarest-daemon
COPY sgad.sh .
COPY sgad.cfg .
VOLUME /sgad/projects
VOLUME /sgad/algorithms
VOLUME /sgad/logs
ENTRYPOINT ["./sgad.sh"]
# CMD ["sgad.cfg"]
FROM alpine:3.7
ENV LUA_VERSION 5.3.3
ENV LUAROCKS_VERSION 2.4.2
RUN apk add --no-cache bc \
curl \
g++ \
gcc \
git \
loksh \
lua5.3-dev \
libssh-dev \
openssl-dev \
lua5.3 \
make \
perl \
unzip
RUN ln -s /usr/bin/lua5.3 /usr/bin/lua && \
ln -s /usr/bin/luac5.3 /usr/bin/luac
# RUN curl -L http://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar xzf - && \
# cd /lua-$LUA_VERSION && \
# make linux test && \
# make install && \
# cd .. && rm /lua-$LUA_VERSION -rf
RUN curl -L http://luarocks.github.io/luarocks/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar xzf - && \
cd /luarocks-$LUAROCKS_VERSION && \
./configure && \
make build && \
make install && \
cd .. && rm -rf /lua-$LUAROCKS_VERSION
RUN mkdir sgarest-daemon && \
curl -L https://git.tecgraf.puc-rio.br/csbase-dev/sgarest-daemon/-/archive/master/sgarest-daemon-master.tar.gz | tar xzf - -C sgarest-daemon --strip-components 1 && \
cd sgarest-daemon && \
luarocks install lua-schema-scm-1.rockspec && \
luarocks make sga-daemon-scm-1.rockspec && \
luarocks make sga-driver-posix-scm-1.rockspec
WORKDIR /sgad
RUN mkdir projects && mkdir algorithms && mkdir logs
RUN cp ../sgarest-daemon/collect_execution_data .
RUN cp ../sgarest-daemon/flowmonitor .
RUN rm -rf ../sgarest-daemon
COPY sgad.sh .
COPY sgad.cfg .
VOLUME /sgad/projects
VOLUME /sgad/algorithms
VOLUME /sgad/logs
ENTRYPOINT ["./sgad.sh"]
# CMD ["sgad.cfg"]
FROM repo.tecgraf.puc-rio.br:18089/soma/sga:latest-alpine
RUN apk add --no-cache docker
#!/bin/sh
timestamp=$(date +%Y%m%d%H%M%S)
logfile="logs/sgad_${timestamp}.log"
confgfile=${1}
if [ -z ${confgfile} ]; then
echo "Using default file configuration" > ${logfile};
confgfile=sgad.cfg
fi
eval $(luarocks path --bin)
# sgad ${confgfile} >> "${logfile}" 2>&1
sgad ${confgfile}
\ No newline at end of file
......@@ -6,11 +6,19 @@ sga_name = os.getenv("SGAD_NAME")
status_interval_s = 10
exec_polling_interval_s = 5
register_retry_s = 3
project_root_dir = "/sgad/projects"
algorithm_root_dir = "/sgad/algorithms"
project_root_dir = "/sgad/data/projects"
algorithm_root_dir = "/sgad/data/algorithms"
runtime_data_dir = "/tmp/sgad"
sandbox_root_dir = "/tmp/sgad/sandbox"
driver = "sga.driver.posix"
resources = {
"docker"
}
if os.getenv("SGAD_SCP_PATH") then
extra_config = {
csbase_transfer_name = "ssh-datatransfer",
csbase_csfs_root_dir = os.getenv("SGAD_SCP_PATH"),
ssh_host = os.getenv("SGAD_SCP_HOST"),
ssh_port = 22,
ssh_user_name = os.getenv("SGAD_SCP_USER"),
ssh_user_pass = os.getenv("SGAD_SCP_PASS"),
ssh_private_key_path = os.getenv("SGAD_SCP_PKEY"),
}
end
FROM ubuntu:16.04
ENV LUA_VERSION 5.2.4
ENV LUAROCKS_VERSION 2.4.2
RUN apt-get update && apt-get install -y \
bc \
curl \
g++ \
gcc \
git \
ksh \
liblua5.2-dev \
libssh-dev \
lua5.2 \
make \
perl \
unzip
RUN ln -sf /usr/bin/lua5.2 /usr/bin/lua && \
ln -sf /usr/bin/luac5.2 /usr/bin/luac
# RUN curl -L http://www.lua.org/ftp/lua-${LUA_VERSION}.tar.gz | tar xzf - && \
# cd /lua-$LUA_VERSION && \
# make linux test && \
# make install && \
# cd .. && rm /lua-$LUA_VERSION -rf
RUN curl -L http://luarocks.github.io/luarocks/releases/luarocks-${LUAROCKS_VERSION}.tar.gz | tar xzf - && \
cd /luarocks-$LUAROCKS_VERSION && \
./configure && \
make build && \
make install && \
cd .. && rm -rf /lua-$LUAROCKS_VERSION
RUN mkdir sgarest-daemon && \
curl -L https://git.tecgraf.puc-rio.br/csbase-dev/sgarest-daemon/-/archive/master/sgarest-daemon-master.tar.gz | tar xzf - -C sgarest-daemon --strip-components 1 && \
cd sgarest-daemon && \
luarocks install lua-schema-scm-1.rockspec && \
luarocks make sga-daemon-scm-1.rockspec && \
luarocks make sga-exec-scm-1.rockspec && \
luarocks make sga-driver-slurm-scm-1.rockspec
WORKDIR /sgad
ARG RUNTIME_DIR
ENV RUNTIME_DIR ${RUNTIME_DIR}
RUN mkdir projects && mkdir algorithms && mkdir logs && mkdir -p ${RUNTIME_DIR}/tmp/sgad/sandbox
RUN cp ../sgarest-daemon/collect_execution_data .
RUN cp ../sgarest-daemon/flowmonitor .
RUN rm -rf ../sgarest-daemon
COPY sgad.sh .
COPY sgad.cfg .
VOLUME /sgad/projects
VOLUME /sgad/algorithms
VOLUME /sgad/logs
ENTRYPOINT ["./sgad.sh"]
# CMD ["sgad.cfg"]
csbase_server = os.getenv("CSBASE_SERVER")
platform = os.getenv("SGAD_PLATFORM") or "Linux44_64"
sgad_host = os.getenv("SGAD_HOST")
sgad_port = tonumber(os.getenv("SGAD_PORT"))
sga_name = os.getenv("SGAD_NAME")
status_interval_s = 10
exec_polling_interval_s = 5
register_retry_s = 3
project_root_dir = "/sgad/projects"
algorithm_root_dir = "/sgad/algorithms"
runtime_data_dir = (os.getenv("RUNTIME_DIR") or "").."/tmp/sgad"
sandbox_root_dir = (os.getenv("RUNTIME_DIR") or "").."/tmp/sgad/sandbox"
driver = "sga.driver.slurm"
--driver_config = {
-- exec_driver = "sga.exec.ssh",
-- exec_config = {
-- host = os.getenv("SLURM_USER").."@"..os.getenv("SLURM_HOST"),
-- port = 22,
-- }
--}
extra_config = {
csbase_transfer_name = "ssh-datatransfer",
csbase_csfs_root_dir = (os.getenv("RUNTIME_DIR") or "").."/tmp",
ssh_host = os.getenv("SLURM_HOST"),
ssh_port = 22,
ssh_user_name = os.getenv("SLURM_USER"),
ssh_user_pass = os.getenv("SLURM_PWD")
}
#!/bin/sh
timestamp=$(date +%Y%m%d%H%M%S)
logfile="logs/sgad_${timestamp}.log"
confgfile=${1}
if [ -z ${confgfile} ]; then
echo "Using default file configuration" > ${logfile};
confgfile=sgad.cfg
fi
if [ ! -z ${RUNTIME_DIR} ]; then
echo "Using specified runtime root directory: ${RUNTIME_DIR}"
fi
eval $(luarocks path --bin)
sgad ${confgfile} 2>&1 | tee -a "${logfile}"
......@@ -58,7 +58,48 @@ extra_config = {
The `ssh_private_key_path` value is the private key path in the CSGrid server machine, and the user csgrid must be able to authenticate in the SGA machine using this key.
## Multiple SGAs with same installation directory
## Installation
Requirements:
+ gcc 4.8.5
+ g++
+ make
+ curl
+ unzip
+ openssl_dev 1.1
+ perl
+ ksh
For a Microsoft Windows installation, it is recommended to use [Cygwin](https://www.cygwin.com/) for the dependency management.
Clone the git repository.
```console
$ git clone https://git.tecgraf.puc-rio.br/csbase/sgarest-daemon
$ cd sgarest-daemon
```
Run the installation script providing an installation path.
```console
$ export SGAD_HOME=/cygdrive/c/csbase/sgad
$ ./install.sh --posix ${SGAD_HOME}
```
Alternatively, use the following options of the script to choose which drivers shall be installed:
- `--posix`: Install the POSIX driver
- `--pbs`: Install the PBS driver
- `--slurm`: Install the Slurm driver
## Execution
Run the installed SGA daemon using command `sgad`, and providing a SGA configuration file.
```console
$ ${SGAD_HOME}/bin/sgad ./sgad.cfg
```
The SGA configuration file is a Lua script, thus the use standard Lua functions -- like `os.getenv` to read environment variables -- is allowed and could be used to adapt a single configuration file to multiple SGA instances, as seen in the `sgad.cfg` file example below.
......@@ -89,49 +130,15 @@ env CSBASE_SERVER="http://localhost:40409" \
SGAD_PORT=40100 \
SGAD_NAME=$HOSTNAME \
SERVER_DATA_DIR="/mnt/csgrid_data \
sgad.sh
${SGAD_HOME}/bin/sgad ./sgad.cfg
```
## Install
Requirements:
+ gcc 4.8.5
+ g++
+ make
+ curl
+ unzip
+ openssl_dev 1.1
+ perl
+ ksh
For a Microsoft Windows installation, it is recommended to use [Cygwin](https://www.cygwin.com/) for the dependency management.
Clone the git repository
```console
$ git clone https://git.tecgraf.puc-rio.br/csbase/sgarest-daemon
$ cd sgarest-daemon
```
Run the installation script providing an installation path
```console
$ ./install.sh --posix ~/csgrid/sga
```
Additionally, use the following options of the script to install at least one driver:
- `--posix`: Install the POSIX driver
- `--pbs`: Install the PBS driver
- `--slurm`: Install the Slurm driver
## List Version
## Check SGA Version
The version of SGA can be found in `sga-daemon-\*.rockspec` file. The same version is also found in each component: `sga-driver-posix`, `sga-driver-pbs`, `sga-driver-slurm `and `sga-exec`. One can execute the following command to list the SGA version:
```console
$ luarocks list sga-daemon
$ ${SGAD_HOME}/bin/luarocks list sga-daemon
```
## Release a new version
......@@ -152,7 +159,7 @@ The Docker image only works with the POSIX driver.
### Build
```console
docker build . --network host -t soma/sga
$ docker build -f Docker/Dockerfile --network host -t soma/sga .
  • Pode retirar o argumento --network host, acho que não é mais necessário.

Please register or sign in to reply
```
### Run
......@@ -176,6 +183,7 @@ docker run -d \
## Kubernetes
The SGA Rest Daemon can be installed via Helm Chat. The installation process can be seen at [README](./kubernetes/posix/helm/README.md).
## Credits
This next-generation SGA was designed and implemented at LabLua, PUC-Rio by Hisham Muhammad <hisham@gobolinux.org> and Ana Lúcia de Moura <amoura@inf.puc-rio.br>.
......
......@@ -22,7 +22,7 @@ download() {
tarball="$2.tar.gz"
if [ ! -e "$tarball" ]
then
wget -c "$1/$tarball"
curl -O "$1/$tarball"
fi
tar -xzvf "$tarball"
}
......
......@@ -7,5 +7,7 @@ logsdir="logs"
logfile="${logsdir}/sgad_${timestamp}.log"
eval $(luarocks path --bin)
instdir=${SGAD_HOME:-/usr/local}
eval $(${instdir}/bin/luarocks path --bin)
sgad "$@" 2>&1 | tee -a "${logfile}"
  • Será que o script sgad.sh pode ser movido para o diretório Docker? Pois na documentação ele nem é mencionado e para usá-lo é preciso definir a variável SGAD_HOME -- quando o SGA for instalado em um diretório diferente de /usr/local.

Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment