From e53ee58d504f56b584c46317e9548577d4ec7147 Mon Sep 17 00:00:00 2001 From: zxl Date: Tue, 11 Jul 2023 09:28:45 +0800 Subject: [PATCH] Complete preliminary docker packaging of cube and mci --- .../csst-ifs-l1-cube-adm/Dockerfile | 20 +++++++++++++ L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile | 21 ++++++++++++++ .../csst-ifs-l1-cube-adm/add_task.sh | 18 ++++++++++++ L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh | 26 +++++++++++++++++ .../csst-ifs-l1-cube-adm/test/Makefile | 16 +++++++++++ .../csst-ifs-l1-cube-adm/test/add_task.sh | 18 ++++++++++++ .../csst-ifs-l1-cube-adm/test/app.yaml | 28 +++++++++++++++++++ .../csst-ifs-l1-cube-adm/test/zjs.env | 3 ++ .../csst-ifs-l1-cube-exe/Dockerfile | 20 +++++++++++++ L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile | 21 ++++++++++++++ .../csst-ifs-l1-cube-exe/add_task.sh | 18 ++++++++++++ L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh | 24 ++++++++++++++++ .../csst-ifs-l1-cube-exe/test/Makefile | 16 +++++++++++ .../csst-ifs-l1-cube-exe/test/add_task.sh | 18 ++++++++++++ .../csst-ifs-l1-cube-exe/test/app.yaml | 28 +++++++++++++++++++ .../csst-ifs-l1-cube-exe/test/zjs.env | 3 ++ L1/dockerfiles/csst-mci-l1/Dockerfile | 13 +++++++++ L1/dockerfiles/csst-mci-l1/Makefile | 19 +++++++++++++ L1/dockerfiles/csst-mci-l1/run.sh | 7 +++++ L1/dockerfiles/csst-mci-l1/test/Makefile | 16 +++++++++++ L1/dockerfiles/csst-mci-l1/test/app.yaml | 26 +++++++++++++++++ L1/dockerfiles/csst-mci-l1/test/zjs.env | 3 ++ 22 files changed, 382 insertions(+) create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml create mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env create mode 100644 L1/dockerfiles/csst-mci-l1/Dockerfile create mode 100644 L1/dockerfiles/csst-mci-l1/Makefile create mode 100644 L1/dockerfiles/csst-mci-l1/run.sh create mode 100644 L1/dockerfiles/csst-mci-l1/test/Makefile create mode 100644 L1/dockerfiles/csst-mci-l1/test/app.yaml create mode 100644 L1/dockerfiles/csst-mci-l1/test/zjs.env diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile new file mode 100644 index 0000000..4137e3d --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile @@ -0,0 +1,20 @@ +FROM csst/csst-ifs-l1-cube-adm:latest +LABEL maintainer="Xiaoli Zhang" + +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh +COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin + +RUN mkdir /work/ +#WORKDIR /work/ +WORKDIR /L1Pipeline/workspace + +ENTRYPOINT ["goagent"] + + +# RUN mkdir -p /L1Pipeline/workspace +# WORKDIR /L1Pipeline/workspace +# COPY code/ /L1Pipeline/workspace +# RUN /app/bin/run.sh + + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile new file mode 100644 index 0000000..68a5f33 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile @@ -0,0 +1,21 @@ +IMAGE_NAME:=csst/ifs-l1-cube-adm + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar + ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar + ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar + ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) +scp: + scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-adm/ + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh new file mode 100644 index 0000000..a41e902 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +POSTGRES_USER=scalebox +POSTGRES_HOST=localhost +POSTGRES_DB=scalebox +PGPORT=5432 + +job_id=$1 + +objids=("NGC6217" "NGC6397") +for((i=0;i<${#objids[@]};i++)); do + m=${objids[$i]} + echo $m + docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ + -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" +done + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh new file mode 100644 index 0000000..b710853 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# python helloworld.py + + +# obsid="30100000001" + +obsid=$1 + +python l1_pipeline_script_ifs_cube_adm_simdata.py $obsid > ./stdout + +exit_code=$? + +mkdir -p /work +# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt +# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt + +# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +# mv /tmp/timestamps.txt /work/timestamps.txt + +echo finish ifs cube adm. + +cat ./stdout + +exit $exit_code + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile new file mode 100644 index 0000000..50026e2 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile @@ -0,0 +1,16 @@ +CLUSTER=csst + +all: reset build + +run: + scalebox app create --env-file zjs.env + +reset: + cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - + +down: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down + +list: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh new file mode 100644 index 0000000..a41e902 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +POSTGRES_USER=scalebox +POSTGRES_HOST=localhost +POSTGRES_DB=scalebox +PGPORT=5432 + +job_id=$1 + +objids=("NGC6217" "NGC6397") +for((i=0;i<${#objids[@]};i++)); do + m=${objids[$i]} + echo $m + docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ + -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" +done + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml new file mode 100644 index 0000000..c29cee7 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml @@ -0,0 +1,28 @@ +name: ifs-l1-cube-adm.app.process +cluster: csst +parameters: + initial_status: RUNNING + +jobs: + ifs-l1-cube-adm: + base_image: csst/ifs-l1-cube-adm + schedule_mode: HEAD + variables: + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + # start_message: scalebox + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + paths: + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs + # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} + # sink_jobs: + # hosts: + # - h0:1 + # - c0:2 + # - c1:2 + # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env new file mode 100644 index 0000000..9ff022c --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env @@ -0,0 +1,3 @@ +AUX_DIR=/sharewcl/shao/L1Pipeline/aux +CRDS_DIR=/sharewcl/shao/L1Pipeline/crds +DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile new file mode 100644 index 0000000..ae87f6b --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile @@ -0,0 +1,20 @@ +FROM csst/csst-ifs-l1-cube-exe:latest +LABEL maintainer="Xiaoli Zhang" + +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh +COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin + +RUN mkdir /work/ +#WORKDIR /work/ +WORKDIR /L1Pipeline/workspace + +ENTRYPOINT ["goagent"] + + +# RUN mkdir -p /L1Pipeline/workspace +# WORKDIR /L1Pipeline/workspace +# COPY code/ /L1Pipeline/workspace +# RUN /app/bin/run.sh + + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile new file mode 100644 index 0000000..4d7449a --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile @@ -0,0 +1,21 @@ +IMAGE_NAME:=csst/ifs-l1-cube-exe + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar + ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar + ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar + ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) +scp: + scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-exe/ + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh new file mode 100644 index 0000000..a41e902 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +POSTGRES_USER=scalebox +POSTGRES_HOST=localhost +POSTGRES_DB=scalebox +PGPORT=5432 + +job_id=$1 + +objids=("NGC6217" "NGC6397") +for((i=0;i<${#objids[@]};i++)); do + m=${objids[$i]} + echo $m + docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ + -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" +done + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh new file mode 100644 index 0000000..7f03a20 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# python helloworld.py + + +# obsid="30100000001" + +obsid=$1 + +python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout + +exit_code=$? + +mkdir -p /work +# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt +# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt + +# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +# mv /tmp/timestamps.txt /work/timestamps.txt + +echo finish ifs cube exe. + +exit $exit_code + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile new file mode 100644 index 0000000..50026e2 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile @@ -0,0 +1,16 @@ +CLUSTER=csst + +all: reset build + +run: + scalebox app create --env-file zjs.env + +reset: + cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - + +down: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down + +list: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh new file mode 100644 index 0000000..a41e902 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +POSTGRES_USER=scalebox +POSTGRES_HOST=localhost +POSTGRES_DB=scalebox +PGPORT=5432 + +job_id=$1 + +objids=("NGC6217" "NGC6397") +for((i=0;i<${#objids[@]};i++)); do + m=${objids[$i]} + echo $m + docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ + -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" +done + diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml new file mode 100644 index 0000000..68970d7 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml @@ -0,0 +1,28 @@ +name: ifs-l1-cube-exe.app.process +cluster: csst +parameters: + initial_status: RUNNING + +jobs: + ifs-l1-cube-exe: + base_image: csst/ifs-l1-cube-exe + schedule_mode: HEAD + variables: + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + # start_message: scalebox + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + paths: + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs + # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} + # sink_jobs: + # hosts: + # - h0:1 + # - c0:2 + # - c1:2 + # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env new file mode 100644 index 0000000..9ff022c --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env @@ -0,0 +1,3 @@ +AUX_DIR=/sharewcl/shao/L1Pipeline/aux +CRDS_DIR=/sharewcl/shao/L1Pipeline/crds +DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs \ No newline at end of file diff --git a/L1/dockerfiles/csst-mci-l1/Dockerfile b/L1/dockerfiles/csst-mci-l1/Dockerfile new file mode 100644 index 0000000..fa6c6fe --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/Dockerfile @@ -0,0 +1,13 @@ +FROM csst/csst-mci-l1:latest +LABEL maintainer="Xiaoli Zhang" + +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh +COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin + +RUN mkdir /work/ +WORKDIR /work/ + +ENTRYPOINT ["goagent"] + + diff --git a/L1/dockerfiles/csst-mci-l1/Makefile b/L1/dockerfiles/csst-mci-l1/Makefile new file mode 100644 index 0000000..faa4af8 --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/Makefile @@ -0,0 +1,19 @@ +IMAGE_NAME:=csst/mci-l1 + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/mci-l1.tar + ssh c0 docker load -i /share/tmp/mci-l1.tar + ssh c1 docker load -i /share/tmp/mci-l1.tar + ssh c2 docker load -i /share/tmp/mci-l1.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) + diff --git a/L1/dockerfiles/csst-mci-l1/run.sh b/L1/dockerfiles/csst-mci-l1/run.sh new file mode 100644 index 0000000..1fc1a47 --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash +obsid="xxxxxxxxxxxx" + +# python l1_pipeline_script_mci_shao_simdata.py $obsid > ./stdout +python /L1Pipeline/workspace/helloworld.py + +exit_code=$? \ No newline at end of file diff --git a/L1/dockerfiles/csst-mci-l1/test/Makefile b/L1/dockerfiles/csst-mci-l1/test/Makefile new file mode 100644 index 0000000..50026e2 --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/test/Makefile @@ -0,0 +1,16 @@ +CLUSTER=csst + +all: reset build + +run: + scalebox app create --env-file zjs.env + +reset: + cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - + +down: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down + +list: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list + diff --git a/L1/dockerfiles/csst-mci-l1/test/app.yaml b/L1/dockerfiles/csst-mci-l1/test/app.yaml new file mode 100644 index 0000000..4f98da7 --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/test/app.yaml @@ -0,0 +1,26 @@ +name: mci-l1.app.process +cluster: csst +parameters: + initial_status: RUNNING + +jobs: + mci-l1: + base_image: csst/mci-l1 + variables: + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + # paths: + # - ${AUX_DIR}:/L1Pipeline/aux + # - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds + # - ${DFS_ROOT}:/dfsroot + # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} + # sink_jobs: + # hosts: + # - h0:1 + # - c0:2 + # - c1:2 + # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-mci-l1/test/zjs.env b/L1/dockerfiles/csst-mci-l1/test/zjs.env new file mode 100644 index 0000000..1f64fd8 --- /dev/null +++ b/L1/dockerfiles/csst-mci-l1/test/zjs.env @@ -0,0 +1,3 @@ +AUX_DIR=/sharewcl/L1Pipeline/aux +CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr +DFS_ROOT=/sharewcl/dfs -- GitLab