From 92ceb371faf34c8a0dde893ccb2e187380404f4a Mon Sep 17 00:00:00 2001 From: zxl Date: Tue, 17 Jan 2023 17:03:28 +0800 Subject: [PATCH] Package and integrate mbi and sls2d --- L1/dockerfiles/mbi/Dockerfile | 10 ++++++ L1/dockerfiles/mbi/Makefile | 18 ++++++++++ L1/dockerfiles/mbi/run.sh | 36 ++++++++++++++++++++ L1/dockerfiles/mbi/test/Makefile | 16 +++++++++ L1/dockerfiles/mbi/test/app.yaml | 54 ++++++++++++++++++++++++++++++ L1/dockerfiles/sls2d/Dockerfile | 11 ++++++ L1/dockerfiles/sls2d/Makefile | 17 ++++++++++ L1/dockerfiles/sls2d/run.sh | 36 ++++++++++++++++++++ L1/dockerfiles/sls2d/test/app.yaml | 25 ++++++++++++++ 9 files changed, 223 insertions(+) create mode 100644 L1/dockerfiles/mbi/Dockerfile create mode 100644 L1/dockerfiles/mbi/Makefile create mode 100644 L1/dockerfiles/mbi/run.sh create mode 100644 L1/dockerfiles/mbi/test/Makefile create mode 100644 L1/dockerfiles/mbi/test/app.yaml create mode 100644 L1/dockerfiles/sls2d/Dockerfile create mode 100644 L1/dockerfiles/sls2d/Makefile create mode 100644 L1/dockerfiles/sls2d/run.sh create mode 100644 L1/dockerfiles/sls2d/test/app.yaml diff --git a/L1/dockerfiles/mbi/Dockerfile b/L1/dockerfiles/mbi/Dockerfile new file mode 100644 index 0000000..f2e39d1 --- /dev/null +++ b/L1/dockerfiles/mbi/Dockerfile @@ -0,0 +1,10 @@ +FROM csst-l1ppl:latest + +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin + +RUN mkdir /work/ +WORKDIR /work/ + +ENTRYPOINT ["goagent"] \ No newline at end of file diff --git a/L1/dockerfiles/mbi/Makefile b/L1/dockerfiles/mbi/Makefile new file mode 100644 index 0000000..d3ab275 --- /dev/null +++ b/L1/dockerfiles/mbi/Makefile @@ -0,0 +1,18 @@ +IMAGE_NAME:=csst/mbi + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/mbi.tar + ssh c0 docker load -i /share/tmp/mbi.tar + ssh c1 docker load -i /share/tmp/mbi.tar + ssh c2 docker load -i /share/tmp/mbi.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) diff --git a/L1/dockerfiles/mbi/run.sh b/L1/dockerfiles/mbi/run.sh new file mode 100644 index 0000000..69b5fa9 --- /dev/null +++ b/L1/dockerfiles/mbi/run.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# clear directories +rm -rf /L1Pipeline/L0/* +rm -rf /L1Pipeline/L1/* + +# obsid 100000100 ~ 100000154 +python /L1Pipeline/build/csst_l1/app/l1_mbi.py $1 > /work/stdout 2> /work/stderr + +exit_code=$? + +# 标准错误和标准输出，输出到文件 +echo "obsid : "$1 >> /var/log/scalebox/stdout +cat /work/stdout >> /var/log/scalebox/stdout + +echo "obsid : "$1 >> /var/log/scalebox/stderr +cat /work/stderr >> /var/log/scalebox/stderr + +rm -f /work/stdout /work/stderr + +obsid=$1 +#cat /L1Pipeline/L1/csst-l1ppl.log +#cat /L1Pipeline/L1/csst-l1ppl.log >> /var/log/scalebox/${obsid}.log +#创建临时文件 +touch /L1Pipeline/L1/csst-l1mod-tmp.log +#取/L1Pipeline/L1/csst-l1mod.log后100行 +cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 > /L1Pipeline/L1/csst-l1mod-tmp.log +# log文件记录到标准错误或标准输出 +cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log +cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log >> /var/log/scalebox/${obsid}.log + +rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +echo finish mbi. + +exit $exit_code \ No newline at end of file diff --git a/L1/dockerfiles/mbi/test/Makefile b/L1/dockerfiles/mbi/test/Makefile new file mode 100644 index 0000000..2024898 --- /dev/null +++ b/L1/dockerfiles/mbi/test/Makefile @@ -0,0 +1,16 @@ +CLUSTER=csst + +all: reset build + +run: + scalebox app create + +reset: + cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - + +down: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down + +list: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list + diff --git a/L1/dockerfiles/mbi/test/app.yaml b/L1/dockerfiles/mbi/test/app.yaml new file mode 100644 index 0000000..a5b4d2a --- /dev/null +++ b/L1/dockerfiles/mbi/test/app.yaml @@ -0,0 +1,54 @@ +name: mbi.app.process +label: csst-mbi +cluster: csst +parameters: + initial_status: RUNNING + +jobs: + redis-cli: + label: 消息队列接收模块 + base_image: csst/redis-cli + schedule_mode: HEAD + parameters: + start_message: abcd + sink_jobs: + - mbi + admL1: + label: 1级流水线消息网关 + base_image: csst/adml1 + schedule_mode: HEAD + # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% + sink_jobs: + - mbi + mbi: + label: 多色成像 + base_image: csst/mbi + schedule_mode: HEAD + paths: + - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux + - /sharewcl/dfs:/dfsroot + # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% + # sink_jobs: + hosts: + - h0:1 + - c0:2 + - c1:2 + - c2:2 + sls2d: + label: 无缝光谱 + base_image: csst/sls2d + schedule_mode: HEAD + paths: + - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux + - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds + - /sharewcl/dfs:/dfsroot + # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% + # sink_jobs: + hosts: + - h0:1 + - c0:2 + - c1:2 + - c2:2 + + + diff --git a/L1/dockerfiles/sls2d/Dockerfile b/L1/dockerfiles/sls2d/Dockerfile new file mode 100644 index 0000000..12b56b9 --- /dev/null +++ b/L1/dockerfiles/sls2d/Dockerfile @@ -0,0 +1,11 @@ +FROM csst-l1ppl:6.0.137 + + +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin + +RUN mkdir /work/ +WORKDIR /work/ + +ENTRYPOINT ["goagent"] diff --git a/L1/dockerfiles/sls2d/Makefile b/L1/dockerfiles/sls2d/Makefile new file mode 100644 index 0000000..fe8212e --- /dev/null +++ b/L1/dockerfiles/sls2d/Makefile @@ -0,0 +1,17 @@ +IMAGE_NAME:=csst/sls2d + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: build + #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/sls2d.tar + ssh c0 docker load -i /share/tmp/sls2d.tar + ssh c1 docker load -i /share/tmp/sls2d.tar + ssh c2 docker load -i /share/tmp/sls2d.tar +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) diff --git a/L1/dockerfiles/sls2d/run.sh b/L1/dockerfiles/sls2d/run.sh new file mode 100644 index 0000000..1336b15 --- /dev/null +++ b/L1/dockerfiles/sls2d/run.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# clear directories +rm -rf /L1Pipeline/L0/* +rm -rf /L1Pipeline/L1/* + +# obsid 100000100 ~ 100000154 +python /L1Pipeline/build/csst_l1/app/l1_sls2d.py $1 > /work/stdout 2> /work/stderr + +exit_code=$? + +# 标准错误和标准输出，输出到文件 +echo "obsid : "$1 >> /var/log/scalebox/stdout +cat /work/stdout >> /var/log/scalebox/stdout + +echo "obsid : "$1 >> /var/log/scalebox/stderr +cat /work/stderr >> /var/log/scalebox/stderr + +rm -f /work/stdout /work/stderr + +obsid=$1 +#cat /L1Pipeline/L1/csst-l1ppl.log +#cat /L1Pipeline/L1/csst-l1ppl.log >> /var/log/scalebox/${obsid}.log +#创建临时文件 +touch /L1Pipeline/L1/csst-l1mod-tmp.log +#取/L1Pipeline/L1/csst-l1mod.log后100行 +cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 > /L1Pipeline/L1/csst-l1mod-tmp.log +# log文件记录到标准错误或标准输出 +cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log +cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log >> /var/log/scalebox/${obsid}.log + +rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +echo finish sls2d. + +exit $exit_code diff --git a/L1/dockerfiles/sls2d/test/app.yaml b/L1/dockerfiles/sls2d/test/app.yaml new file mode 100644 index 0000000..ea2e2db --- /dev/null +++ b/L1/dockerfiles/sls2d/test/app.yaml @@ -0,0 +1,25 @@ +name: sls2d.app.process +label: csst-sls2d +cluster: csst +parameters: + initial_status: RUNNING + +jobs: + sls2d: + label: 无缝光谱2D + base_image: csst/sls2d + schedule_mode: HEAD + variables: + repeated: yes + parameters: + start_message: 100000100 + paths: + - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux + - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds + - /sharewcl/dfs:/dfsroot + # sink_jobs: + # hosts: + # - h0:1 + # - c0:1 + # - c1:1 + # - c2:1 -- GitLab