diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e1db86219d3e4d1a8b7605cd857fda43ab730071 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile @@ -0,0 +1,14 @@ +FROM csst/csst-ifs-l1-cube-adm:latest +LABEL maintainer="Xiaoli Zhang" + +USER root +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh + +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin/ +RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc +#WORKDIR /work/ +WORKDIR /L1Pipeline/workspace + + +ENTRYPOINT [ "goagent" ] diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..29892e3eb8ff9b917140b400b325e7a40e0dec12 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile @@ -0,0 +1,22 @@ +IMAGE_NAME:=cnic/csst-ifs-l1-cube-adm + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar + ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar + ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar + ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) +scp: + scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube/adm/ +save: + docker save $(IMAGE_NAME) > /sharewcl/temp/ifs-l1-cube-adm.tar \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..18aa6807e8b9f23e06cf42155fe888911e361a9c --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# pipeline=/pipeline/output/pipeline.log +# if test -f "$pipeline"; then +# echo "=====pipeline.log====" > /work/user-file.txt +# cat /pipeline/output/pipeline.log >> /work/user-file.txt +# fi + +# timefile=/pipeline/output/timestamps.txt +# if test -f "$timefile"; then +# echo "$timefile exist" +# mv /pipeline/output/timestamps.txt /work/timestamps.txt +# fi + +# obsid="30100000001" + +python l1_pipeline_script_ifs_cube_adm.py > ./stdout + +exit_code=$? + +mkdir -p /work +# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt +# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt + +# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +# mv /tmp/timestamps.txt /work/timestamps.txt + +cat ./stdout + +mv /L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt +echo finish csst-ifs-l1-cube-adm. + +exit $exit_code diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..0587a591c6ccfed9817d57127ba0db9f24ec3e74 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile @@ -0,0 +1,11 @@ +FROM csst/csst-ifs-l1-cube-exe:latest + +USER root +COPY run.sh /app/bin/ +RUN chmod +x /app/bin/run.sh + +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ +RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc +WORKDIR /L1Pipeline/workspace + +ENTRYPOINT [ "goagent" ] diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..911419694e4a23c175774720b7cd468be8cb2441 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile @@ -0,0 +1,22 @@ +IMAGE_NAME:=cnic/csst-ifs-l1-cube-exe + +build: + docker build --network=host -t $(IMAGE_NAME) . + +dist: +#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' + docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar + ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar + ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar + ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar + +push: + docker push $(IMAGE_NAME) +run: + docker run -it --entrypoint bash $(IMAGE_NAME) +down: + docker stop $(IMAGE_NAME) +scp: + scp -r ./ csst-zjs:/root/csst/ifs-l1-cube/exe/ +save: + docker save $(IMAGE_NAME) > /sharewcl/temp/ifs-l1-cube-exe.tar \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..05e0cd3eb50fafef93532058016be885c9fe298e --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +echo input:$1* + +python l1_pipeline_script_ifs_cube_simdata.py $1 > ./stdout + +exit_code=$? + +mkdir -p /work +# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt +# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt + +# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* + +# mv /tmp/timestamps.txt /work/timestamps.txt + +echo finish csst-ifs-l1-cube-exe. + +exit $exit_code diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a5846ba0c15a76e1e820c83ee76e010937ca1d86 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile @@ -0,0 +1,18 @@ +CLUSTER=local + +all: reset build + +run: + scalebox app create --env-file zjs-ztm.env + +reset: + cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - + +down: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down + +list: + make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list + +scp: + scp -r ../ csst-zjs:/root/csst/csst-ifs-l1-cube/ \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59d8db6e68f69d1c65a9e43e68e07c887047e7f7 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml @@ -0,0 +1,69 @@ +name: ifs-l1-cube.app.process +label: 多次曝光合并 +comment: 积分场光谱仪多次曝光合并一级流水线 +cluster: local +parameters: + initial_status: RUNNING + +jobs: + + csst-ifs-l1-cube-adm: + label: 多次曝光合并管理模块 + base_image: cnic/csst-ifs-l1-cube-adm:latest + schedule_mode: HEAD + variables: + # always_running: yes + output_text_size: 100000 + # DFS入库错误重试3次 + retry_rules: "['202:3']" + text_tranc_mode: TAIL + locale_mode: NONE + # grpc_server: 172.24.23.6:50051 + parameters: + start_message: ANY + environments: + - CSST_DFS_API_MODE=cluster + - CSST_DFS_GATEWAY=172.24.27.2:30880 + - CSST_DFS_APP_ID=test + - CSST_DFS_APP_TOKEN=test + paths: # change /share to /sharewcl in PML production environment + - ${CSST_AUX_DIR}:/pipeline/aux + - ${CSST_DFS_ROOT}:/dfsroot + - ${CSST_CRDS_ROOT}:/crdsroot + sink_jobs: + - csst-ifs-l1-cube-exe + # command: > + # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" + # hosts: + # - h0:1 + # - c0:1 + # - c1:1 + # - c2:1 + csst-ifs-l1-cube-exe: + label: 多次曝光合并执行模块 + base_image: cnic/csst-ifs-l1-cube-exe:latest + schedule_mode: HEAD + variables: + # always_running: yes + output_text_size: 100000 + # DFS入库错误重试3次 + retry_rules: "['202:3']" + text_tranc_mode: TAIL + locale_mode: NONE + # grpc_server: 172.24.23.6:50051 + environments: + - CSST_DFS_API_MODE=cluster + - CSST_DFS_GATEWAY=172.24.27.2:30880 + - CSST_DFS_APP_ID=test + - CSST_DFS_APP_TOKEN=test + paths: # change /share to /sharewcl in PML production environment + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs + # command: > + # python /pipeline/src/run.py {\"brick_id\":1} + # hosts: + # - h0:1 + # - c0:1 + # - c1:1 + # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env b/L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env new file mode 100644 index 0000000000000000000000000000000000000000..d39a0bb7adeedae5775fb01496db97ed174e4a53 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env @@ -0,0 +1,17 @@ +# DFS +CSST_DFS_API_MODE=cluster +CSST_DFS_GATEWAY=172.24.27.2:30880 +CSST_DFS_APP_ID=test +CSST_DFS_APP_TOKEN=test +# CRDS +CRDS_SERVER_URL=http://172.24.27.2:29000 + +# VOLUMES ================== +CSST_DFS_ROOT=/share/dfs +CSST_CRDS_ROOT=/share/crdsdata/data +CSST_AUX_DIR=/share/pipeline/aux + +# IFS +AUX_DIR=/share/shao/L1Pipeline/aux +CRDS_DIR=/share/shao/L1Pipeline/crds +DFS_ROOT=/share/dfs \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env b/L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env new file mode 100644 index 0000000000000000000000000000000000000000..c16c2638e1c2e070f42f3a904a9bda41a2eef5e6 --- /dev/null +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env @@ -0,0 +1,17 @@ +# DFS +CSST_DFS_API_MODE=cluster +CSST_DFS_GATEWAY=172.24.27.2:30880 +CSST_DFS_APP_ID=test +CSST_DFS_APP_TOKEN=test +# CRDS +CRDS_SERVER_URL=http://172.24.27.2:29000 + +# VOLUMES ================== +CSST_DFS_ROOT=/sharewcl/dfs +CSST_CRDS_ROOT=/sharewcl/crdsdata/data +CSST_AUX_DIR=/sharewcl/pipeline/aux + +# IFS +AUX_DIR=/sharewcl/shao/L1Pipeline/aux +CRDS_DIR=/sharewcl/shao/L1Pipeline/crds +DFS_ROOT=/sharewcl/dfs \ No newline at end of file