From 28b3cedbc13c7b1c630ac9ecd973ffead505a37a Mon Sep 17 00:00:00 2001 From: zxl Date: Tue, 22 Aug 2023 10:43:33 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=AD=E6=9C=9F=E7=89=88=E6=9C=AC=E5=BD=92?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- L1/dockerfiles/adm-L1/Dockerfile | 2 +- L1/dockerfiles/csst-cpic-l1/Makefile | 15 ++- L1/dockerfiles/csst-hstdm-l1/Makefile | 15 ++- L1/dockerfiles/csst-hstdm-l1/run.sh | 6 +- L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile | 15 ++- L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh | 26 +--- L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile | 15 ++- L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh | 8 +- L1/dockerfiles/csst-ifs-l1-cube/test/Makefile | 4 +- L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml | 20 +-- L1/dockerfiles/csst-ifs-l1-rss/Dockerfile | 2 +- L1/dockerfiles/csst-ifs-l1-rss/Makefile | 15 ++- L1/dockerfiles/csst-ifs-l1-rss/run.sh | 2 +- L1/dockerfiles/csst-mci-l1/Dockerfile | 2 +- L1/dockerfiles/csst-mci-l1/Makefile | 15 ++- L1/dockerfiles/csst-mci-l1/test/app.yaml | 7 +- L1/dockerfiles/csst-msc-l1-mbi/Makefile | 22 ++-- L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile | 15 ++- L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile | 15 ++- L1/dockerfiles/csst-msc-l1-ooc/test/Makefile | 2 +- L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml | 16 ++- L1/dockerfiles/csst-msc-l1-phot/Makefile | 22 ++-- .../csst-msc-l1-sls1d/adm/Dockerfile | 2 +- L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile | 15 ++- .../csst-msc-l1-sls1d/exe/Dockerfile | 2 +- L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile | 21 ++- .../csst-msc-l1-sls1d/test/app.yaml | 16 ++- L1/dockerfiles/csst-msc-l1-sls2d/Makefile | 22 +++- .../csst-msc-l1-sls2d/test/Makefile | 2 +- L1/dockerfiles/redis-cli/Dockerfile | 2 +- L1/pipelines/app.yaml | 123 ++++++++++++++---- L1/pipelines/zjs-ztm.env | 17 +++ L1/pipelines/zjs.env | 6 +- L1/redis_server/README.md | 55 +++++++- L2/dockerfiles/csst-hstdm-l2/Makefile | 15 ++- L2/dockerfiles/csst-hstdm-l2/run.sh | 2 +- L2/dockerfiles/csst-hstdm-l2/test/app.yaml | 22 ++-- L2/dockerfiles/csst-msc-l2-xcat/Makefile | 20 --- .../csst-msc-l2-xcat/adm/Dockerfile | 2 +- L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile | 15 ++- L2/dockerfiles/csst-msc-l2-xcat/app.yaml | 78 ----------- .../csst-msc-l2-xcat/exe/Dockerfile | 2 +- L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile | 15 ++- L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml | 12 +- 44 files changed, 402 insertions(+), 325 deletions(-) create mode 100644 L1/pipelines/zjs-ztm.env delete mode 100644 L2/dockerfiles/csst-msc-l2-xcat/Makefile delete mode 100644 L2/dockerfiles/csst-msc-l2-xcat/app.yaml diff --git a/L1/dockerfiles/adm-L1/Dockerfile b/L1/dockerfiles/adm-L1/Dockerfile index 3515d7c..f540d5c 100644 --- a/L1/dockerfiles/adm-L1/Dockerfile +++ b/L1/dockerfiles/adm-L1/Dockerfile @@ -1,4 +1,4 @@ -FROM hub.cstcloud.cn/scalebox/agent:0.9 +FROM hub.cstcloud.cn/scalebox/agent LABEL maintainer="Xiaoli Zhang" diff --git a/L1/dockerfiles/csst-cpic-l1/Makefile b/L1/dockerfiles/csst-cpic-l1/Makefile index a38cda9..99ce550 100644 --- a/L1/dockerfiles/csst-cpic-l1/Makefile +++ b/L1/dockerfiles/csst-cpic-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-cpic-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/cpic-l1.tar - ssh c0 docker load -i /share/tmp/cpic-l1.tar - ssh c1 docker load -i /share/tmp/cpic-l1.tar - ssh c2 docker load -i /share/tmp/cpic-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/cpic-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/cpic-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-hstdm-l1/Makefile b/L1/dockerfiles/csst-hstdm-l1/Makefile index cf50ffd..d3da3a0 100644 --- a/L1/dockerfiles/csst-hstdm-l1/Makefile +++ b/L1/dockerfiles/csst-hstdm-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-hstdm-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/hstdm-l1.tar - ssh c0 docker load -i /share/tmp/hstdm-l1.tar - ssh c1 docker load -i /share/tmp/hstdm-l1.tar - ssh c2 docker load -i /share/tmp/hstdm-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-hstdm-l1/run.sh b/L1/dockerfiles/csst-hstdm-l1/run.sh index f739491..115da07 100644 --- a/L1/dockerfiles/csst-hstdm-l1/run.sh +++ b/L1/dockerfiles/csst-hstdm-l1/run.sh @@ -1,5 +1,6 @@ #!/bin/bash +rm -rf /pipeline/input/* /pipeline/output/* # obsid 50100000001 - 50100000002 obsid=$1 python /pipeline/src/run.py $obsid @@ -11,7 +12,10 @@ if test -f "$timefile"; then mv /pipeline/output/timestamps.txt /work/timestamps.txt fi -rm -rf /pipeline/input/* /pipeline/output/* +if [ $exit_code -eq 0 ]; then + echo "send message $obsid to hstdm-l2." + send-message $obsid +fi echo finish hstdm. diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile index 29892e3..990442c 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-cube-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh index 18aa680..e75d271 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh @@ -1,34 +1,14 @@ #!/bin/bash -# pipeline=/pipeline/output/pipeline.log -# if test -f "$pipeline"; then -# echo "=====pipeline.log====" > /work/user-file.txt -# cat /pipeline/output/pipeline.log >> /work/user-file.txt -# fi - -# timefile=/pipeline/output/timestamps.txt -# if test -f "$timefile"; then -# echo "$timefile exist" -# mv /pipeline/output/timestamps.txt /work/timestamps.txt -# fi - -# obsid="30100000001" - -python l1_pipeline_script_ifs_cube_adm.py > ./stdout +python l1_pipeline_script_ifs_cube_adm.py exit_code=$? mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt - -cat ./stdout mv /L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt +sed -i "s/^/${SINK_JOB},/g" /work/messages.txt + echo finish csst-ifs-l1-cube-adm. exit $exit_code diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile index 9114196..9740a4a 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-cube-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh index 05e0cd3..3dfb7f0 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh @@ -2,17 +2,11 @@ echo input:$1* -python l1_pipeline_script_ifs_cube_simdata.py $1 > ./stdout +python l1_pipeline_script_ifs_cube_simdata.py $1 exit_code=$? mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt echo finish csst-ifs-l1-cube-exe. diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile index a5846ba..3c5b525 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile @@ -1,9 +1,9 @@ -CLUSTER=local +CLUSTER=csst all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml index 59d8db6..3ac3472 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml @@ -1,7 +1,7 @@ name: ifs-l1-cube.app.process label: 多次曝光合并 comment: 积分场光谱仪多次曝光合并一级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -10,7 +10,7 @@ jobs: csst-ifs-l1-cube-adm: label: 多次曝光合并管理模块 base_image: cnic/csst-ifs-l1-cube-adm:latest - schedule_mode: HEAD + #schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -27,22 +27,23 @@ jobs: - CSST_DFS_APP_ID=test - CSST_DFS_APP_TOKEN=test paths: # change /share to /sharewcl in PML production environment - - ${CSST_AUX_DIR}:/pipeline/aux - - ${CSST_DFS_ROOT}:/dfsroot - - ${CSST_CRDS_ROOT}:/crdsroot + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs sink_jobs: - csst-ifs-l1-cube-exe # command: > # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 # - c2:1 + - n0:1 csst-ifs-l1-cube-exe: label: 多次曝光合并执行模块 base_image: cnic/csst-ifs-l1-cube-exe:latest - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -62,8 +63,9 @@ jobs: - ${DFS_ROOT}:/L1Pipeline/dfs # command: > # python /pipeline/src/run.py {\"brick_id\":1} - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 - # - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile b/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile index caee1d0..81bac12 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile +++ b/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang" USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin # RUN mkdir /work/ # WORKDIR /work/ diff --git a/L1/dockerfiles/csst-ifs-l1-rss/Makefile b/L1/dockerfiles/csst-ifs-l1-rss/Makefile index 1e8db7b..b3ba1b7 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-rss/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-rss - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-rss.tar - ssh c0 docker load -i /share/tmp/ifs-l1-rss.tar - ssh c1 docker load -i /share/tmp/ifs-l1-rss.tar - ssh c2 docker load -i /share/tmp/ifs-l1-rss.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-rss/run.sh b/L1/dockerfiles/csst-ifs-l1-rss/run.sh index 6a4dba9..55a72d4 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-rss/run.sh @@ -6,7 +6,7 @@ obsid=$1 # python l1_pipeline_script_mci_shao_simdata.py $obsid > ./stdout #python /L1Pipeline/workspace/helloworld.py $obsid -python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout +python l1_pipeline_script_ifs_rss_simdata.py $obsid exit_code=$? diff --git a/L1/dockerfiles/csst-mci-l1/Dockerfile b/L1/dockerfiles/csst-mci-l1/Dockerfile index 9b51842..888ccc6 100644 --- a/L1/dockerfiles/csst-mci-l1/Dockerfile +++ b/L1/dockerfiles/csst-mci-l1/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang" COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin RUN mkdir /work/ WORKDIR /work/ diff --git a/L1/dockerfiles/csst-mci-l1/Makefile b/L1/dockerfiles/csst-mci-l1/Makefile index 9377deb..021c83e 100644 --- a/L1/dockerfiles/csst-mci-l1/Makefile +++ b/L1/dockerfiles/csst-mci-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-mci-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/mci-l1.tar - ssh c0 docker load -i /share/tmp/mci-l1.tar - ssh c1 docker load -i /share/tmp/mci-l1.tar - ssh c2 docker load -i /share/tmp/mci-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/mci-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/mci-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-mci-l1/test/app.yaml b/L1/dockerfiles/csst-mci-l1/test/app.yaml index a906eca..7155494 100644 --- a/L1/dockerfiles/csst-mci-l1/test/app.yaml +++ b/L1/dockerfiles/csst-mci-l1/test/app.yaml @@ -6,7 +6,7 @@ parameters: jobs: mci-l1: base_image: cnic/csst-mci-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: output_text_size: 100000 text_tranc_mode: TAIL @@ -14,7 +14,7 @@ jobs: parameters: key_group_regex: ^(.{6})(.{3})$ key_group_seq: 1,2 - start_message: 20100000001 + # start_message: 20100000001 paths: - ${AUX_DIR}:/L1Pipeline/aux - ${CRDS_DIR}:/L1Pipeline/crds @@ -26,8 +26,9 @@ jobs: # - ${DFS_ROOT}:/dfsroot # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} # sink_jobs: - # hosts: + hosts: # - h0:1 # - c0:2 # - c1:2 # - c2:2 + - n0:1 diff --git a/L1/dockerfiles/csst-msc-l1-mbi/Makefile b/L1/dockerfiles/csst-msc-l1-mbi/Makefile index d86b3f2..c29bfa3 100644 --- a/L1/dockerfiles/csst-msc-l1-mbi/Makefile +++ b/L1/dockerfiles/csst-msc-l1-mbi/Makefile @@ -1,15 +1,21 @@ IMAGE_NAME:=cnic/csst-msc-l1-mbi - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-mbi.tar - ssh c0 docker load -i /share/tmp/msc-l1-mbi.tar - ssh c1 docker load -i /share/tmp/msc-l1-mbi.tar - ssh c2 docker load -i /share/tmp/msc-l1-mbi.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar push: docker push $(IMAGE_NAME) run: @@ -17,4 +23,4 @@ run: down: docker stop $(IMAGE_NAME) scp: - scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/ \ No newline at end of file + scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/ diff --git a/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile index 2f8ff0a..38c9af4 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-ooc-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-adm.tar - ssh c0 docker load -i /share/tmp/msc-l1-ooc-adm.tar - ssh c1 docker load -i /share/tmp/msc-l1-ooc-adm.tar - ssh c2 docker load -i /share/tmp/msc-l1-ooc-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile index 978950b..a27de40 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-ooc-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-exe.tar - ssh c0 docker load -i /share/tmp/msc-l1-ooc-exe.tar - ssh c1 docker load -i /share/tmp/msc-l1-ooc-exe.tar - ssh c2 docker load -i /share/tmp/msc-l1-ooc-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile index 4cdab67..6622c73 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile @@ -3,7 +3,7 @@ CLUSTER=local all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml index 39176d4..2c3e2dd 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml @@ -1,7 +1,7 @@ name: msc-l1-ooc.app.process -label: 在轨定标 +label: 主巡天定标 comment: 主巡天在轨定标一级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -37,9 +37,9 @@ jobs: # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" csst-msc-l1-ooc-exe: - label: 在轨定标执行模块 + label: 主巡天定标执行模块 base_image: cnic/csst-msc-l1-ooc-exe:latest - schedule_mode: HEAD + # schedule_mode: HEAD parameters: start_message: ANY variables: @@ -63,8 +63,10 @@ jobs: - ${CSST_CRDS_ROOT}:/crdsroot # command: > # python /pipeline/src/run.py {\"brick_id\":1} - # hosts: - # - h0:1 + hosts: + # - h0:1 # - c0:1 # - c1:1 - # - c2:1 \ No newline at end of file + # - c2:1 + - n0:1 + # - w2:1 \ No newline at end of file diff --git a/L1/dockerfiles/csst-msc-l1-phot/Makefile b/L1/dockerfiles/csst-msc-l1-phot/Makefile index f97a9b9..ff45863 100644 --- a/L1/dockerfiles/csst-msc-l1-phot/Makefile +++ b/L1/dockerfiles/csst-msc-l1-phot/Makefile @@ -1,15 +1,21 @@ IMAGE_NAME:=cnic/csst-msc-l1-phot - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-phot.tar - ssh c0 docker load -i /share/tmp/msc-l1-phot.tar - ssh c1 docker load -i /share/tmp/msc-l1-phot.tar - ssh c2 docker load -i /share/tmp/msc-l1-phot.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-phot.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar push: docker push $(IMAGE_NAME) run: @@ -17,4 +23,4 @@ run: down: docker stop $(IMAGE_NAME) scp: - scp -r ./ csst-zjs:/root/csst/csst-msc-l1-phot/ \ No newline at end of file + scp -r ./ csst-zjs:/root/csst/csst-msc-l1-phot/ diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile index d350295..7212449 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc # RUN mkdir -p /work /app/bin && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile index ab8d2c6..b0456ce 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls1d-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls1d-adm.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile index 5929097..792f41a 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile index de53f3a..74222c0 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile @@ -1,15 +1,22 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls1d-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls1d-exe.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml b/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml index bac4637..ee4dd81 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml +++ b/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml @@ -38,7 +38,8 @@ jobs: # - h0:1 # - c0:1 # - c1:1 - - c2:1 + # - c2:1 + - n0:1 csst-msc-l1-sls1d-exe: label: 无缝光谱1D执行模块 base_image: cnic/csst-msc-l1-sls1d-exe:latest @@ -65,6 +66,13 @@ jobs: # python /pipeline/src/run.py {\"brick_id\":1} hosts: # - h0:1 - # - c0:1 - # - c1:1 - - c2:2 \ No newline at end of file + - c0:10 + - c1:10 + - c2:10 + - n0:5 + - n1:15 + - w2:15 + - w3:15 + - w4:15 + - w5:15 + - w6:15 \ No newline at end of file diff --git a/L1/dockerfiles/csst-msc-l1-sls2d/Makefile b/L1/dockerfiles/csst-msc-l1-sls2d/Makefile index 6538ad5..71b9a04 100644 --- a/L1/dockerfiles/csst-msc-l1-sls2d/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls2d/Makefile @@ -1,14 +1,22 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls2d - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls2d.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls2d.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls2d.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls2d.tar + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + push: docker push $(IMAGE_NAME) run: @@ -18,4 +26,4 @@ down: scp: scp -r ./ csst-zjs:/root/csst/csst-msc-l1-sls2d/ save: - docker save $(IMAGE_NAME) > /sharewcl/temp/msc-l1-sls2d.tar \ No newline at end of file + docker save $(IMAGE_NAME) > /sharewcl/temp/msc-l1-sls2d.tar diff --git a/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile b/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile index 65fefa3..0e709ce 100644 --- a/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile @@ -3,7 +3,7 @@ CLUSTER=csst # all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/redis-cli/Dockerfile b/L1/dockerfiles/redis-cli/Dockerfile index 07c087e..1e668f2 100644 --- a/L1/dockerfiles/redis-cli/Dockerfile +++ b/L1/dockerfiles/redis-cli/Dockerfile @@ -1,4 +1,4 @@ -FROM hub.cstcloud.cn/scalebox/agent:0.9 +FROM hub.cstcloud.cn/scalebox/agent # 安装redis-cli RUN apt-get update \ diff --git a/L1/pipelines/app.yaml b/L1/pipelines/app.yaml index dfa3002..fc34202 100644 --- a/L1/pipelines/app.yaml +++ b/L1/pipelines/app.yaml @@ -1,5 +1,5 @@ -name: ${PIPELINE_NAME}.app.process -label: csst-${PIPELINE_NAME} +name: csst-L1.app.process +label: csst-L1 #output_basedir: /data/${PIPELINE_NAME} cluster: csst parameters: @@ -20,13 +20,15 @@ jobs: label: 消息队列接收模块 base_image: csst/redis-cli schedule_mode: HEAD + environments: + - REDIS_SERVER=192.169.23.2 # parameters: # start_message: abcd sink_jobs: - - admL1 - admL1: + - adm-L1 + adm-L1: label: 消息分发网关 - base_image: csst/adml1 + base_image: csst/adm-l1 schedule_mode: HEAD variables: repeated: yes @@ -37,6 +39,9 @@ jobs: - cpic-l1 - hstdm-l1 - ifs-l1-rss + - mci-l1 + environments: + - CSST_PIPELINE_GRPC_SERVER="192.169.23.2:50051" msc-l1-mbi: label: 多色成像 base_image: cnic/csst-msc-l1-mbi @@ -67,10 +72,11 @@ jobs: sink_jobs: - msc-l1-phot hosts: - - h0:1 - # - c0:1 - # - c1:1 - # - c2:1 +# - h0:1 + - c0:1 + - c[1-2]:2 + - w[2-6]:1 + - n1:2 msc-l1-phot: label: 测光 base_image: cnic/csst-msc-l1-phot @@ -100,17 +106,16 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% # sink_jobs: hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:1 + - w[2-6]:20 + - n1:5 +# - c[12]:12 msc-l1-sls2d: label: 无缝光谱2D base_image: cnic/csst-msc-l1-sls2d - schedule_mode: HEAD + # schedule_mode: HEAD variables: - always_running: yes - reserved_on_exit: yes + # always_running: yes + # reserved_on_exit: yes # DFS入库错误重试3次 retry_rules: "['202:3']" output_text_size: 100000 @@ -132,14 +137,13 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% # sink_jobs: hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:1 + - c0:1 + - w[2-6]:1 +# - c[0-2]:1 cpic-l1: label: 星冕仪 base_image: cnic/csst-cpic-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes # reserved_on_exit: yes @@ -167,8 +171,9 @@ jobs: hosts: # - h0:1 # - c0:1 - - c1:1 + # - c1:1 # - c2:1 + - n0:1 hstdm-l1: label: 太赫兹 base_image: cnic/csst-hstdm-l1 @@ -196,12 +201,45 @@ jobs: - ${CSST_DFS_ROOT}:/dfsroot - ${CSST_CRDS_ROOT}:/crdsref # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% - # sink_jobs: + sink_jobs: + - hstdm-l2 hosts: # - h0:1 # - c0:1 - - c1:1 + # - c1:1 + # - c2:1 + - n0:1 + hstdm-l2: + label: 太赫兹L2 + base_image: cnic/csst-hstdm-l2 + # schedule_mode: HEAD + variables: + # always_running: yes + # reserved_on_exit: yes + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + # start_message: 50100000001 + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + environments: + - CSST_DFS_API_MODE=${CSST_DFS_API_MODE} + - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY} + - CSST_DFS_APP_ID=${CSST_DFS_APP_ID} + - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN} + - CRDS_SERVER_URL=${CRDS_SERVER_URL} + paths: + - ${CSST_AUX_DIR}:/pipeline/aux + - ${CSST_DFS_ROOT}:/dfsroot + - ${CSST_CRDS_ROOT}:/crdsref + # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% + hosts: + # - h0:1 + # - c0:1 + # - c1:1 # - c2:1 + - n0:1 ifs-l1-rss: label: IFS单次曝光 base_image: cnic/csst-ifs-l1-rss @@ -221,10 +259,43 @@ jobs: - ${AUX_DIR}:/L1Pipeline/aux - ${CRDS_DIR}:/L1Pipeline/crds - ${DFS_ROOT}:/L1Pipeline/dfs + - /sharewcl/shao/docker/csst_ifs_l1_rss/code:/L1Pipeline/workspace # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} # sink_jobs: hosts: # - h0:1 - - c0:1 + # - c0:1 + # - c1:2 + # - c2:2 + - n0:1 + mci-l1: + label: 多通道成像仪定标 + base_image: cnic/csst-mci-l1 + # schedule_mode: HEAD + variables: + # DFS入库错误重试3次 + retry_rules: "['202:3']" + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + # start_message: 20100000001 + paths: + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs + - /sharewcl/shao/docker/csst_mci_l1/code:/L1Pipeline/workspace + # paths: + # - ${AUX_DIR}:/L1Pipeline/aux + # - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds + # - ${DFS_ROOT}:/dfsroot + # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} + # sink_jobs: + hosts: + # - h0:1 + # - c0:2 # - c1:2 - # - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 diff --git a/L1/pipelines/zjs-ztm.env b/L1/pipelines/zjs-ztm.env new file mode 100644 index 0000000..72ea99c --- /dev/null +++ b/L1/pipelines/zjs-ztm.env @@ -0,0 +1,17 @@ +# DFS +CSST_DFS_API_MODE=cluster +CSST_DFS_GATEWAY=172.24.27.2:30880 +CSST_DFS_APP_ID=test +CSST_DFS_APP_TOKEN=test +# CRDS +CRDS_SERVER_URL=http://172.24.27.2:29000 + +# VOLUMES ================== +CSST_DFS_ROOT=/share/dfs +CSST_CRDS_ROOT=/share/crdsdata/data +CSST_AUX_DIR=/share/pipeline/aux + +# IFS CHENWEI +AUX_DIR=/share/shao/L1Pipeline/aux +CRDS_DIR=/share/shao/L1Pipeline/crds +DFS_ROOT=/share/dfs \ No newline at end of file diff --git a/L1/pipelines/zjs.env b/L1/pipelines/zjs.env index ba290c3..cbb1b22 100644 --- a/L1/pipelines/zjs.env +++ b/L1/pipelines/zjs.env @@ -12,9 +12,9 @@ CSST_CRDS_ROOT=/sharewcl/crdsdata/data CSST_AUX_DIR=/sharewcl/pipeline/aux # IFS CHENWEI -AUX_DIR=/share/shao/L1Pipeline/aux -CRDS_DIR=/share/shao/L1Pipeline/crds -DFS_ROOT=/share/dfs +AUX_DIR=/sharewcl/shao/L1Pipeline/aux +CRDS_DIR=/sharewcl/shao/L1Pipeline/crds +DFS_ROOT=/sharewcl/dfs PIPELINE_NAME=csst_lever1 #PIPELINE_NAME=mbi diff --git a/L1/redis_server/README.md b/L1/redis_server/README.md index d289a88..52713f8 100644 --- a/L1/redis_server/README.md +++ b/L1/redis_server/README.md @@ -1,11 +1,60 @@ # 模拟DFS发送redis消息 ```bash -for line in {10160000000..10160000001}; do +# mbi sls2d +for line in {10160000000..10160000136}; do echo $line; - redis-cli -h 192.169.23.2 -a 123456 lpush single-image-reduction:obsid $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done +#mci +for line in {20100000001..20100000005}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#ifs-rss +for line in {30100000003..30100000050}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#cpic +for line in {41000000101..41000000104}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#hstdm +for line in {50100000001..50100000002}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; done ``` redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 10160000000; +redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 20100000001; redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 30100000003; redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 41000000101; -redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 50100000001; \ No newline at end of file +redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 50100000001; + + +## 数据库导入导出 +## 进入database容器 +### 数据导出 +pg_dump -h 127.0.0.1 -p 5432 -U scalebox -d scalebox -Fc -f ~/scalebox.sql +### 数据导入 +pg_restore -h 127.0.0.1 -U scalebox -d scalebox /scalebox.sql + + + +## 导数据步骤 +### 1紫金山考出数据 +docker exec cf32ca2842e8 pg_dump -U scalebox --data-only --dbname=scalebox > scalebox.sql + +### 2传入到要拷贝数据的数据库容器内 +docker cp scalebox.sql a3e398828977:/tmp/scalebox.sql + +### 3进入容器并登陆数据库 +docker exec -it a3e398828977 bash +psql -U scalebox -d scalebox + +### 4执行插入命令 +\i /tmp/scalebox.sql \ No newline at end of file diff --git a/L2/dockerfiles/csst-hstdm-l2/Makefile b/L2/dockerfiles/csst-hstdm-l2/Makefile index 570bb1b..c44d73f 100644 --- a/L2/dockerfiles/csst-hstdm-l2/Makefile +++ b/L2/dockerfiles/csst-hstdm-l2/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-hstdm-l2 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/hstdm-l2.tar - ssh c0 docker load -i /share/tmp/hstdm-l2.tar - ssh c1 docker load -i /share/tmp/hstdm-l2.tar - ssh c2 docker load -i /share/tmp/hstdm-l2.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l2.tar + ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l2.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-hstdm-l2/run.sh b/L2/dockerfiles/csst-hstdm-l2/run.sh index f739491..02ec2c7 100644 --- a/L2/dockerfiles/csst-hstdm-l2/run.sh +++ b/L2/dockerfiles/csst-hstdm-l2/run.sh @@ -13,6 +13,6 @@ fi rm -rf /pipeline/input/* /pipeline/output/* -echo finish hstdm. +echo finish hstdm-l2. exit $exit_code diff --git a/L2/dockerfiles/csst-hstdm-l2/test/app.yaml b/L2/dockerfiles/csst-hstdm-l2/test/app.yaml index 07e46c4..68ef754 100644 --- a/L2/dockerfiles/csst-hstdm-l2/test/app.yaml +++ b/L2/dockerfiles/csst-hstdm-l2/test/app.yaml @@ -1,7 +1,7 @@ name: hstdm.app.process label: 太赫兹 comment: 精测模块太赫兹1+2级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -9,7 +9,7 @@ jobs: hstdm-l1: label: 太赫兹 base_image: cnic/csst-hstdm-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes # reserved_on_exit: yes @@ -33,18 +33,19 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% sink_jobs: - hstdm-l2 - # hosts: - # - h0:1 - # - c0:1 + hosts: + # - h0:1 + # - c0:1 # - c1:1 # - c2:1 + - n0:1 hstdm-l2: label: 太赫兹L2 base_image: cnic/csst-hstdm-l2 - schedule_mode: HEAD + # schedule_mode: HEAD variables: - always_running: yes - reserved_on_exit: yes + # always_running: yes + # reserved_on_exit: yes output_text_size: 100000 text_tranc_mode: TAIL locale_mode: NONE @@ -63,10 +64,11 @@ jobs: - ${CSST_DFS_ROOT}:/dfsroot - ${CSST_CRDS_ROOT}:/crdsref # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 - # - c2:1 + # - c2:1 + - n0:1 diff --git a/L2/dockerfiles/csst-msc-l2-xcat/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/Makefile deleted file mode 100644 index 7508326..0000000 --- a/L2/dockerfiles/csst-msc-l2-xcat/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -IMAGE_ADM:=cnic/csst-msc-l1-mbi - -build: - make -C adm build - make -C exe build - -sync: - rsync -av --del . csst-zjs:/tmp/csst-msc-l2-xcat - -dist: - docker save hub.cstcloud.cn/scalebox/base:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - # docker save hub.cstcloud.cn/scalebox/controld:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - # docker save hub.cstcloud.cn/scalebox/database:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - -push: - docker push $(IMAGE_NAME) -run: - docker run -it --entrypoint bash $(IMAGE_NAME) -down: - docker stop $(IMAGE_NAME) diff --git a/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile b/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile index 2031bd2..6420be8 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile +++ b/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc # RUN mkdir -p /work /app/bin && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile index 2e5cff1..be6869f 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile +++ b/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l2-xcat-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-adm.tar - ssh c0 docker load -i /share/tmp/msc-l2-xcat-adm.tar - ssh c1 docker load -i /share/tmp/msc-l2-xcat-adm.tar - ssh c2 docker load -i /share/tmp/msc-l2-xcat-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-msc-l2-xcat/app.yaml b/L2/dockerfiles/csst-msc-l2-xcat/app.yaml deleted file mode 100644 index cae73f8..0000000 --- a/L2/dockerfiles/csst-msc-l2-xcat/app.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: msc-l2-xcat.app.process -label: 星表检索 -comment: 星表检索 -cluster: csst -parameters: - initial_status: RUNNING - -jobs: - # msc-l1-mbi: - # label: 星表管理模块 - # base_image: cnic/csst-msc-l1-mbi - # schedule_mode: HEAD - # variables: - # always_runing: yes - # output_text_size: 100000 - # text_tranc_mode: TAIL - # locale_mode: NONE - # paths: - # - ${AUX_DIR}:/pipeline/aux - # - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds - # - ${DFS_ROOT}:/dfsroot - - csst-msc-l2-xcat-adm: - label: 星表管理模块 - base_image: cnic/csst-msc-l2-xcat-adm:latest - schedule_mode: HEAD - variables: - #always_running: yes - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - parameters: - start_message: 202306212339 - environments: - - CSST_DFS_API_MODE=cluster - - CSST_DFS_GATEWAY=172.24.27.2:30880 - - CSST_DFS_APP_ID=test - - CSST_DFS_APP_TOKEN=test - paths: # change /share to /sharewcl in PML production environment - - /sharewcl/pipeline/output-csst-msc-l2-xcat-adm:/pipeline/output # output directory, omit this line in production - - /sharewcl/pipeline/aux:/pipeline/aux - - /sharewcl/dfs:/dfsroot - - /sharewcl/crdsdata/data/references:/crdsref - sink_jobs: - - csst-msc-l2-xcat-exe - # command: > - # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - hosts: - # - h0:1 - # - c0:1 - # - c1:1 - # - c2:1 - csst-msc-l2-xcat-exe: - base_image: cnic/csst-msc-l2-xcat-exe:latest - #schedule_mode: HEAD - variables: - always_running: yes - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - environments: - - CSST_DFS_API_MODE=cluster - - CSST_DFS_GATEWAY=172.24.27.2:30880 - - CSST_DFS_APP_ID=test - - CSST_DFS_APP_TOKEN=test - paths: # change /share to /sharewcl in PML production environment - # output directory, omit this line in production - - /sharewcl/pipeline/output-csst-msc-l2-xcat-exe:/pipeline/output - - /sharewcl/pipeline/aux:/pipeline/aux - - /sharewcl/dfs:/dfsroot - - /sharewcl/crdsdata/data/references:/crdsref - # command: > - # python /pipeline/src/run.py {\"brick_id\":1} - hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:2 \ No newline at end of file diff --git a/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile b/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile index 74e3e25..1793ca9 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile +++ b/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile index ad4976b..cf70426 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile +++ b/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l2-xcat-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-exe.tar - ssh c0 docker load -i /share/tmp/msc-l2-xcat-exe.tar - ssh c1 docker load -i /share/tmp/msc-l2-xcat-exe.tar - ssh c2 docker load -i /share/tmp/msc-l2-xcat-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml b/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml index fe71971..8ea576e 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml +++ b/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml @@ -9,7 +9,7 @@ jobs: csst-msc-l2-xcat-adm: label: 星表合并管理模块 base_image: cnic/csst-msc-l2-xcat-adm:latest - schedule_mode: HEAD + #schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -30,7 +30,12 @@ jobs: - csst-msc-l2-xcat-exe # command: > # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - + hosts: + # - h0:1 + # - c0:1 + # - c1:1 + # - c2:2 + - n0:1 csst-msc-l2-xcat-exe: label: 星表合并执行模块 base_image: cnic/csst-msc-l2-xcat-exe:latest @@ -58,4 +63,5 @@ jobs: # - h0:1 # - c0:1 # - c1:1 - - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 \ No newline at end of file -- GitLab