diff --git a/L1/dockerfiles/adm-L1/Dockerfile b/L1/dockerfiles/adm-L1/Dockerfile index 3515d7c7313eac33225158b1f64dc55a218c313c..f540d5cd3d232ab15540da5d1d285a3541b8d1e8 100644 --- a/L1/dockerfiles/adm-L1/Dockerfile +++ b/L1/dockerfiles/adm-L1/Dockerfile @@ -1,4 +1,4 @@ -FROM hub.cstcloud.cn/scalebox/agent:0.9 +FROM hub.cstcloud.cn/scalebox/agent LABEL maintainer="Xiaoli Zhang" diff --git a/L1/dockerfiles/csst-cpic-l1/Makefile b/L1/dockerfiles/csst-cpic-l1/Makefile index a38cda90cdff5c6f3ceb3aa4ebf2ef2f987bec03..99ce550e2a4cf18dc9ed3b5d30a9f17bb67bd4c9 100644 --- a/L1/dockerfiles/csst-cpic-l1/Makefile +++ b/L1/dockerfiles/csst-cpic-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-cpic-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/cpic-l1.tar - ssh c0 docker load -i /share/tmp/cpic-l1.tar - ssh c1 docker load -i /share/tmp/cpic-l1.tar - ssh c2 docker load -i /share/tmp/cpic-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/cpic-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/cpic-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/cpic-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-hstdm-l1/Makefile b/L1/dockerfiles/csst-hstdm-l1/Makefile index cf50ffd4998ad7daab3d8182036b37978c4bb1ad..d3da3a01985fa219160251d8d4eb4d67f9021fd4 100644 --- a/L1/dockerfiles/csst-hstdm-l1/Makefile +++ b/L1/dockerfiles/csst-hstdm-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-hstdm-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/hstdm-l1.tar - ssh c0 docker load -i /share/tmp/hstdm-l1.tar - ssh c1 docker load -i /share/tmp/hstdm-l1.tar - ssh c2 docker load -i /share/tmp/hstdm-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-hstdm-l1/run.sh b/L1/dockerfiles/csst-hstdm-l1/run.sh index f73949105faaeb7dfc5efc99a1d408c0626fd827..115da0725a5f459813eb33979d428c4d90478fa8 100644 --- a/L1/dockerfiles/csst-hstdm-l1/run.sh +++ b/L1/dockerfiles/csst-hstdm-l1/run.sh @@ -1,5 +1,6 @@ #!/bin/bash +rm -rf /pipeline/input/* /pipeline/output/* # obsid 50100000001 - 50100000002 obsid=$1 python /pipeline/src/run.py $obsid @@ -11,7 +12,10 @@ if test -f "$timefile"; then mv /pipeline/output/timestamps.txt /work/timestamps.txt fi -rm -rf /pipeline/input/* /pipeline/output/* +if [ $exit_code -eq 0 ]; then + echo "send message $obsid to hstdm-l2." + send-message $obsid +fi echo finish hstdm. diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile index 29892e3eb8ff9b917140b400b325e7a40e0dec12..990442cba95ee8996b0d5070bd8c112dd3a2cab8 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-cube-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh index 18aa6807e8b9f23e06cf42155fe888911e361a9c..e75d271a6aecb37f0f201257cd0973eb60b6d89c 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh @@ -1,34 +1,14 @@ #!/bin/bash -# pipeline=/pipeline/output/pipeline.log -# if test -f "$pipeline"; then -# echo "=====pipeline.log====" > /work/user-file.txt -# cat /pipeline/output/pipeline.log >> /work/user-file.txt -# fi - -# timefile=/pipeline/output/timestamps.txt -# if test -f "$timefile"; then -# echo "$timefile exist" -# mv /pipeline/output/timestamps.txt /work/timestamps.txt -# fi - -# obsid="30100000001" - -python l1_pipeline_script_ifs_cube_adm.py > ./stdout +python l1_pipeline_script_ifs_cube_adm.py exit_code=$? mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt - -cat ./stdout mv /L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt +sed -i "s/^/${SINK_JOB},/g" /work/messages.txt + echo finish csst-ifs-l1-cube-adm. exit $exit_code diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile index 911419694e4a23c175774720b7cd468be8cb2441..9740a4ad66e43d1accb2a5cacab27ca1bd729742 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-cube-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh index 05e0cd3eb50fafef93532058016be885c9fe298e..3dfb7f0b0bf1684925ce9b4faf534f3c29ef2085 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh @@ -2,17 +2,11 @@ echo input:$1* -python l1_pipeline_script_ifs_cube_simdata.py $1 > ./stdout +python l1_pipeline_script_ifs_cube_simdata.py $1 exit_code=$? mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt echo finish csst-ifs-l1-cube-exe. diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile index a5846ba0c15a76e1e820c83ee76e010937ca1d86..3c5b5257491149f2e5c3911568e018c7af578828 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/Makefile @@ -1,9 +1,9 @@ -CLUSTER=local +CLUSTER=csst all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml index 59d8db6e68f69d1c65a9e43e68e07c887047e7f7..3ac3472c47066d0c2195c2a0e90bd14ab28f9b91 100644 --- a/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml +++ b/L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml @@ -1,7 +1,7 @@ name: ifs-l1-cube.app.process label: 多次曝光合并 comment: 积分场光谱仪多次曝光合并一级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -10,7 +10,7 @@ jobs: csst-ifs-l1-cube-adm: label: 多次曝光合并管理模块 base_image: cnic/csst-ifs-l1-cube-adm:latest - schedule_mode: HEAD + #schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -27,22 +27,23 @@ jobs: - CSST_DFS_APP_ID=test - CSST_DFS_APP_TOKEN=test paths: # change /share to /sharewcl in PML production environment - - ${CSST_AUX_DIR}:/pipeline/aux - - ${CSST_DFS_ROOT}:/dfsroot - - ${CSST_CRDS_ROOT}:/crdsroot + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs sink_jobs: - csst-ifs-l1-cube-exe # command: > # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 # - c2:1 + - n0:1 csst-ifs-l1-cube-exe: label: 多次曝光合并执行模块 base_image: cnic/csst-ifs-l1-cube-exe:latest - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -62,8 +63,9 @@ jobs: - ${DFS_ROOT}:/L1Pipeline/dfs # command: > # python /pipeline/src/run.py {\"brick_id\":1} - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 - # - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile b/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile index caee1d01476914b0cac26c6f3626fe090b610240..81bac1203068c104686dedc2b1caef82f9148dc6 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile +++ b/L1/dockerfiles/csst-ifs-l1-rss/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang" USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin # RUN mkdir /work/ # WORKDIR /work/ diff --git a/L1/dockerfiles/csst-ifs-l1-rss/Makefile b/L1/dockerfiles/csst-ifs-l1-rss/Makefile index 1e8db7b26030e551e3cc7d9e1ea43921dafbea41..b3ba1b77934336055cc108eaee0587cddc9555a8 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/Makefile +++ b/L1/dockerfiles/csst-ifs-l1-rss/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-ifs-l1-rss - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-rss.tar - ssh c0 docker load -i /share/tmp/ifs-l1-rss.tar - ssh c1 docker load -i /share/tmp/ifs-l1-rss.tar - ssh c2 docker load -i /share/tmp/ifs-l1-rss.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar + ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-ifs-l1-rss/run.sh b/L1/dockerfiles/csst-ifs-l1-rss/run.sh index 6a4dba9e91fc66e29736649815dc14a56f1c6b04..55a72d4bfcc6312f49c90913164da8daca5f7b4d 100644 --- a/L1/dockerfiles/csst-ifs-l1-rss/run.sh +++ b/L1/dockerfiles/csst-ifs-l1-rss/run.sh @@ -6,7 +6,7 @@ obsid=$1 # python l1_pipeline_script_mci_shao_simdata.py $obsid > ./stdout #python /L1Pipeline/workspace/helloworld.py $obsid -python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout +python l1_pipeline_script_ifs_rss_simdata.py $obsid exit_code=$? diff --git a/L1/dockerfiles/csst-mci-l1/Dockerfile b/L1/dockerfiles/csst-mci-l1/Dockerfile index 9b5184293905c5b339fe65ae4110cd2bdcc56850..888ccc6001a73bd5cca53a9e61e2c0c2f63160a6 100644 --- a/L1/dockerfiles/csst-mci-l1/Dockerfile +++ b/L1/dockerfiles/csst-mci-l1/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang" COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin RUN mkdir /work/ WORKDIR /work/ diff --git a/L1/dockerfiles/csst-mci-l1/Makefile b/L1/dockerfiles/csst-mci-l1/Makefile index 9377deb5d1baf55c59e7675426ec6290107a6ce1..021c83e9e4575a22873c4e0b492b5b13ea7fd91e 100644 --- a/L1/dockerfiles/csst-mci-l1/Makefile +++ b/L1/dockerfiles/csst-mci-l1/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-mci-l1 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/mci-l1.tar - ssh c0 docker load -i /share/tmp/mci-l1.tar - ssh c1 docker load -i /share/tmp/mci-l1.tar - ssh c2 docker load -i /share/tmp/mci-l1.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/mci-l1.tar + ssh c0 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh c1 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh c2 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh n0 docker load -i $(IMAGE_PATH)/mci-l1.tar + ssh n1 docker load -i $(IMAGE_PATH)/mci-l1.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-mci-l1/test/app.yaml b/L1/dockerfiles/csst-mci-l1/test/app.yaml index a906eca30a7b7a8fa7e5d658ed296fd631c69189..715549456bcfe51b623cfdf16d2de0db89bd9c22 100644 --- a/L1/dockerfiles/csst-mci-l1/test/app.yaml +++ b/L1/dockerfiles/csst-mci-l1/test/app.yaml @@ -6,7 +6,7 @@ parameters: jobs: mci-l1: base_image: cnic/csst-mci-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: output_text_size: 100000 text_tranc_mode: TAIL @@ -14,7 +14,7 @@ jobs: parameters: key_group_regex: ^(.{6})(.{3})$ key_group_seq: 1,2 - start_message: 20100000001 + # start_message: 20100000001 paths: - ${AUX_DIR}:/L1Pipeline/aux - ${CRDS_DIR}:/L1Pipeline/crds @@ -26,8 +26,9 @@ jobs: # - ${DFS_ROOT}:/dfsroot # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} # sink_jobs: - # hosts: + hosts: # - h0:1 # - c0:2 # - c1:2 # - c2:2 + - n0:1 diff --git a/L1/dockerfiles/csst-msc-l1-mbi/Makefile b/L1/dockerfiles/csst-msc-l1-mbi/Makefile index d86b3f213bd6e1a09381cf3f28cbe04be638ace1..c29bfa30a6dc443c731650ce517421c1a6aa4b27 100644 --- a/L1/dockerfiles/csst-msc-l1-mbi/Makefile +++ b/L1/dockerfiles/csst-msc-l1-mbi/Makefile @@ -1,15 +1,21 @@ IMAGE_NAME:=cnic/csst-msc-l1-mbi - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-mbi.tar - ssh c0 docker load -i /share/tmp/msc-l1-mbi.tar - ssh c1 docker load -i /share/tmp/msc-l1-mbi.tar - ssh c2 docker load -i /share/tmp/msc-l1-mbi.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar push: docker push $(IMAGE_NAME) run: @@ -17,4 +23,4 @@ run: down: docker stop $(IMAGE_NAME) scp: - scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/ \ No newline at end of file + scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/ diff --git a/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile index 2f8ff0a4fc549b6971181b90428e2902a28ac47a..38c9af4251b78269a553242f21a41bf5a82a88be 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-ooc-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-adm.tar - ssh c0 docker load -i /share/tmp/msc-l1-ooc-adm.tar - ssh c1 docker load -i /share/tmp/msc-l1-ooc-adm.tar - ssh c2 docker load -i /share/tmp/msc-l1-ooc-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile index 978950b6b0b2bc31bde534632910c2b4cd95db87..a27de4061e4f096c9cf292769d1fef84c24c9d7b 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-ooc-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-exe.tar - ssh c0 docker load -i /share/tmp/msc-l1-ooc-exe.tar - ssh c1 docker load -i /share/tmp/msc-l1-ooc-exe.tar - ssh c2 docker load -i /share/tmp/msc-l1-ooc-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile index 4cdab67e08e9a38f980a20fc63a68eb7084ac591..6622c734e0deef6e8be8e029df7ca6e97334dde4 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile @@ -3,7 +3,7 @@ CLUSTER=local all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml index 39176d4af7fd0db27de02942f246a446ab7cb8d0..2c3e2ddb7a74024eee0734cf9d5319cc98ebeb06 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml @@ -1,7 +1,7 @@ name: msc-l1-ooc.app.process -label: 在轨定标 +label: 主巡天定标 comment: 主巡天在轨定标一级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -37,9 +37,9 @@ jobs: # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" csst-msc-l1-ooc-exe: - label: 在轨定标执行模块 + label: 主巡天定标执行模块 base_image: cnic/csst-msc-l1-ooc-exe:latest - schedule_mode: HEAD + # schedule_mode: HEAD parameters: start_message: ANY variables: @@ -63,8 +63,10 @@ jobs: - ${CSST_CRDS_ROOT}:/crdsroot # command: > # python /pipeline/src/run.py {\"brick_id\":1} - # hosts: - # - h0:1 + hosts: + # - h0:1 # - c0:1 # - c1:1 - # - c2:1 \ No newline at end of file + # - c2:1 + - n0:1 + # - w2:1 \ No newline at end of file diff --git a/L1/dockerfiles/csst-msc-l1-phot/Makefile b/L1/dockerfiles/csst-msc-l1-phot/Makefile index f97a9b9c361c84ce6068114ad63336eb2af646d6..ff45863554f109844f93a23a95f65d7a3f10dbe8 100644 --- a/L1/dockerfiles/csst-msc-l1-phot/Makefile +++ b/L1/dockerfiles/csst-msc-l1-phot/Makefile @@ -1,15 +1,21 @@ IMAGE_NAME:=cnic/csst-msc-l1-phot - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-phot.tar - ssh c0 docker load -i /share/tmp/msc-l1-phot.tar - ssh c1 docker load -i /share/tmp/msc-l1-phot.tar - ssh c2 docker load -i /share/tmp/msc-l1-phot.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-phot.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-phot.tar push: docker push $(IMAGE_NAME) run: @@ -17,4 +23,4 @@ run: down: docker stop $(IMAGE_NAME) scp: - scp -r ./ csst-zjs:/root/csst/csst-msc-l1-phot/ \ No newline at end of file + scp -r ./ csst-zjs:/root/csst/csst-msc-l1-phot/ diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile index d35029539cfd7f465dd0d36febb33398014581e7..721244910d27174fc35f05d59dfd9536b0bdbfd5 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc # RUN mkdir -p /work /app/bin && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile index ab8d2c6e8aec99bf0a15dffc2b71ea891d766c33..b0456ce1ed991fd71745b2f9e8ff62f677393558 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls1d-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls1d-adm.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls1d-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile index 59290976de85e20d42162464a3389eccc82a7e57..792f41af5cdee86c9ceb5b244435c8f81b9d7748 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile index de53f3a174810c6cd9592686af5a65ebd8f31690..74222c06efd44d5c56dd507cd7d381a16a0d47a4 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls1d/exe/Makefile @@ -1,15 +1,22 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls1d-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls1d-exe.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls1d-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-sls1d-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml b/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml index bac4637b39e336b84868fed1fc8b6bac002a72ca..ee4dd81307d94829042b014fb0fb6ba84b320c7f 100644 --- a/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml +++ b/L1/dockerfiles/csst-msc-l1-sls1d/test/app.yaml @@ -38,7 +38,8 @@ jobs: # - h0:1 # - c0:1 # - c1:1 - - c2:1 + # - c2:1 + - n0:1 csst-msc-l1-sls1d-exe: label: 无缝光谱1D执行模块 base_image: cnic/csst-msc-l1-sls1d-exe:latest @@ -65,6 +66,13 @@ jobs: # python /pipeline/src/run.py {\"brick_id\":1} hosts: # - h0:1 - # - c0:1 - # - c1:1 - - c2:2 \ No newline at end of file + - c0:10 + - c1:10 + - c2:10 + - n0:5 + - n1:15 + - w2:15 + - w3:15 + - w4:15 + - w5:15 + - w6:15 \ No newline at end of file diff --git a/L1/dockerfiles/csst-msc-l1-sls2d/Makefile b/L1/dockerfiles/csst-msc-l1-sls2d/Makefile index 6538ad5be224455fe93ae9f970a7ea3ded14e3ea..71b9a04747f093e56a45a49f4898aeaad8a5fe20 100644 --- a/L1/dockerfiles/csst-msc-l1-sls2d/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls2d/Makefile @@ -1,14 +1,22 @@ IMAGE_NAME:=cnic/csst-msc-l1-sls2d - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls2d.tar - ssh c0 docker load -i /share/tmp/msc-l1-sls2d.tar - ssh c1 docker load -i /share/tmp/msc-l1-sls2d.tar - ssh c2 docker load -i /share/tmp/msc-l1-sls2d.tar + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-sls2d.tar + push: docker push $(IMAGE_NAME) run: @@ -18,4 +26,4 @@ down: scp: scp -r ./ csst-zjs:/root/csst/csst-msc-l1-sls2d/ save: - docker save $(IMAGE_NAME) > /sharewcl/temp/msc-l1-sls2d.tar \ No newline at end of file + docker save $(IMAGE_NAME) > /sharewcl/temp/msc-l1-sls2d.tar diff --git a/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile b/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile index 65fefa37a0f1d236c95b2d9d95e97c34f6b1969b..0e709ce8a1ebf4ba2de48f361fe44c1d1624a36a 100644 --- a/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile +++ b/L1/dockerfiles/csst-msc-l1-sls2d/test/Makefile @@ -3,7 +3,7 @@ CLUSTER=csst # all: reset build run: - scalebox app create --env-file zjs-ztm.env + scalebox app create --env-file zjs.env reset: cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - diff --git a/L1/dockerfiles/redis-cli/Dockerfile b/L1/dockerfiles/redis-cli/Dockerfile index 07c087efd08c5c86c1d0c1195017090e1ae462e9..1e668f263f80bfadcac58a1df704a715e6d23cc5 100644 --- a/L1/dockerfiles/redis-cli/Dockerfile +++ b/L1/dockerfiles/redis-cli/Dockerfile @@ -1,4 +1,4 @@ -FROM hub.cstcloud.cn/scalebox/agent:0.9 +FROM hub.cstcloud.cn/scalebox/agent # 安装redis-cli RUN apt-get update \ diff --git a/L1/pipelines/app.yaml b/L1/pipelines/app.yaml index dfa30029cc337070d220f402b6a8fb7bbb119bd9..fc342029b38a0deb18dd2fd23689670722bf1f8b 100644 --- a/L1/pipelines/app.yaml +++ b/L1/pipelines/app.yaml @@ -1,5 +1,5 @@ -name: ${PIPELINE_NAME}.app.process -label: csst-${PIPELINE_NAME} +name: csst-L1.app.process +label: csst-L1 #output_basedir: /data/${PIPELINE_NAME} cluster: csst parameters: @@ -20,13 +20,15 @@ jobs: label: 消息队列接收模块 base_image: csst/redis-cli schedule_mode: HEAD + environments: + - REDIS_SERVER=192.169.23.2 # parameters: # start_message: abcd sink_jobs: - - admL1 - admL1: + - adm-L1 + adm-L1: label: 消息分发网关 - base_image: csst/adml1 + base_image: csst/adm-l1 schedule_mode: HEAD variables: repeated: yes @@ -37,6 +39,9 @@ jobs: - cpic-l1 - hstdm-l1 - ifs-l1-rss + - mci-l1 + environments: + - CSST_PIPELINE_GRPC_SERVER="192.169.23.2:50051" msc-l1-mbi: label: 多色成像 base_image: cnic/csst-msc-l1-mbi @@ -67,10 +72,11 @@ jobs: sink_jobs: - msc-l1-phot hosts: - - h0:1 - # - c0:1 - # - c1:1 - # - c2:1 +# - h0:1 + - c0:1 + - c[1-2]:2 + - w[2-6]:1 + - n1:2 msc-l1-phot: label: 测光 base_image: cnic/csst-msc-l1-phot @@ -100,17 +106,16 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% # sink_jobs: hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:1 + - w[2-6]:20 + - n1:5 +# - c[12]:12 msc-l1-sls2d: label: 无缝光谱2D base_image: cnic/csst-msc-l1-sls2d - schedule_mode: HEAD + # schedule_mode: HEAD variables: - always_running: yes - reserved_on_exit: yes + # always_running: yes + # reserved_on_exit: yes # DFS入库错误重试3次 retry_rules: "['202:3']" output_text_size: 100000 @@ -132,14 +137,13 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% # sink_jobs: hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:1 + - c0:1 + - w[2-6]:1 +# - c[0-2]:1 cpic-l1: label: 星冕仪 base_image: cnic/csst-cpic-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes # reserved_on_exit: yes @@ -167,8 +171,9 @@ jobs: hosts: # - h0:1 # - c0:1 - - c1:1 + # - c1:1 # - c2:1 + - n0:1 hstdm-l1: label: 太赫兹 base_image: cnic/csst-hstdm-l1 @@ -196,12 +201,45 @@ jobs: - ${CSST_DFS_ROOT}:/dfsroot - ${CSST_CRDS_ROOT}:/crdsref # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% - # sink_jobs: + sink_jobs: + - hstdm-l2 hosts: # - h0:1 # - c0:1 - - c1:1 + # - c1:1 + # - c2:1 + - n0:1 + hstdm-l2: + label: 太赫兹L2 + base_image: cnic/csst-hstdm-l2 + # schedule_mode: HEAD + variables: + # always_running: yes + # reserved_on_exit: yes + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + # start_message: 50100000001 + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + environments: + - CSST_DFS_API_MODE=${CSST_DFS_API_MODE} + - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY} + - CSST_DFS_APP_ID=${CSST_DFS_APP_ID} + - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN} + - CRDS_SERVER_URL=${CRDS_SERVER_URL} + paths: + - ${CSST_AUX_DIR}:/pipeline/aux + - ${CSST_DFS_ROOT}:/dfsroot + - ${CSST_CRDS_ROOT}:/crdsref + # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% + hosts: + # - h0:1 + # - c0:1 + # - c1:1 # - c2:1 + - n0:1 ifs-l1-rss: label: IFS单次曝光 base_image: cnic/csst-ifs-l1-rss @@ -221,10 +259,43 @@ jobs: - ${AUX_DIR}:/L1Pipeline/aux - ${CRDS_DIR}:/L1Pipeline/crds - ${DFS_ROOT}:/L1Pipeline/dfs + - /sharewcl/shao/docker/csst_ifs_l1_rss/code:/L1Pipeline/workspace # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} # sink_jobs: hosts: # - h0:1 - - c0:1 + # - c0:1 + # - c1:2 + # - c2:2 + - n0:1 + mci-l1: + label: 多通道成像仪定标 + base_image: cnic/csst-mci-l1 + # schedule_mode: HEAD + variables: + # DFS入库错误重试3次 + retry_rules: "['202:3']" + output_text_size: 100000 + text_tranc_mode: TAIL + locale_mode: NONE + parameters: + key_group_regex: ^(.{6})(.{3})$ + key_group_seq: 1,2 + # start_message: 20100000001 + paths: + - ${AUX_DIR}:/L1Pipeline/aux + - ${CRDS_DIR}:/L1Pipeline/crds + - ${DFS_ROOT}:/L1Pipeline/dfs + - /sharewcl/shao/docker/csst_mci_l1/code:/L1Pipeline/workspace + # paths: + # - ${AUX_DIR}:/L1Pipeline/aux + # - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds + # - ${DFS_ROOT}:/dfsroot + # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} + # sink_jobs: + hosts: + # - h0:1 + # - c0:2 # - c1:2 - # - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 diff --git a/L1/pipelines/zjs-ztm.env b/L1/pipelines/zjs-ztm.env new file mode 100644 index 0000000000000000000000000000000000000000..72ea99c7df5550f7b55bad5113d40ff85b547faf --- /dev/null +++ b/L1/pipelines/zjs-ztm.env @@ -0,0 +1,17 @@ +# DFS +CSST_DFS_API_MODE=cluster +CSST_DFS_GATEWAY=172.24.27.2:30880 +CSST_DFS_APP_ID=test +CSST_DFS_APP_TOKEN=test +# CRDS +CRDS_SERVER_URL=http://172.24.27.2:29000 + +# VOLUMES ================== +CSST_DFS_ROOT=/share/dfs +CSST_CRDS_ROOT=/share/crdsdata/data +CSST_AUX_DIR=/share/pipeline/aux + +# IFS CHENWEI +AUX_DIR=/share/shao/L1Pipeline/aux +CRDS_DIR=/share/shao/L1Pipeline/crds +DFS_ROOT=/share/dfs \ No newline at end of file diff --git a/L1/pipelines/zjs.env b/L1/pipelines/zjs.env index ba290c3aa1acf6e18e9933ee41284709cd9c930f..cbb1b222152a0c4765ff3d30eec3d5ec26b22371 100644 --- a/L1/pipelines/zjs.env +++ b/L1/pipelines/zjs.env @@ -12,9 +12,9 @@ CSST_CRDS_ROOT=/sharewcl/crdsdata/data CSST_AUX_DIR=/sharewcl/pipeline/aux # IFS CHENWEI -AUX_DIR=/share/shao/L1Pipeline/aux -CRDS_DIR=/share/shao/L1Pipeline/crds -DFS_ROOT=/share/dfs +AUX_DIR=/sharewcl/shao/L1Pipeline/aux +CRDS_DIR=/sharewcl/shao/L1Pipeline/crds +DFS_ROOT=/sharewcl/dfs PIPELINE_NAME=csst_lever1 #PIPELINE_NAME=mbi diff --git a/L1/redis_server/README.md b/L1/redis_server/README.md index d289a88984b85ea6923f5a20d08f55efbf8c1853..52713f87da0f2b6b0933246feb364713e2c51743 100644 --- a/L1/redis_server/README.md +++ b/L1/redis_server/README.md @@ -1,11 +1,60 @@ # 模拟DFS发送redis消息 ```bash -for line in {10160000000..10160000001}; do +# mbi sls2d +for line in {10160000000..10160000136}; do echo $line; - redis-cli -h 192.169.23.2 -a 123456 lpush single-image-reduction:obsid $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done +#mci +for line in {20100000001..20100000005}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#ifs-rss +for line in {30100000003..30100000050}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#cpic +for line in {41000000101..41000000104}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; +done + +#hstdm +for line in {50100000001..50100000002}; do + echo $line; + redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid $line; done ``` redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 10160000000; +redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 20100000001; redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 30100000003; redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 41000000101; -redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 50100000001; \ No newline at end of file +redis-cli -h 127.0.0.1 -a 123456 lpush single-image-reduction:obsid 50100000001; + + +## 数据库导入导出 +## 进入database容器 +### 数据导出 +pg_dump -h 127.0.0.1 -p 5432 -U scalebox -d scalebox -Fc -f ~/scalebox.sql +### 数据导入 +pg_restore -h 127.0.0.1 -U scalebox -d scalebox /scalebox.sql + + + +## 导数据步骤 +### 1紫金山考出数据 +docker exec cf32ca2842e8 pg_dump -U scalebox --data-only --dbname=scalebox > scalebox.sql + +### 2传入到要拷贝数据的数据库容器内 +docker cp scalebox.sql a3e398828977:/tmp/scalebox.sql + +### 3进入容器并登陆数据库 +docker exec -it a3e398828977 bash +psql -U scalebox -d scalebox + +### 4执行插入命令 +\i /tmp/scalebox.sql \ No newline at end of file diff --git a/L2/dockerfiles/csst-hstdm-l2/Makefile b/L2/dockerfiles/csst-hstdm-l2/Makefile index 570bb1bd0bd3ba81f849f5211adc5f25bbf50b7e..c44d73fa1b7b020dcdca5a79f69933ead594f469 100644 --- a/L2/dockerfiles/csst-hstdm-l2/Makefile +++ b/L2/dockerfiles/csst-hstdm-l2/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-hstdm-l2 - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/hstdm-l2.tar - ssh c0 docker load -i /share/tmp/hstdm-l2.tar - ssh c1 docker load -i /share/tmp/hstdm-l2.tar - ssh c2 docker load -i /share/tmp/hstdm-l2.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l2.tar + ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l2.tar + ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l2.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-hstdm-l2/run.sh b/L2/dockerfiles/csst-hstdm-l2/run.sh index f73949105faaeb7dfc5efc99a1d408c0626fd827..02ec2c7bef1b9b1edacf474c032913e402d91498 100644 --- a/L2/dockerfiles/csst-hstdm-l2/run.sh +++ b/L2/dockerfiles/csst-hstdm-l2/run.sh @@ -13,6 +13,6 @@ fi rm -rf /pipeline/input/* /pipeline/output/* -echo finish hstdm. +echo finish hstdm-l2. exit $exit_code diff --git a/L2/dockerfiles/csst-hstdm-l2/test/app.yaml b/L2/dockerfiles/csst-hstdm-l2/test/app.yaml index 07e46c4b9a8b1be6ca66c82a64384a42f9c18726..68ef75440abe0a95f416e844b01eb7b8141829b7 100644 --- a/L2/dockerfiles/csst-hstdm-l2/test/app.yaml +++ b/L2/dockerfiles/csst-hstdm-l2/test/app.yaml @@ -1,7 +1,7 @@ name: hstdm.app.process label: 太赫兹 comment: 精测模块太赫兹1+2级流水线 -cluster: local +cluster: csst parameters: initial_status: RUNNING @@ -9,7 +9,7 @@ jobs: hstdm-l1: label: 太赫兹 base_image: cnic/csst-hstdm-l1 - schedule_mode: HEAD + # schedule_mode: HEAD variables: # always_running: yes # reserved_on_exit: yes @@ -33,18 +33,19 @@ jobs: # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% sink_jobs: - hstdm-l2 - # hosts: - # - h0:1 - # - c0:1 + hosts: + # - h0:1 + # - c0:1 # - c1:1 # - c2:1 + - n0:1 hstdm-l2: label: 太赫兹L2 base_image: cnic/csst-hstdm-l2 - schedule_mode: HEAD + # schedule_mode: HEAD variables: - always_running: yes - reserved_on_exit: yes + # always_running: yes + # reserved_on_exit: yes output_text_size: 100000 text_tranc_mode: TAIL locale_mode: NONE @@ -63,10 +64,11 @@ jobs: - ${CSST_DFS_ROOT}:/dfsroot - ${CSST_CRDS_ROOT}:/crdsref # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% - # hosts: + hosts: # - h0:1 # - c0:1 # - c1:1 - # - c2:1 + # - c2:1 + - n0:1 diff --git a/L2/dockerfiles/csst-msc-l2-xcat/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/Makefile deleted file mode 100644 index 7508326f4e607a5f7021883ae2c4f8f66241402e..0000000000000000000000000000000000000000 --- a/L2/dockerfiles/csst-msc-l2-xcat/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -IMAGE_ADM:=cnic/csst-msc-l1-mbi - -build: - make -C adm build - make -C exe build - -sync: - rsync -av --del . csst-zjs:/tmp/csst-msc-l2-xcat - -dist: - docker save hub.cstcloud.cn/scalebox/base:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - # docker save hub.cstcloud.cn/scalebox/controld:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - # docker save hub.cstcloud.cn/scalebox/database:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load' - -push: - docker push $(IMAGE_NAME) -run: - docker run -it --entrypoint bash $(IMAGE_NAME) -down: - docker stop $(IMAGE_NAME) diff --git a/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile b/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile index 2031bd2aaedfc37b11676df94a8cd027435a71b9..6420be846d4c978ed84fdecf1bbee7e4b1da900c 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile +++ b/L2/dockerfiles/csst-msc-l2-xcat/adm/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc # RUN mkdir -p /work /app/bin && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile index 2e5cff1a97be85c47d23c5a7f1362c8299fefd50..be6869f96f3c815b0705ca7d199cdf5e60a87dcd 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile +++ b/L2/dockerfiles/csst-msc-l2-xcat/adm/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l2-xcat-adm - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-adm.tar - ssh c0 docker load -i /share/tmp/msc-l2-xcat-adm.tar - ssh c1 docker load -i /share/tmp/msc-l2-xcat-adm.tar - ssh c2 docker load -i /share/tmp/msc-l2-xcat-adm.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-adm.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-msc-l2-xcat/app.yaml b/L2/dockerfiles/csst-msc-l2-xcat/app.yaml deleted file mode 100644 index cae73f832659380b8eabc570dac650b340ad6539..0000000000000000000000000000000000000000 --- a/L2/dockerfiles/csst-msc-l2-xcat/app.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: msc-l2-xcat.app.process -label: 星表检索 -comment: 星表检索 -cluster: csst -parameters: - initial_status: RUNNING - -jobs: - # msc-l1-mbi: - # label: 星表管理模块 - # base_image: cnic/csst-msc-l1-mbi - # schedule_mode: HEAD - # variables: - # always_runing: yes - # output_text_size: 100000 - # text_tranc_mode: TAIL - # locale_mode: NONE - # paths: - # - ${AUX_DIR}:/pipeline/aux - # - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds - # - ${DFS_ROOT}:/dfsroot - - csst-msc-l2-xcat-adm: - label: 星表管理模块 - base_image: cnic/csst-msc-l2-xcat-adm:latest - schedule_mode: HEAD - variables: - #always_running: yes - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - parameters: - start_message: 202306212339 - environments: - - CSST_DFS_API_MODE=cluster - - CSST_DFS_GATEWAY=172.24.27.2:30880 - - CSST_DFS_APP_ID=test - - CSST_DFS_APP_TOKEN=test - paths: # change /share to /sharewcl in PML production environment - - /sharewcl/pipeline/output-csst-msc-l2-xcat-adm:/pipeline/output # output directory, omit this line in production - - /sharewcl/pipeline/aux:/pipeline/aux - - /sharewcl/dfs:/dfsroot - - /sharewcl/crdsdata/data/references:/crdsref - sink_jobs: - - csst-msc-l2-xcat-exe - # command: > - # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - hosts: - # - h0:1 - # - c0:1 - # - c1:1 - # - c2:1 - csst-msc-l2-xcat-exe: - base_image: cnic/csst-msc-l2-xcat-exe:latest - #schedule_mode: HEAD - variables: - always_running: yes - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - environments: - - CSST_DFS_API_MODE=cluster - - CSST_DFS_GATEWAY=172.24.27.2:30880 - - CSST_DFS_APP_ID=test - - CSST_DFS_APP_TOKEN=test - paths: # change /share to /sharewcl in PML production environment - # output directory, omit this line in production - - /sharewcl/pipeline/output-csst-msc-l2-xcat-exe:/pipeline/output - - /sharewcl/pipeline/aux:/pipeline/aux - - /sharewcl/dfs:/dfsroot - - /sharewcl/crdsdata/data/references:/crdsref - # command: > - # python /pipeline/src/run.py {\"brick_id\":1} - hosts: - # - h0:1 - # - c0:1 - # - c1:1 - - c2:2 \ No newline at end of file diff --git a/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile b/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile index 74e3e25fca38d49f59fff298d1a4a94df0889316..1793ca9dacc358c5c46c493051f767a04a198f8e 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile +++ b/L2/dockerfiles/csst-msc-l2-xcat/exe/Dockerfile @@ -4,7 +4,7 @@ USER root COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin/ /usr/local/sbin/ +COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/ RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc WORKDIR /work diff --git a/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile b/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile index ad4976beae0dd42c8c13ba8be989d17cee920759..cf70426306983d3ad59d0dbf28548ec7eaf9632b 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile +++ b/L2/dockerfiles/csst-msc-l2-xcat/exe/Makefile @@ -1,15 +1,16 @@ IMAGE_NAME:=cnic/csst-msc-l2-xcat-exe - +IMAGE_PATH:=/sharewcl/temp build: docker build --network=host -t $(IMAGE_NAME) . -dist: +dist: build #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-exe.tar - ssh c0 docker load -i /share/tmp/msc-l2-xcat-exe.tar - ssh c1 docker load -i /share/tmp/msc-l2-xcat-exe.tar - ssh c2 docker load -i /share/tmp/msc-l2-xcat-exe.tar - + docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh c2 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh n0 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar + ssh n1 docker load -i $(IMAGE_PATH)/msc-l2-xcat-exe.tar push: docker push $(IMAGE_NAME) run: diff --git a/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml b/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml index fe71971c07c6dd6f011b57f58d91956e1631fcac..8ea576ea0d58df321975ea6e1f5e31ae9486b5b4 100644 --- a/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml +++ b/L2/dockerfiles/csst-msc-l2-xcat/test/app.yaml @@ -9,7 +9,7 @@ jobs: csst-msc-l2-xcat-adm: label: 星表合并管理模块 base_image: cnic/csst-msc-l2-xcat-adm:latest - schedule_mode: HEAD + #schedule_mode: HEAD variables: # always_running: yes output_text_size: 100000 @@ -30,7 +30,12 @@ jobs: - csst-msc-l2-xcat-exe # command: > # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" - + hosts: + # - h0:1 + # - c0:1 + # - c1:1 + # - c2:2 + - n0:1 csst-msc-l2-xcat-exe: label: 星表合并执行模块 base_image: cnic/csst-msc-l2-xcat-exe:latest @@ -58,4 +63,5 @@ jobs: # - h0:1 # - c0:1 # - c1:1 - - c2:2 \ No newline at end of file + # - c2:2 + - n0:1 \ No newline at end of file