中期版本归档

L1/dockerfiles/adm-L1/Dockerfile

-FROM hub.cstcloud.cn/scalebox/agent:0.9
+FROM hub.cstcloud.cn/scalebox/agent
 
 LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
 

L1/dockerfiles/csst-cpic-l1/Makefile

 IMAGE_NAME:=cnic/csst-cpic-l1
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/cpic-l1.tar
-	ssh c0 docker load -i /share/tmp/cpic-l1.tar
-	ssh c1 docker load -i /share/tmp/cpic-l1.tar
-	ssh c2 docker load -i /share/tmp/cpic-l1.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/cpic-l1.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/cpic-l1.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/cpic-l1.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/cpic-l1.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/cpic-l1.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/cpic-l1.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-hstdm-l1/Makefile

 IMAGE_NAME:=cnic/csst-hstdm-l1
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/hstdm-l1.tar
-	ssh c0 docker load -i /share/tmp/hstdm-l1.tar
-	ssh c1 docker load -i /share/tmp/hstdm-l1.tar
-	ssh c2 docker load -i /share/tmp/hstdm-l1.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l1.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-hstdm-l1/run.sh

 #!/bin/bash
 
+rm -rf /pipeline/input/* /pipeline/output/*
 # obsid 50100000001 -  50100000002
 obsid=$1
 python /pipeline/src/run.py $obsid
@@ -11,7 +12,10 @@ if test -f "$timefile"; then
     mv /pipeline/output/timestamps.txt /work/timestamps.txt
 fi
 
-rm -rf /pipeline/input/* /pipeline/output/*
+if [ $exit_code -eq 0 ]; then
+    echo "send message $obsid to hstdm-l2."
+    send-message $obsid
+fi
 
 echo finish hstdm.
 

L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile

 IMAGE_NAME:=cnic/csst-ifs-l1-cube-adm
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar
-	ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar
-	ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar
-	ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-adm.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-adm.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh

 #!/bin/bash
 
-# pipeline=/pipeline/output/pipeline.log
-# if test -f "$pipeline"; then
-#     echo "=====pipeline.log====" > /work/user-file.txt
-#     cat /pipeline/output/pipeline.log >> /work/user-file.txt
-# fi
-
-# timefile=/pipeline/output/timestamps.txt
-# if test -f "$timefile"; then
-#     echo "$timefile exist"
-#     mv /pipeline/output/timestamps.txt /work/timestamps.txt
-# fi
-
-# obsid="30100000001"
-
-python l1_pipeline_script_ifs_cube_adm.py > ./stdout
+python l1_pipeline_script_ifs_cube_adm.py
 
 exit_code=$?
 
 mkdir -p /work
-# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
-# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
-
-# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
-
-# mv /tmp/timestamps.txt /work/timestamps.txt
-
-cat ./stdout
 
 mv /L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt
+sed -i "s/^/${SINK_JOB},/g" /work/messages.txt
+
 echo finish csst-ifs-l1-cube-adm.
 
 exit $exit_code

L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile

 IMAGE_NAME:=cnic/csst-ifs-l1-cube-exe
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar
-	ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar
-	ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar
-	ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube-exe.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-cube-exe.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh

@@ -2,17 +2,11 @@
 
 echo input:$1*
 
-python l1_pipeline_script_ifs_cube_simdata.py $1 > ./stdout
+python l1_pipeline_script_ifs_cube_simdata.py $1
 
 exit_code=$?
 
 mkdir -p /work
-# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
-# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
-
-# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
-
-# mv /tmp/timestamps.txt /work/timestamps.txt
 
 echo finish csst-ifs-l1-cube-exe.
 

L1/dockerfiles/csst-ifs-l1-cube/test/Makefile

-CLUSTER=local
+CLUSTER=csst
 
 all: reset build
 
 run:
-	scalebox app create --env-file zjs-ztm.env
+	scalebox app create --env-file zjs.env
 
 reset:
 	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -

L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml

 name: ifs-l1-cube.app.process
 label: 多次曝光合并
 comment: 积分场光谱仪多次曝光合并一级流水线
-cluster: local
+cluster: csst
 parameters:
   initial_status: RUNNING
   
@@ -10,7 +10,7 @@ jobs:
   csst-ifs-l1-cube-adm:
     label: 多次曝光合并管理模块
     base_image: cnic/csst-ifs-l1-cube-adm:latest
-    schedule_mode: HEAD
+    #schedule_mode: HEAD
     variables:
       # always_running: yes
       output_text_size: 100000
@@ -27,22 +27,23 @@ jobs:
       - CSST_DFS_APP_ID=test
       - CSST_DFS_APP_TOKEN=test
     paths: # change /share to /sharewcl in PML production environment
-      - ${CSST_AUX_DIR}:/pipeline/aux
-      - ${CSST_DFS_ROOT}:/dfsroot
-      - ${CSST_CRDS_ROOT}:/crdsroot
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/crds
+      - ${DFS_ROOT}:/L1Pipeline/dfs
     sink_jobs:
       - csst-ifs-l1-cube-exe
     # command: >
     #   bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt"
-    # hosts:
+    hosts:
     #  - h0:1
     #  - c0:1
     #  - c1:1
     # - c2:1
+      - n0:1
   csst-ifs-l1-cube-exe:
     label: 多次曝光合并执行模块
     base_image: cnic/csst-ifs-l1-cube-exe:latest
-    schedule_mode: HEAD
+    # schedule_mode: HEAD
     variables:
       # always_running: yes
       output_text_size: 100000
@@ -62,8 +63,9 @@ jobs:
       - ${DFS_ROOT}:/L1Pipeline/dfs
     # command: >
     #   python /pipeline/src/run.py {\"brick_id\":1}
-    # hosts:
+    hosts:
     #  - h0:1
     #  - c0:1
     #  - c1:1
-    # - c2:2
\ No newline at end of file
+    # - c2:2
+      - n0:1
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-rss/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
 USER root
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
 
 # RUN mkdir /work/
 # WORKDIR /work/

L1/dockerfiles/csst-ifs-l1-rss/Makefile

 IMAGE_NAME:=cnic/csst-ifs-l1-rss
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-rss.tar
-	ssh c0 docker load -i /share/tmp/ifs-l1-rss.tar
-	ssh c1 docker load -i /share/tmp/ifs-l1-rss.tar
-	ssh c2 docker load -i /share/tmp/ifs-l1-rss.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-rss.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/ifs-l1-rss.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-ifs-l1-rss/run.sh

@@ -6,7 +6,7 @@ obsid=$1
 # python l1_pipeline_script_mci_shao_simdata.py $obsid > ./stdout
 #python /L1Pipeline/workspace/helloworld.py $obsid
 
-python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout
+python l1_pipeline_script_ifs_rss_simdata.py $obsid
 
 exit_code=$?
 

L1/dockerfiles/csst-mci-l1/Dockerfile

@@ -4,7 +4,7 @@ LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
 
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
 
 RUN mkdir /work/
 WORKDIR /work/

L1/dockerfiles/csst-mci-l1/Makefile

 IMAGE_NAME:=cnic/csst-mci-l1
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/mci-l1.tar
-	ssh c0 docker load -i /share/tmp/mci-l1.tar
-	ssh c1 docker load -i /share/tmp/mci-l1.tar
-	ssh c2 docker load -i /share/tmp/mci-l1.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/mci-l1.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/mci-l1.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/mci-l1.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/mci-l1.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/mci-l1.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/mci-l1.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-mci-l1/test/app.yaml

@@ -6,7 +6,7 @@ parameters:
 jobs:
   mci-l1: 
     base_image: cnic/csst-mci-l1
-    schedule_mode: HEAD
+    # schedule_mode: HEAD
     variables:
       output_text_size: 100000
       text_tranc_mode: TAIL
@@ -14,7 +14,7 @@ jobs:
     parameters:
       key_group_regex: ^(.{6})(.{3})$
       key_group_seq: 1,2
-      start_message: 20100000001
+      # start_message: 20100000001
     paths: 
       - ${AUX_DIR}:/L1Pipeline/aux
       - ${CRDS_DIR}:/L1Pipeline/crds
@@ -26,8 +26,9 @@ jobs:
     #   - ${DFS_ROOT}:/dfsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
     # sink_jobs:
-    # hosts:
+    hosts:
     #   - h0:1
     #   - c0:2
     #   - c1:2
     #   - c2:2
+      - n0:1

L1/dockerfiles/csst-msc-l1-mbi/Makefile

 IMAGE_NAME:=cnic/csst-msc-l1-mbi
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/msc-l1-mbi.tar
-	ssh c0 docker load -i /share/tmp/msc-l1-mbi.tar
-	ssh c1 docker load -i /share/tmp/msc-l1-mbi.tar
-	ssh c2 docker load -i /share/tmp/msc-l1-mbi.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+	ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
 push:
 	docker push $(IMAGE_NAME)
 run:
@@ -17,4 +23,4 @@ run:
 down:
 	docker stop $(IMAGE_NAME)
 scp:
-	scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/
\ No newline at end of file
+	scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/

L1/dockerfiles/csst-msc-l1-ooc/adm/Makefile

 IMAGE_NAME:=cnic/csst-msc-l1-ooc-adm
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-adm.tar
-	ssh c0 docker load -i /share/tmp/msc-l1-ooc-adm.tar
-	ssh c1 docker load -i /share/tmp/msc-l1-ooc-adm.tar
-	ssh c2 docker load -i /share/tmp/msc-l1-ooc-adm.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-adm.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-adm.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-msc-l1-ooc/exe/Makefile

 IMAGE_NAME:=cnic/csst-msc-l1-ooc-exe
-
+IMAGE_PATH:=/sharewcl/temp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
-dist:
+dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-	docker save $(IMAGE_NAME) > /share/tmp/msc-l1-ooc-exe.tar
-	ssh c0 docker load -i /share/tmp/msc-l1-ooc-exe.tar
-	ssh c1 docker load -i /share/tmp/msc-l1-ooc-exe.tar
-	ssh c2 docker load -i /share/tmp/msc-l1-ooc-exe.tar
-
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc-exe.tar
+	ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar
+	ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar
+	ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar
+	ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar
+	ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-ooc-exe.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-msc-l1-ooc/test/Makefile

@@ -3,7 +3,7 @@ CLUSTER=local
 all: reset build
 
 run:
-	scalebox app create --env-file zjs-ztm.env
+	scalebox app create --env-file zjs.env
 
 reset:
 	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -