配置公有云镜像库

L1/dockerfiles/csst-cpic-l1-qc0/Makefile

-IMAGE_NAME:=cnic/csst-cpic-l1-qc0
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-cpic-l1-qc0
 IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-cpic-l1-qc0
@@ -18,4 +18,4 @@ run:
 down:
 	docker stop $(IMAGE_NAME)
 scp:
-	scp -r ./ csst-csu:/root/csst/csst-cpic-l1/
\ No newline at end of file
+	scp -r ./ csst-csu:/root/csst/csst-cpic-l1-qc0/
\ No newline at end of file

L1/dockerfiles/csst-cpic-l1-qc0/run.sh

@@ -38,6 +38,7 @@ echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
 
 if [ $exit_code -eq 0 ]; then  
     # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "$1" > ${WORK_DIR}/messages.txt
     echo "finish cpic-l1-qc0."  
 else  
     echo "finish cpic-l1-qc0, exit_code: $exit_code"  

L1/dockerfiles/csst-cpic-l1/Makefile

-IMAGE_NAME:=cnic/csst-cpic-l1
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-cpic-l1
 IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-cpic-l1

L1/dockerfiles/csst-hstdm-l1/Dockerfile

-FROM csst/csst-hstdm-l1:latest
+FROM csu-harbor.csst.nao:10443/csst/csst-hstdm-l1:latest
 
 USER root
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
 
 RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
 WORKDIR /work/
 
+USER csst
+
 ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-hstdm-l1/Makefile

-IMAGE_NAME:=cnic/csst-hstdm-l1
-IMAGE_PATH:=/sharewcl/temp
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-hstdm-l1
+IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-hstdm-l1
 	docker build --network=host -t $(IMAGE_NAME) .
 
 dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
 	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l1.tar
-	ssh c0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
-	ssh c1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
-	ssh c2 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
-	ssh n0 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
-	ssh n1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/hstdm-l1.tar
+
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-hstdm-l1/run.sh

 #!/bin/bash
 
-rm -rf /pipeline/input/* /pipeline/output/*
-# obsid 50100000001 -  50100000002
-obsid=$1
-python /pipeline/src/run.py $obsid
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
 exit_code=$?
 
-timefile=/pipeline/output/timestamps.txt
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
 if test -f "$timefile"; then
     echo "$timefile exist"
-    mv /pipeline/output/timestamps.txt /work/timestamps.txt
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
 fi
 
-if [ $exit_code -eq 0 ]; then
-    echo "send message $obsid to hstdm-l2."
-    send-message $obsid
+rm -rf /pipeline/input/* /pipeline/output/*
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
 fi
 
-echo finish hstdm.
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+
+if [ $exit_code -eq 0 ]; then  
+    # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "$1" > ${WORK_DIR}/messages.txt
+    echo "finish hstdm-l1."  
+else  
+    echo "finish hstdm-l1, exit_code: $exit_code"  
+fi
 
 exit $exit_code

L1/dockerfiles/csst-hstdm-l1/test/app.yaml

 name: hstdm-l1.app.process
 label: 太赫兹
-comment: 精测模块太赫兹一级流水线
+comment: 太赫兹一级流水线
 cluster: csst
 parameters:
   initial_status: RUNNING

L1/dockerfiles/csst-hstdm-l2/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-hstdm-l2:latest
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-hstdm-l2/Makefile

+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-hstdm-l2
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-hstdm-l2
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: build
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/hstdm-l2.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/hstdm-l2.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/hstdm-l2.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/hstdm-l2.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-zjs:/root/csst/csst-hstdm-l2/
\ No newline at end of file

L1/dockerfiles/csst-hstdm-l2/run.sh

+#!/bin/bash
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+
+if [ $exit_code -eq 0 ]; then  
+    # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "$1" > ${WORK_DIR}/messages.txt
+    echo "finish hstdm-l2."  
+else  
+    echo "finish hstdm-l2, exit_code: $exit_code"  
+fi
+
+exit $exit_code

L1/dockerfiles/csst-hstdm-l2/test/Makefile

+CLUSTER=csst
+
+all: reset build
+
+run:
+	scalebox app create --env-file zjs.env
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/csst-hstdm-l2/test/add_task.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+for m in {50100000001..50100000002}; do
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+
+done

L1/dockerfiles/csst-hstdm-l2/test/app.yaml

+name: hstdm-l2.app.process
+label: 太赫兹
+comment: 太赫兹二级流水线
+cluster: csst
+parameters:
+  initial_status: RUNNING
+  
+jobs:
+  hstdm-l1:
+    label: 太赫兹L2
+    base_image: cnic/csst-hstdm-l2
+    schedule_mode: HEAD
+    variables:
+      always_running: yes
+      reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 50100000001
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsref
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    # hosts:
+    #   - h0:1
+    #   - c0:1
+      # - c1:1
+      # - c2:1
+      
+
+      

L1/dockerfiles/csst-hstdm-l2/test/tencent.env

+AUX_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux
+CRDS_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr
+DFS_ROOT=/goosefsx/x_c60_o19xp6c1_proxy/dfs_root

L1/dockerfiles/csst-hstdm-l2/test/zjs-ztm.env

+# DFS
+CSST_DFS_API_MODE=cluster
+CSST_DFS_GATEWAY=172.24.27.2:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+# CRDS
+CRDS_SERVER_URL=http://172.24.27.2:29000
+
+# VOLUMES ==================
+CSST_DFS_ROOT=/share/dfs
+CSST_CRDS_ROOT=/share/crdsdata/data
+CSST_AUX_DIR=/share/pipeline/aux

L1/dockerfiles/csst-hstdm-l2/test/zjs.env

+# DFS
+CSST_DFS_API_MODE=cluster
+CSST_DFS_GATEWAY=172.24.27.2:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+# CRDS
+CRDS_SERVER_URL=http://172.24.27.2:29000
+
+# VOLUMES ==================
+CSST_DFS_ROOT=/sharewcl/dfs
+CSST_CRDS_ROOT=/sharewcl/crdsdata/data/references
+CSST_AUX_DIR=/sharewcl/pipeline/aux

L1/dockerfiles/csst-ifs-l1-cube/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-ifs-l1-cube:latest
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-ifs-l1-cube/Makefile

+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-ifs-l1-cube
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-ifs-l1-cube
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: build
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-cube.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/ifs-l1-cube.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/ifs-l1-cube.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/ifs-l1-cube.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-csu:/root/csst/csst-ifs-l1-cube/
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-cube/run.sh

+#!/bin/bash
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+
+if [ $exit_code -eq 0 ]; then  
+    # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "finish ifs-l1-cube."  
+else  
+    echo "finish ifs-l1-cube, exit_code: $exit_code"  
+fi
+
+exit $exit_code

L1/dockerfiles/csst-ifs-l1-qc0/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-ifs-l1-qc0:latest
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]