配置公有云镜像库

L1/dockerfiles/csst-ifs-l1-qc0/Makefile

+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-ifs-l1-qc0
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-ifs-l1-qc0
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: build
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/ifs-l1-qc0.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/ifs-l1-qc0.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/ifs-l1-qc0.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/ifs-l1-qc0.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-csu:/root/csst/csst-ifs-l1-qc0/
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-qc0/run.sh

+#!/bin/bash
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+
+if [ $exit_code -eq 0 ]; then  
+    # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "finish ifs-l1-qc0."  
+else  
+    echo "finish ifs-l1-qc0, exit_code: $exit_code"  
+fi
+
+exit $exit_code

L1/dockerfiles/csst-ifs-l1-rss/Makefile

-IMAGE_NAME:=cnic/csst-ifs-l1-rss
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-ifs-l1-rss
 IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-ifs-l1-rss

L1/dockerfiles/csst-mci-l1/Makefile

-IMAGE_NAME:=cnic/csst-mci-l1
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-mci-l1
 IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-mci-l1

L1/dockerfiles/csst-msc-l1-ast-sso-match/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-ast-sso-match
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-msc-l1-ast-sso-match/Makefile

+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-ast-sso-match
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-ast-sso-match
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: 
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ast-sso.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-ast-sso.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-ast-sso.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-ast-sso.tar
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+

L1/dockerfiles/csst-msc-l1-ast-sso-match/run.sh

+#!/bin/bash
+
+#TRACE=yes 临时办法
+rm -rf /work/*
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+if [ $exit_code -eq 0 ]; then  
+  echo finish ast-sso.
+  # scalebox task add $1
+else  
+  echo "finish ast-sso, exit_code: $exit_code"  
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+exit $exit_code

L1/dockerfiles/csst-msc-l1-ast/Makefile

-IMAGE_NAME:=cnic/csst-msc-l1-ast-astrometry
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-ast-astrometry
 IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-ast-astrometry

L1/dockerfiles/csst-msc-l1-mbi-photmix/Makefile

-IMAGE_NAME:=cnic/csst-msc-l1-mbi-phot
-IMAGE_PATH:=/nfs/tmp/scalebox-images
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-mbi-phot
+IMAGE_PATH:=/mnt/tmp/scalebox-images
 build:
 	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-mbi-phot
 	docker build --network=host -t $(IMAGE_NAME) .

L1/dockerfiles/csst-msc-l1-mbi/Dockerfile

-FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-mbi
+FROM zjlab-harbor.csst.nao:10443/csst/csst-msc-l1-mbi
 
 USER root
 COPY run.sh /app/bin/

L1/dockerfiles/csst-msc-l1-mbi/Makefile

-IMAGE_NAME:=cnic/csst-msc-l1-mbi
-IMAGE_PATH:=/nfs/tmp/scalebox-images
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-mbi
+IMAGE_PATH:=/mnt/tmp/scalebox-images
 build:
-	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-mbi
+	docker pull zjlab-harbor.csst.nao:10443/csst/csst-msc-l1-mbi
 	docker build --network=host -t $(IMAGE_NAME) .
+push:
+	docker push $(IMAGE_NAME)
 
 dist: 
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
@@ -10,8 +12,6 @@ dist:
 	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
 	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
 	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-push:
-	docker push $(IMAGE_NAME)
 run:
 	docker run -it --entrypoint bash $(IMAGE_NAME)
 down:

L1/dockerfiles/csst-msc-l1-ooc/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-ooc
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-msc-l1-ooc/Makefile

+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-ooc
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-ooc
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: build
+        #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ooc.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-ooc.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-ooc.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-ooc.tar
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ tencent-p0:/root/csst/csst-msc-l1-ooc/
+save:
+	docker save $(IMAGE_NAME) >  $(IMAGE_PATH)/msc-l1-ooc.tar

L1/dockerfiles/csst-msc-l1-ooc/run.sh

+#!/bin/bash
+
+#TRACE=yes 临时办法
+rm -rf /work/*
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+timefile=/pipeline/output/timestamps.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamps.txt /work/timestamps.txt
+fi
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+# pattern='"sorted_tag":"([^"]+)"'
+# if [[ $headers =~ $pattern ]]; then
+#     sorted_tag="${BASH_REMATCH[1]}"
+#     echo "sorted_tag: $sorted_tag"
+# else
+#     # no sorted_tag in json
+#     sorted_tag=1
+# fi
+
+if [ $exit_code -eq 0 ]; then  
+  echo "finish ooc."  
+#   scalebox task add --header sorted_tag=${sorted_tag} --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+else  
+  echo "finish ooc, exit_code: $exit_code"  
+fi
+
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+exit $exit_code

L1/dockerfiles/csst-msc-l1-qc0/Dockerfile

-FROM zjlab-harbor.csst.nao/csst/csst-msc-l1-qc0
+FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-qc0
 
 USER root
 COPY run.sh /app/bin/

L1/dockerfiles/csst-msc-l1-qc0/Makefile

-IMAGE_NAME:=cnic/csst-msc-l1-qc0
-IMAGE_PATH:=/data/scalebox/tmp
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-qc0
+IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-qc0
 	docker build --network=host -t $(IMAGE_NAME) .
 
 dist: 
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
 	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-qc0.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-qc0.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-qc0.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-qc0.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-msc-l1-qc0/run.sh

@@ -37,8 +37,8 @@ fi
 echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
 
 if [ $exit_code -eq 0 ]; then  
-    scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
-    echo "finish qc0, start sink-job."  
+    # scalebox task add --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+    echo "finish qc0."  
 else  
     echo "finish qc0, exit_code: $exit_code"  
 fi

L1/dockerfiles/csst-msc-l1-sls-extraction-zero/Dockerfile

-FROM 10.3.10.10:8113/csst/csst-msc-l1-sls-extraction-zero
+FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-sls-extraction-zero
 
 USER root
 COPY run.sh /app/bin/

L1/dockerfiles/csst-msc-l1-sls-extraction-zero/Makefile

-IMAGE_NAME:=cnic/csst-msc-l1-sls-extraction-zero
-IMAGE_PATH:=/nfsdata/tmp
+IMAGE_NAME:=csu-harbor.csst.nao:10443/cnic/csst-msc-l1-sls-extraction-zero
+IMAGE_PATH:=/nfs/tmp/scalebox-images
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
 dist: 
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
 	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-sls-extraction-zero.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-sls-extraction-zero.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-sls-extraction-zero.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-sls-extraction-zero.tar
 push:
 	docker push $(IMAGE_NAME)
 run:

L1/dockerfiles/csst-msc-l1-sls-extraction-zero/run.sh

 #!/bin/bash
 
-# obsid 10160000000 -  10160000136
-arr=($(echo $1 | tr "-" " "))
-obsid="${arr[0]}"
-detector="${arr[1]}"
-echo "obsid: "$obsid
-echo "detector: "$detector
+#TRACE=yes 临时办法
+rm -rf /work/*
+
+message=$1
+echo "message: "$message
 
 cd /pipeline/output/
 python /pipeline/app/run.py $obsid $detector
@@ -26,8 +25,25 @@ if test -f "$timefile"; then
     mv /pipeline/output/timestamp.txt /work/timestamps.txt
 fi
 
-rm -rf /pipeline/input/* /pipeline/output/*
 
-echo finish sls-extraction-zero.
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+if [ $exit_code -eq 0 ]; then  
+  echo finish sls-extraction-zero.
+  # scalebox task add $1
+else  
+  echo "finish sls-extraction-zero, exit_code: $exit_code"  
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
 
 exit $exit_code