天测csst-msc-l1-ast-astrometry单元测试成功

L1/dockerfiles/csst-msc-l1-ast/Dockerfile

+FROM csu-harbor.csst.nao:10443/csst/csst-msc-l1-ast-astrometry
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-msc-l1-ast/Makefile

+IMAGE_NAME:=cnic/csst-msc-l1-ast-astrometry
+IMAGE_PATH:=/nfs/tmp/scalebox-images
+build:
+	docker pull csu-harbor.csst.nao:10443/csst/csst-msc-l1-ast-astrometry
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: 
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-ast.tar
+	ssh sc1 docker load -i $(IMAGE_PATH)/msc-l1-ast.tar
+	ssh sc2 docker load -i $(IMAGE_PATH)/msc-l1-ast.tar
+	ssh sc3 docker load -i $(IMAGE_PATH)/msc-l1-ast.tar
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-csu:/root/csst/csst-msc-l1-ast/

L1/dockerfiles/csst-msc-l1-ast/run.sh

+#!/bin/bash
+
+#TRACE=yes 临时办法
+rm -rf /work/*
+
+message=$1
+echo "message: "$message
+
+cd /pipeline/output/
+python /pipeline/app/run.py $message
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+
+headers=$2
+pattern='"dag_run_id":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    dag_run_id="${BASH_REMATCH[1]}"
+    echo "dag_run_id: $dag_run_id"
+else
+    # no dag_run_id in json
+    dag_run_id=""
+fi
+
+echo "dag_run_id:$dag_run_id" >> /work/extra-attributes.txt
+
+pattern='"sorted_tag":"([^"]+)"'
+if [[ $headers =~ $pattern ]]; then
+    sorted_tag="${BASH_REMATCH[1]}"
+    echo "sorted_tag: $sorted_tag"
+else
+    # no sorted_tag in json
+    sorted_tag=1
+fi
+
+if [ $exit_code -eq 0 ]; then  
+  echo "finish ast, start sink-job."  
+  scalebox task add --header sorted_tag=${sorted_tag} --header dag_run_id=${dag_run_id} --header repeatable=yes --upsert ${message}
+  # scalebox task add $1
+else  
+  echo "finish ast, exit_code: $exit_code"  
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+exit $exit_code

L1/pipelines/csst-msc-l1-ast/Makefile

+CLUSTER=csst-csu
+
+all: reset build
+
+run:
+# PGHOST=192.168.25.27 GRPC_SERVER=192.168.25.27 scalebox app create --env-file csu.env
+	scalebox app create --env-file scalebox.env
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/pipelines/csst-msc-l1-ast/app.yaml

+name: csst-msc-l1-ast.apps2
+label: 天测一级流水线(ast)
+comment: 天测一级流水线
+cluster: csst-csu
+parameters:
+  initial_status: RUNNING
+  message_router: message-router-csst
+
+jobs: 
+  message-router-csst:
+    label: 主消息路由
+    base_image: csst/message-router-ast
+   # schedule_mode: HEAD
+    parameters:
+      tasks_per_queue: 500
+      #key_group_regex: ^(.+)$
+      #start_message: any
+    arguments:
+      task_timeout_seconds: 3000
+      max_sleep_count: 6000
+      slot_options: slot_on_head
+    environments:
+      - message=tar1266932744
+      - dataset_id=${DATASET_ID}
+      - star_s=${STAR_S}
+      - end_st=${END_ST}
+      - size=${SIZE}
+      - DAG_NAME=${DAG_NAME}
+      # - PGHOST=10.3.10.28:9090
+    paths:
+      - ${CSST_DAT_YAML_PATH}:/dag-yaml
+    sink_vjobs:
+      - csst-msc-l1-ast-astrometry
+      # - csst-msc-l1-ast-sso 
+  csst-msc-l1-ast-astrometry:
+    label: 天测
+    base_image: cnic/csst-msc-l1-ast-astrometry
+    # schedule_mode: HEAD
+    arguments:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 10.3.10.28:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_TOKEN=${CSST_DFS_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+      # - PGHOST=10.3.10.28:9090
+      - TRACE=yes
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfs_root
+      - ${CCDS_ROOT}:/ccds_root
+      - ${CSST_AST_TEMP}:/pipeline/temp
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%    
+    # sink_vjobs:
+    #   - csst-msc-l1-ast-sso    
+    hosts:
+      - h0:1
+      # - c0:1
+      # - c1:1
+      # - c2:1
+  # csst-msc-l1-ast-sso:
+  #   label: 近地天体探测
+  #   base_image: cnic/csst-msc-l1-ast-sso
+  #   # schedule_mode: HEAD
+  #   arguments:
+  #     # always_running: yes
+  #     # reserved_on_exit: yes
+  #     output_text_size: 100000
+  #     text_tranc_mode: TAIL
+  #     locale_mode: NONE
+  #     # grpc_server: 172.24.23.6:50051      
+  #   parameters:
+  #     # start_message: 10160000068
+  #     key_group_regex: ^(.{6})(.{3})$
+  #     key_group_seq: 1,2
+  #   environments:
+  #     - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+  #     - CSST_DFS_TOKEN=${CSST_DFS_TOKEN}
+  #     - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+  #     # - PGHOST=10.3.10.28:9090
+  #     - TRACE=yes  
+  #   paths: 
+  #     - ${CSST_AUX_ROOT}:/pipeline/aux
+  #     - ${CSST_DFS_ROOT}:/dfs_root
+  #     - ${CCDS_ROOT}:/ccds_root
+  #     - ${CSST_AST_TEMP}:/pipeline/temp
+  #   # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+  #   hosts:
+  #     - h0:1      
+  #     # - c0:1
+  #     # - c1:1
+  #     # - c2:1      
+ 
\ No newline at end of file

L1/pipelines/csst-msc-l1-ast/scalebox.env

+# DFS
+CSST_DFS_GATEWAY=192.168.25.89:28000
+CSST_DFS_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjQ4ODU0NTA2NjQsInN1YiI6InN5c3RlbSJ9.POsuUABytu8-WMtZiYehiYEa5BnlgqNTXT6X3OTyix0
+
+# CCDS
+CCDS_SERVER_URL=http://192.168.25.89:29000
+CCDS_USER=USER
+CCDS_PASS=PASS
+
+# VOLUMES
+CSST_DFS_ROOT=/nfs/dfs                   # /dfs_root:ro
+CCDS_ROOT=/nfs/ccds                      # /ccds_root:ro
+CSST_AST_TEMP=/nfs/pipeline-inttest/ast_temp      # /pipeline/temp:rw
+CSST_AUX_ROOT=/nfs/pipeline-inttest/aux           # /pipeline/aux:ro
+
+# TEST
+CSST_INTTEST_ROOT=/nfs/pipeline-inttest
+VERBOSE=true
+
+# DAG yaml /dag-yaml:ro
+CSST_DAT_YAML_PATH=/nfs/scalebox/dag-yaml
+DAG_NAME=csst-msc-l1-ast
\ No newline at end of file