测试新格式msg-all.txt消息处理，先解析流水线名称提取消息，再进行处理

L1/pipelines/pipeline-nao/Makefile

+CLUSTER=csst-nao
+
+all: reset build
+
+run:
+# PGHOST=192.168.25.27 GRPC_SERVER=192.168.25.27 scalebox app create --env-file csu.env
+	PGPORT=9090 scalebox app create --env-file nao.env
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/pipelines/pipeline-nao/add_mbi.sh

+#!/bin/bash
+
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+# obsid="test.txt"
+obsid="/home/scalebox/csst/csst-msc-l1-mbi.txt"
+while IFS= read -r line
+do
+  msg="$line"
+  echo $msg
+  docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
+done < $obsid
+~

L1/pipelines/pipeline-nao/add_sls.sh

+#!/bin/bash
+
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+# obsid="test.txt"
+obsid="/home/scalebox/csst/csst-msc-l1-sls.txt"
+while IFS= read -r line
+do
+  msg="$line"
+  echo $msg
+  docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
+done < $obsid
+~

L1/pipelines/pipeline-nao/add_task.sh

+#!/bin/bash
+
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+# obsid="test.txt"
+obsid="/nfsdata/share/simulation-collection/msc/v092/raw/msg-all.txt"
+while IFS= read -r line
+do
+  m="$line"
+  arr=($(echo $m | tr " " " "))
+  obsid="${arr[0]}"
+  detector="${arr[1]}"
+  msg=$obsid-$detector
+  echo $msg
+  docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
+done < $obsid
+~

L1/pipelines/pipeline-nao/app-l1.yaml

+name: csst-msc-l1.app.process
+label: 主巡天一级流水线
+comment: 主巡天一级流水线
+cluster: csst-nao
+parameters:
+  initial_status: RUNNING
+  
+jobs:
+  msc-l1-qc0:
+    label: QC0
+    base_image: cnic/csst-msc-l1-qc0
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # max_sleep_count: 60000
+      # grpc_server: 172.24.23.6:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    sink_jobs:
+      - adm-L1   
+    sink_vjobs:
+      - adm-L1  
+    hosts:
+      - h0:1   
+  adm-L1:
+    label: 消息分发网关
+    base_image: csst/adm-l1
+    # schedule_mode: HEAD
+    variables:
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # max_sleep_count: 60000
+      # grpc_server: 10.3.10.28:50051    
+      # repeated: yes
+    parameters:
+      slot_on_head: yes
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2     
+    #command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
+    sink_jobs:
+      - msc-l1-mbi
+      - msc-l1-sls
+    sink_vjobs:
+      - msc-l1-mbi
+      - msc-l1-sls
+    environments:
+      - CSST_PIPELINE_GRPC_SERVER="10.3.10.28:50051"
+    hosts:
+      - h0:1      
+  msc-l1-mbi:
+    label: 多色成像
+    base_image: cnic/csst-msc-l1-mbi
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 10.3.10.28:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    sink_jobs:
+      - msc-l1-mbi-photmix 
+    sink_vjobs:
+      - msc-l1-mbi-photmix          
+    hosts:
+      # - h0:1
+      - c0:10
+      # - c1:1
+      # - c2:1   
+  msc-l1-sls:
+    label: 无缝光谱
+    base_image: cnic/csst-msc-l1-sls
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      # retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
+    parameters:
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    sink_jobs:
+      - msc-l1-sls-extraction-zero
+    sink_vjobs:
+      - msc-l1-sls-extraction-zero      
+    hosts:
+    #  - h0:1
+     - c0:10
+    #  - c1:3
+    #  - c2:3
+  msc-l1-mbi-photmix:
+    label: 多色成像测光
+    base_image: cnic/csst-msc-l1-mbi-photmix
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    hosts:
+      - h0:5      
+  msc-l1-sls-extraction-zero:
+    label: 无缝光谱抽谱
+    base_image: cnic/csst-msc-l1-sls-extraction-zero
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051      
+    parameters:
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    hosts:
+      - h0:5
\ No newline at end of file

L1/pipelines/pipeline-nao/app.yaml

+name: csst-msc-l1.app.process
+label: 主巡天一级流水线(mbi+sls)
+comment: 主巡天一级流水线
+cluster: csst-nao
+parameters:
+  initial_status: RUNNING
+  
+jobs:    
+  csst-msc-l1-mbi:
+    label: 多色成像
+    base_image: cnic/csst-msc-l1-mbi
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 10.3.10.28:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    #   - msc-l1-mbi-photmix 
+    # sink_vjobs:
+    #   - msc-l1-mbi-photmix          
+    hosts:
+      - h0:10
+      # - c0:10
+      # - c1:1
+      # - c2:1   
+  csst-msc-l1-sls:
+    label: 无缝光谱
+    base_image: cnic/csst-msc-l1-sls
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      # retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
+    parameters:
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # sink_jobs:
+    #   - msc-l1-sls-extraction-zero
+    # sink_vjobs:
+    #   - msc-l1-sls-extraction-zero      
+    hosts:
+    #  - h0:1
+     - c0:20
+    #  - c1:3
+    #  - c2:3
+ 
\ No newline at end of file

L1/pipelines/pipeline-nao/nao.env

+# DFS
+CSST_DFS_GATEWAY=10.3.10.28:30880
+CSST_DFS_API_MODE=cluster
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+CSST_DFS_LOGS_DIR="."
+
+# CCDS
+CCDS_SERVER_URL=http://10.3.10.28:29000
+CCDS_USER=USER
+CCDS_PASS=PASS
+
+# VOLUMES
+CSST_DFS_ROOT=/nfsdata/share/dfs/dfs_root                   # /dfs_root:ro
+CCDS_ROOT=/nfsdata/share/dfs/ccds_root                      # /ccds_root:ro
+CSST_AUX_ROOT=/nfsdata/share/pipeline-inttest/aux           # /pipeline/aux:ro
+CSST_AST_TEMP=/nfsdata/share/pipeline-inttest/ast_temp      # /pipeline/temp:rw
+