completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot...

completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot ,sls2d, cpic, hstdm, ifs-rss.

L1/pipelines/Makefile

-CLUSTER=local
+CLUSTER=csst
 
 # all: reset build
 

L1/pipelines/app.yaml

 name: ${PIPELINE_NAME}.app.process
 label: csst-${PIPELINE_NAME}
 #output_basedir: /data/${PIPELINE_NAME}
-cluster: local
+cluster: csst
 parameters:
   initial_status: RUNNING
   
 jobs:
+  cron:
+    label: 主触发模块
+    base_image: hub.cstcloud.cn/scalebox/cron
+    schedule_mode: HEAD
+    parameters:
+      start_message: ANY
+    paths:
+      - ${PWD}/cron.txt:/cron.txt:ro
+    sink_jobs: 
+     - redis-cli
   redis-cli:
     label: 消息队列接收模块
     base_image: csst/redis-cli
     schedule_mode: HEAD
-    parameters:
-      start_message: abcd
+    # parameters:
+    #   start_message: abcd
     sink_jobs:
       - admL1 
   admL1:
-    label: 1级流水线消息网关
+    label: 消息分发网关
     base_image: csst/adml1
     schedule_mode: HEAD
     variables:
       repeated: yes
-    command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    #command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
     sink_jobs:
-      - mbi
-      - sls2d
-  mbi:
+      - msc-l1-mbi
+      - msc-l1-sls2d
+      - cpic-l1
+      - hstdm-l1
+      - ifs-l1-rss
+  msc-l1-mbi:
     label: 多色成像
-    base_image: csst/mbi
-    #schedule_mode: HEAD
+    base_image: cnic/csst-msc-l1-mbi
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"      
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
     parameters:
+      # start_message: 10160000068
       key_group_regex: ^(.{6})(.{3})$
       key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}      
     paths: 
-      - ${AUX_DIR}:/L1Pipeline/aux
-      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-      - ${DFS_ROOT}:/dfsroot
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
-   # sink_jobs:
+    sink_jobs:
+      - msc-l1-phot
     hosts:
       - h0:1
-     - c0:1
-     - c1:1
+      # - c0:1
+      # - c1:1
+      # - c2:1
+  msc-l1-phot:
+    label: 测光
+    base_image: cnic/csst-msc-l1-phot
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      # - c1:1
       - c2:1      
-  sls2d:
+  msc-l1-sls2d:
     label: 无缝光谱2D
-    base_image: csst/sls2d
-    #schedule_mode: HEAD
+    base_image: cnic/csst-msc-l1-sls2d
+    schedule_mode: HEAD
+    variables:
+      always_running: yes
+      reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
     parameters:
       key_group_regex: ^(.{6})(.{3})$
       key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=cluster
+      - CSST_DFS_GATEWAY=172.24.27.2:30880
+      - CSST_DFS_APP_ID=test
+      - CSST_DFS_APP_TOKEN=test      
     paths:
-      - ${AUX_DIR}:/L1Pipeline/aux
-      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-      - ${DFS_ROOT}:/dfsroot
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
     # sink_jobs:
     hosts:
-     - h0:1
-     - c0:1
-     - c1:1
+      # - h0:1
+      # - c0:1
+      # - c1:1
       - c2:1
+  cpic-l1:
+    label: 星冕仪
+    base_image: cnic/csst-cpic-l1
+    schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 41000000101
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsref
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      - c1:1
+      # - c2:1
+  hstdm-l1:
+    label: 太赫兹
+    base_image: cnic/csst-hstdm-l1
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 50100000001
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsref
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      - c1:1
+      # - c2:1
+  ifs-l1-rss:
+    label: IFS单次曝光
+    base_image: cnic/csst-ifs-l1-rss
+    # schedule_mode: HEAD
+    variables:
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
+    parameters:
+      # start_message: scalebox
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    paths: 
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/crds
+      - ${DFS_ROOT}:/L1Pipeline/dfs
+    # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      - c0:1
+      # - c1:2
+      # - c2:2     
\ No newline at end of file

L1/pipelines/cron.txt

+# comments
+@every 1m,redis-cli
+# @every 1m30s,msc-l1-sls1d-adm
\ No newline at end of file

L1/pipelines/zjs.env

-AUX_DIR=/sharewcl/L1Pipeline/aux
-CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
-DFS_ROOT=/sharewcl/dfs
+# DFS
+CSST_DFS_API_MODE=cluster
+CSST_DFS_GATEWAY=172.24.27.2:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+# CRDS
+CRDS_SERVER_URL=http://172.24.27.2:29000
 
+# VOLUMES ==================
+CSST_DFS_ROOT=/sharewcl/dfs
+CSST_CRDS_ROOT=/sharewcl/crdsdata/data
+CSST_AUX_DIR=/sharewcl/pipeline/aux
+
+# IFS CHENWEI
+AUX_DIR=/share/shao/L1Pipeline/aux
+CRDS_DIR=/share/shao/L1Pipeline/crds
+DFS_ROOT=/share/dfs 
 
 PIPELINE_NAME=csst_lever1
 #PIPELINE_NAME=mbi