completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot...

completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot ,sls2d, cpic, hstdm, ifs-rss.

completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot...
completed level 1 pipelines test. Include: cron, redist-cli, admL1, mbi, phot ,sls2d, cpic, hstdm, ifs-rss.
b03bbae3 · Zhang Xiaoli · 6f511e73 · b03bbae3 · b03bbae3 · b03bbae3
Commit b03bbae3 authored Jul 21, 2023 by Zhang Xiaoli
Showing with 220 additions and 36 deletions

L1/pipelines/Makefile L1/pipelines/Makefile +1 -1

L1/pipelines/app.yaml L1/pipelines/app.yaml +200 -32

L1/pipelines/cron.txt L1/pipelines/cron.txt +3 -0

L1/pipelines/zjs.env L1/pipelines/zjs.env +16 -3

No files found.
--- a/L1/pipelines/Makefile
+++ b/L1/pipelines/Makefile
-CLUSTER=local
+CLUSTER=csst
 # all: reset build

--- a/L1/pipelines/app.yaml
+++ b/L1/pipelines/app.yaml
 name: ${PIPELINE_NAME}.app.process
 label: csst-${PIPELINE_NAME}
 #output_basedir: /data/${PIPELINE_NAME}
-cluster: local
+cluster: csst
 parameters:
  initial_status: RUNNING
 jobs:
+  cron:
+    label: 主触发模块
+    base_image: hub.cstcloud.cn/scalebox/cron
+    schedule_mode: HEAD
+    parameters:
+      start_message: ANY
+    paths:
+      - ${PWD}/cron.txt:/cron.txt:ro
+    sink_jobs: 
+     - redis-cli
  redis-cli:
    label: 消息队列接收模块
    base_image: csst/redis-cli
    schedule_mode: HEAD
-    parameters:
+    # parameters:
-      start_message: abcd
+    #   start_message: abcd
    sink_jobs:
      - admL1 
  admL1:
-    label: 1级流水线消息网关
+    label: 消息分发网关
    base_image: csst/adml1
    schedule_mode: HEAD
    variables:
      repeated: yes
-    command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    #command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
    sink_jobs:
-      - mbi
+      - msc-l1-mbi
-      - sls2d
+      - msc-l1-sls2d
-  mbi:
+      - cpic-l1
+      - hstdm-l1
+      - ifs-l1-rss
+  msc-l1-mbi:
    label: 多色成像
-    base_image: csst/mbi
+    base_image: cnic/csst-msc-l1-mbi
-    #schedule_mode: HEAD
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"      
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
    parameters:
+      # start_message: 10160000068
      key_group_regex: ^(.{6})(.{3})$
      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}      
    paths: 
-      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CSST_AUX_DIR}:/pipeline/aux
-      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${CSST_DFS_ROOT}:/dfsroot
-      - ${DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
-   # sink_jobs:
+    sink_jobs:
+      - msc-l1-phot
    hosts:
      - h0:1
-     - c0:1
+      # - c0:1
-     - c1:1
+      # - c1:1
+      # - c2:1
+  msc-l1-phot:
+    label: 测光
+    base_image: cnic/csst-msc-l1-phot
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      # - c1:1
      - c2:1      
-  sls2d:
+  msc-l1-sls2d:
    label: 无缝光谱2D
-    base_image: csst/sls2d
+    base_image: cnic/csst-msc-l1-sls2d
-    #schedule_mode: HEAD
+    schedule_mode: HEAD
+    variables:
+      always_running: yes
+      reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
    parameters:
      key_group_regex: ^(.{6})(.{3})$
      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=cluster
+      - CSST_DFS_GATEWAY=172.24.27.2:30880
+      - CSST_DFS_APP_ID=test
+      - CSST_DFS_APP_TOKEN=test      
    paths:
-      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CSST_AUX_DIR}:/pipeline/aux
-      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${CSST_DFS_ROOT}:/dfsroot
-      - ${DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsroot
    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
    # sink_jobs:
    hosts:
-     - h0:1
+      # - h0:1
-     - c0:1
+      # - c0:1
-     - c1:1
+      # - c1:1
      - c2:1
+  cpic-l1:
+    label: 星冕仪
+    base_image: cnic/csst-cpic-l1
+    schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 41000000101
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsref
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      - c1:1
+      # - c2:1
+  hstdm-l1:
+    label: 太赫兹
+    base_image: cnic/csst-hstdm-l1
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: 50100000001
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_DIR}:/pipeline/aux
+      - ${CSST_DFS_ROOT}:/dfsroot
+      - ${CSST_CRDS_ROOT}:/crdsref
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      # - c0:1
+      - c1:1
+      # - c2:1
+  ifs-l1-rss:
+    label: IFS单次曝光
+    base_image: cnic/csst-ifs-l1-rss
+    # schedule_mode: HEAD
+    variables:
+      # DFS入库错误重试3次
+      retry_rules: "['202:3']"
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051
+    parameters:
+      # start_message: scalebox
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    paths: 
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/crds
+      - ${DFS_ROOT}:/L1Pipeline/dfs
+    # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
+    # sink_jobs:
+    hosts:
+      # - h0:1
+      - c0:1
+      # - c1:2
+      # - c2:2     
\ No newline at end of file
--- a/L1/pipelines/cron.txt
+++ b/L1/pipelines/cron.txt
+# comments
+@every 1m,redis-cli
+# @every 1m30s,msc-l1-sls1d-adm
\ No newline at end of file
--- a/L1/pipelines/zjs.env
+++ b/L1/pipelines/zjs.env
-AUX_DIR=/sharewcl/L1Pipeline/aux
+# DFS
-CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+CSST_DFS_API_MODE=cluster
-DFS_ROOT=/sharewcl/dfs
+CSST_DFS_GATEWAY=172.24.27.2:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+# CRDS
+CRDS_SERVER_URL=http://172.24.27.2:29000
+# VOLUMES ==================
+CSST_DFS_ROOT=/sharewcl/dfs
+CSST_CRDS_ROOT=/sharewcl/crdsdata/data
+CSST_AUX_DIR=/sharewcl/pipeline/aux
+# IFS CHENWEI
+AUX_DIR=/share/shao/L1Pipeline/aux
+CRDS_DIR=/share/shao/L1Pipeline/crds
+DFS_ROOT=/share/dfs 
 PIPELINE_NAME=csst_lever1
 #PIPELINE_NAME=mbi