yayaeef

pipeline-3.yaml

+version: 0.5.0
+name: csst.mcs.single-image-reduction
+label: CSST多色成像流水线C3原型
+priority: 2
+memo: 单组数据多色成像流水线
+jobs:
+  QC0:
+    label: QC0
+    base_image: csst/qc0
+    cluster: csst
+    input_datasets:
+    - csst.single-image-reduction.qc0-input:/tasks
+    - csst.single-image-reduction.data-L00:/L0
+    output_datasets:
+    - csst.single-image-reduction.detector-effect-correction-input:/next-stage
+    paths:
+    - /gfs/single-image-reduction/qc0-log:/qc0-log
+    envs:
+    - CSST_DFS_GATEWAY=192.168.122.205:30880
+    type: CUSTOM-JOB
+    hosts:
+    - h0:2:2
+    next_jobs:
+    - detector-effect-correction
+    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
+  detector-effect-correction:
+    label: 探测器效应改正
+    base_image: csst/detector-effect-correction
+    cluster: csst
+    input_datasets:
+    - csst.single-image-reduction.detector-effect-correction-input:/tasks
+    - csst.single-image-reduction.data-L00:/L0
+    output_datasets:
+    - csst.single-image-reduction.data-L05:/home/csstpipeline/data/L05_test
+    - csst.single-image-reduction.WCS-calibration-input:/next-stage
+    paths:
+    - /gfs/C3_pipeline/backup/CAL:/home/csstpipeline/data/bkg:ro
+    - /gfs/C3_pipeline/backup/CAL:/CAL:ro
+    type: CUSTOM-JOB
+    hosts:
+    - h0:2
+    next_jobs:
+    - WCS-calibration
+    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
+  WCS-calibration:
+    label: 位置定标
+    base_image: csst/wcs-calibration
+    cluster: csst
+    url_pattern: ^(\d{9})((\d{2}))(.*)$;18
+    input_datasets:
+    - csst.single-image-reduction.WCS-calibration-input:/tasks
+    - csst.single-image-reduction.data-L05:/L05
+    output_datasets:
+    - csst.single-image-reduction.flux-calibration-input:/next-stage
+    paths:
+    - /gfs/C3_pipeline/reference_catalogs/gaia/chunks-gaia-dr2-astrom:/gaia
+    - /gfs/single-image-reduction/wcs-calibration-output:/output
+    type: CUSTOM-JOB
+    hosts:
+    - h0:0
+    next_jobs:
+    - flux-calibration
+    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
+  flux-calibration:
+    label: 流量定标
+    base_image: csst/flux-calibration
+    cluster: csst
+    input_datasets:
+    - csst.single-image-reduction.flux-calibration-input:/tasks
+    - csst.single-image-reduction.data-L05:/L05
+    output_datasets:
+    - csst.single-image-reduction.data-L10:/L10
+    paths:
+    - /gfs/single-image-reduction/wcs-calibration-output:/wcs-calibration-output
+    - /gfs/C3_pipeline/input_catalogs:/ref_catalogs
+    type: CUSTOM-JOB
+    hosts:
+    - h0:2
+    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
+  tasklist-QC0:
+    label: tasklist-QC0
+    base_image: csst/qc0-list
+    cluster: csst
+    output_datasets:
+    - csst.single-image-reduction.qc0-input:/qc0-input
+    hosts:
+    - h0
+    type: CUSTOM-WORKER
+    next_jobs:
+    - QC0
+    command: docker run -d --rm --name tasklist-qc0 --network=host %ENVS% %VOLUMES%
+      %IMAGE%
+datasets:
+  csst.single-image-reduction.data-L00:
+    label: 0级数据
+    base_uri: dir:h0:/gfs/single-image-reduction/entity/L00
+    filter: ^.*fits$
+  csst.single-image-reduction.data-L05:
+    label: 0.5级数据
+    base_uri: dir:h0:/gfs/single-image-reduction/entity/L05
+    filter: ^.*\.img\.fits$
+  csst.single-image-reduction.data-L10:
+    label: 1级数据
+    base_uri: dir:h0:/gfs/single-image-reduction/entity/L10
+    filter: ^.*fits$
+  csst.single-image-reduction.qc0-input:
+    label: 扫描队列产生的qc0输入id
+    base_uri: dir:h0:/gfs/single-image-reduction/tasks/qc0-input
+    filter: ^\d*$
+  csst.single-image-reduction.detector-effect-correction-input:
+    label: 探测器效应修正输入id
+    base_uri: dir:h0:/gfs/single-image-reduction/tasks/detector-effect-correction-input
+    filter: ^\d*$
+  csst.single-image-reduction.WCS-calibration-input:
+    label: 位置定标输入id
+    base_uri: dir:h0:/gfs/single-image-reduction/tasks/WCS-calibration-input
+    filter: ^\d*$
+  csst.single-image-reduction.flux-calibration-input:
+    label: 流量定标后输入id
+    base_uri: dir:h0:/gfs/single-image-reduction/tasks/flux-calibration-input
+    filter: ^\d*$
+clusters:
+  specs:
+    vm-00:
+      num_cores: 8
+      mem_gb: 32
+    vm-01:
+      num_cores: 16
+      mem_gb: 64
+  csst:
+    base_dir: /gfs/gateway
+    grpc_server: 192.168.1.46:50051
+    uid: root
+    hosts:
+      h0:
+        label: head node
+        ip_addr: 192.168.1.46
+        spec: vm-00
+        role: head
+      h1:
+        label: head node
+        ip_addr: 192.168.1.45
+        spec: vm-00
+        role: head
+      n1:
+        label: head node
+        ip_addr: 192.168.1.52
+        spec: vm-01
+      n2:
+        label: head node
+        ip_addr: 192.168.1.54
+        spec: vm-01
+      n3:
+        label: head node
+        ip_addr: 192.168.1.55
+        spec: vm-01
+      n4:
+        label: compute node 1
+        ip_addr: 192.168.1.56
+        spec: vm-01
+      n5:
+        label: compute node 1
+        ip_addr: 192.168.1.49
+        spec: vm-01
+      n6:
+        label: compute node 1
+        ip_addr: 192.168.1.47
+        spec: vm-01
+      n7:
+        label: compute node 1
+        ip_addr: 192.168.1.50
+        spec: vm-01
+      n8:
+        label: compute node 1
+        ip_addr: 192.168.1.53
+        spec: vm-01
+      n9:
+        label: compute node 1
+        ip_addr: 192.168.1.48
+        spec: vm-01
+      n10:
+        label: compute node 1
+        ip_addr: 192.168.1.51
+        spec: vm-01