version: 0.5.0
name: csst.mcs.single-image-reduction
label: CSST多色成像流水线C3原型
priority: 2
memo: 单组数据多色成像流水线
jobs:
  QC0:
    label: QC0
    base_image: csst/qc0
    cluster: csst
    input_datasets:
    - csst.single-image-reduction.qc0-input:/tasks
    - csst.single-image-reduction.data-L00:/L0
    output_datasets:
    - csst.single-image-reduction.detector-effect-correction-input:/next-stage
    paths:
    - /gfs/single-image-reduction/qc0-log:/qc0-log
    envs:
    - CSST_DFS_GATEWAY=192.168.122.205:30880
    type: CUSTOM-JOB
    hosts:
    - h0:2:2
    next_jobs:
    - detector-effect-correction
    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
  detector-effect-correction:
    label: 探测器效应改正
    base_image: csst/detector-effect-correction
    cluster: csst
    input_datasets:
    - csst.single-image-reduction.detector-effect-correction-input:/tasks
    - csst.single-image-reduction.data-L00:/L0
    output_datasets:
    - csst.single-image-reduction.data-L05:/home/csstpipeline/data/L05_test
    - csst.single-image-reduction.WCS-calibration-input:/next-stage
    paths:
    - /gfs/C3_pipeline/backup/CAL:/home/csstpipeline/data/bkg:ro
    - /gfs/C3_pipeline/backup/CAL:/CAL:ro
    type: CUSTOM-JOB
    hosts:
    - h0:2
    next_jobs:
    - WCS-calibration
    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
  WCS-calibration:
    label: 位置定标
    base_image: csst/wcs-calibration
    cluster: csst
    url_pattern: ^(\d{9})((\d{2}))(.*)$;18
    input_datasets:
    - csst.single-image-reduction.WCS-calibration-input:/tasks
    - csst.single-image-reduction.data-L05:/L05
    output_datasets:
    - csst.single-image-reduction.flux-calibration-input:/next-stage
    paths:
    - /gfs/C3_pipeline/reference_catalogs/gaia/chunks-gaia-dr2-astrom:/gaia
    - /gfs/single-image-reduction/wcs-calibration-output:/output
    type: CUSTOM-JOB
    hosts:
    - h0:0
    next_jobs:
    - flux-calibration
    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
  flux-calibration:
    label: 流量定标
    base_image: csst/flux-calibration
    cluster: csst
    input_datasets:
    - csst.single-image-reduction.flux-calibration-input:/tasks
    - csst.single-image-reduction.data-L05:/L05
    output_datasets:
    - csst.single-image-reduction.data-L10:/L10
    paths:
    - /gfs/single-image-reduction/wcs-calibration-output:/wcs-calibration-output
    - /gfs/C3_pipeline/input_catalogs:/ref_catalogs
    type: CUSTOM-JOB
    hosts:
    - h0:2
    command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE%
  tasklist-QC0:
    label: tasklist-QC0
    base_image: csst/qc0-list
    cluster: csst
    output_datasets:
    - csst.single-image-reduction.qc0-input:/qc0-input
    hosts:
    - h0
    type: CUSTOM-WORKER
    next_jobs:
    - QC0
    command: docker run -d --rm --name tasklist-qc0 --network=host %ENVS% %VOLUMES%
      %IMAGE%
datasets:
  csst.single-image-reduction.data-L00:
    label: 0级数据
    base_uri: dir:h0:/gfs/single-image-reduction/entity/L00
    filter: ^.*fits$
  csst.single-image-reduction.data-L05:
    label: 0.5级数据
    base_uri: dir:h0:/gfs/single-image-reduction/entity/L05
    filter: ^.*\.img\.fits$
  csst.single-image-reduction.data-L10:
    label: 1级数据
    base_uri: dir:h0:/gfs/single-image-reduction/entity/L10
    filter: ^.*fits$
  csst.single-image-reduction.qc0-input:
    label: 扫描队列产生的qc0输入id
    base_uri: dir:h0:/gfs/single-image-reduction/tasks/qc0-input
    filter: ^\d*$
  csst.single-image-reduction.detector-effect-correction-input:
    label: 探测器效应修正输入id
    base_uri: dir:h0:/gfs/single-image-reduction/tasks/detector-effect-correction-input
    filter: ^\d*$
  csst.single-image-reduction.WCS-calibration-input:
    label: 位置定标输入id
    base_uri: dir:h0:/gfs/single-image-reduction/tasks/WCS-calibration-input
    filter: ^\d*$
  csst.single-image-reduction.flux-calibration-input:
    label: 流量定标后输入id
    base_uri: dir:h0:/gfs/single-image-reduction/tasks/flux-calibration-input
    filter: ^\d*$
clusters:
  specs:
    vm-00:
      num_cores: 8
      mem_gb: 32
    vm-01:
      num_cores: 16
      mem_gb: 64
  csst:
    base_dir: /gfs/gateway
    grpc_server: 192.168.1.46:50051
    uid: root
    hosts:
      h0:
        label: head node
        ip_addr: 192.168.1.46
        spec: vm-00
        role: head
      h1:
        label: head node
        ip_addr: 192.168.1.45
        spec: vm-00
        role: head
      n1:
        label: head node
        ip_addr: 192.168.1.52
        spec: vm-01
      n2:
        label: head node
        ip_addr: 192.168.1.54
        spec: vm-01
      n3:
        label: head node
        ip_addr: 192.168.1.55
        spec: vm-01
      n4:
        label: compute node 1
        ip_addr: 192.168.1.56
        spec: vm-01
      n5:
        label: compute node 1
        ip_addr: 192.168.1.49
        spec: vm-01
      n6:
        label: compute node 1
        ip_addr: 192.168.1.47
        spec: vm-01
      n7:
        label: compute node 1
        ip_addr: 192.168.1.50
        spec: vm-01
      n8:
        label: compute node 1
        ip_addr: 192.168.1.53
        spec: vm-01
      n9:
        label: compute node 1
        ip_addr: 192.168.1.48
        spec: vm-01
      n10:
        label: compute node 1
        ip_addr: 192.168.1.51
        spec: vm-01