From f4bb17f1deca9200fda162d6b5e459499d6072e0 Mon Sep 17 00:00:00 2001 From: Tang Jianing Date: Tue, 28 Dec 2021 07:06:10 +0000 Subject: [PATCH] =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=AC=AC=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...cs.single-image-reduction202112281505.yaml | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 csst.mcs.single-image-reduction202112281505.yaml diff --git a/csst.mcs.single-image-reduction202112281505.yaml b/csst.mcs.single-image-reduction202112281505.yaml new file mode 100644 index 0000000..e5c1085 --- /dev/null +++ b/csst.mcs.single-image-reduction202112281505.yaml @@ -0,0 +1,184 @@ +version: 0.5.0 +name: csst.mcs.single-image-reduction +label: CSST多色成像流水线C3原型 +priority: 2 +memo: 单组数据多色成像流水线 +jobs: + QC0: + label: QC0 + base_image: csst/qc0 + cluster: csst + input_datasets: + - csst.single-image-reduction.qc0-input:/tasks + - csst.single-image-reduction.data-L00:/L0 + output_datasets: + - csst.single-image-reduction.detector-effect-correction-input:/next-stage + paths: + - /gfs/single-image-reduction/qc0-log:/qc0-log + envs: + - CSST_DFS_GATEWAY=192.168.122.205:30880 + type: CUSTOM-JOB + hosts: + - h0:2:2 + next_jobs: + - detector-effect-correction + command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE% + detector-effect-correction: + label: 探测器效应改正 + base_image: csst/detector-effect-correction + cluster: csst + input_datasets: + - csst.single-image-reduction.detector-effect-correction-input:/tasks + - csst.single-image-reduction.data-L00:/L0 + output_datasets: + - csst.single-image-reduction.data-L05:/home/csstpipeline/data/L05_test + - csst.single-image-reduction.WCS-calibration-input:/next-stage + paths: + - /gfs/C3_pipeline/backup/CAL:/home/csstpipeline/data/bkg:ro + - /gfs/C3_pipeline/backup/CAL:/CAL:ro + type: CUSTOM-JOB + hosts: + - h0:2 + next_jobs: + - WCS-calibration + command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE% + WCS-calibration: + label: 位置定标 + base_image: csst/wcs-calibration + cluster: csst + url_pattern: ^(\d{9})((\d{2}))(.*)$;18 + input_datasets: + - csst.single-image-reduction.WCS-calibration-input:/tasks + - csst.single-image-reduction.data-L05:/L05 + output_datasets: + - csst.single-image-reduction.flux-calibration-input:/next-stage + paths: + - /gfs/C3_pipeline/reference_catalogs/gaia/chunks-gaia-dr2-astrom:/gaia + - /gfs/single-image-reduction/wcs-calibration-output:/output + type: CUSTOM-JOB + hosts: + - h0:0 + next_jobs: + - flux-calibration + command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE% + flux-calibration: + label: 流量定标 + base_image: csst/flux-calibration + cluster: csst + input_datasets: + - csst.single-image-reduction.flux-calibration-input:/tasks + - csst.single-image-reduction.data-L05:/L05 + output_datasets: + - csst.single-image-reduction.data-L10:/L10 + paths: + - /gfs/single-image-reduction/wcs-calibration-output:/wcs-calibration-output + - /gfs/C3_pipeline/input_catalogs:/ref_catalogs + type: CUSTOM-JOB + hosts: + - h0:2 + command: docker run -d --network=host --rm %ENVS% %VOLUMES% %IMAGE% + tasklist-QC0: + label: tasklist-QC0 + base_image: csst/qc0-list + cluster: csst + output_datasets: + - csst.single-image-reduction.qc0-input:/qc0-input + hosts: + - h0 + type: CUSTOM-WORKER + next_jobs: + - QC0 + command: docker run -d --rm --name tasklist-qc0 --network=host %ENVS% %VOLUMES% + %IMAGE% +datasets: + csst.single-image-reduction.data-L00: + label: 0级数据 + base_uri: dir:h0:/gfs/single-image-reduction/entity/L00 + filter: ^.*fits$ + csst.single-image-reduction.data-L05: + label: 0.5级数据 + base_uri: dir:h0:/gfs/single-image-reduction/entity/L05 + filter: ^.*\.img\.fits$ + csst.single-image-reduction.data-L10: + label: 1级数据 + base_uri: dir:h0:/gfs/single-image-reduction/entity/L10 + filter: ^.*fits$ + csst.single-image-reduction.qc0-input: + label: 扫描队列产生的qc0输入id + base_uri: dir:h0:/gfs/single-image-reduction/tasks/qc0-input + filter: ^\d*$ + csst.single-image-reduction.detector-effect-correction-input: + label: 探测器效应修正输入id + base_uri: dir:h0:/gfs/single-image-reduction/tasks/detector-effect-correction-input + filter: ^\d*$ + csst.single-image-reduction.WCS-calibration-input: + label: 位置定标输入id + base_uri: dir:h0:/gfs/single-image-reduction/tasks/WCS-calibration-input + filter: ^\d*$ + csst.single-image-reduction.flux-calibration-input: + label: 流量定标后输入id + base_uri: dir:h0:/gfs/single-image-reduction/tasks/flux-calibration-input + filter: ^\d*$ +clusters: + specs: + vm-00: + num_cores: 8 + mem_gb: 32 + vm-01: + num_cores: 16 + mem_gb: 64 + csst: + base_dir: /gfs/gateway + grpc_server: 192.168.1.46:50051 + uid: root + hosts: + h0: + label: head node + ip_addr: 192.168.1.46 + spec: vm-00 + role: head + h1: + label: head node + ip_addr: 192.168.1.45 + spec: vm-00 + role: head + n1: + label: head node + ip_addr: 192.168.1.52 + spec: vm-01 + n2: + label: head node + ip_addr: 192.168.1.54 + spec: vm-01 + n3: + label: head node + ip_addr: 192.168.1.55 + spec: vm-01 + n4: + label: compute node 1 + ip_addr: 192.168.1.56 + spec: vm-01 + n5: + label: compute node 1 + ip_addr: 192.168.1.49 + spec: vm-01 + n6: + label: compute node 1 + ip_addr: 192.168.1.47 + spec: vm-01 + n7: + label: compute node 1 + ip_addr: 192.168.1.50 + spec: vm-01 + n8: + label: compute node 1 + ip_addr: 192.168.1.53 + spec: vm-01 + n9: + label: compute node 1 + ip_addr: 192.168.1.48 + spec: vm-01 + n10: + label: compute node 1 + ip_addr: 192.168.1.51 + spec: vm-01 -- GitLab