Skip to content

GitLab

Commit b5500664 authored by Zhang Xiaoli's avatar Zhang Xiaoli
Browse files

测试新格式msg-all.txt消息处理，先解析流水线名称提取消息，再进行处理

parent 16936a2c
Hide whitespace changes
Inline Side-by-side
Showing with 37005 additions and 0 deletions
L1/pipelines/pipeline-nao/Makefile 0 → 100644
View file @ b5500664
CLUSTER=csst-nao
all: reset build
run:
# PGHOST=192.168.25.27 GRPC_SERVER=192.168.25.27 scalebox app create --env-file csu.env
PGPORT=9090 scalebox app create --env-file nao.env
reset:
cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
down:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
list:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
L1/pipelines/pipeline-nao/add_mbi.sh 0 → 100644
View file @ b5500664
#!/bin/bash
POSTGRES_USER=scalebox
POSTGRES_HOST=localhost
POSTGRES_DB=scalebox
PGPORT=5432
job_id=$1
# obsid="test.txt"
obsid="/home/scalebox/csst/csst-msc-l1-mbi.txt"
while IFS= read -r line
do
msg="$line"
echo $msg
docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
-c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
done < $obsid
~
L1/pipelines/pipeline-nao/add_sls.sh 0 → 100644
View file @ b5500664
#!/bin/bash
POSTGRES_USER=scalebox
POSTGRES_HOST=localhost
POSTGRES_DB=scalebox
PGPORT=5432
job_id=$1
# obsid="test.txt"
obsid="/home/scalebox/csst/csst-msc-l1-sls.txt"
while IFS= read -r line
do
msg="$line"
echo $msg
docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
-c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
done < $obsid
~
L1/pipelines/pipeline-nao/add_task.sh 0 → 100644
View file @ b5500664
#!/bin/bash
POSTGRES_USER=scalebox
POSTGRES_HOST=localhost
POSTGRES_DB=scalebox
PGPORT=5432
job_id=$1
# obsid="test.txt"
obsid="/nfsdata/share/simulation-collection/msc/v092/raw/msg-all.txt"
while IFS= read -r line
do
m="$line"
arr=($(echo $m | tr " " " "))
obsid="${arr[0]}"
detector="${arr[1]}"
msg=$obsid-$detector
echo $msg
docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
-c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
done < $obsid
~
L1/pipelines/pipeline-nao/app-l1.yaml 0 → 100644
View file @ b5500664
name: csst-msc-l1.app.process
label: 主巡天一级流水线
comment: 主巡天一级流水线
cluster: csst-nao
parameters:
initial_status: RUNNING
jobs:
msc-l1-qc0:
label: QC0
base_image: cnic/csst-msc-l1-qc0
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# max_sleep_count: 60000
# grpc_server: 172.24.23.6:50051
parameters:
# start_message: 10160000068
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
sink_jobs:
- adm-L1
sink_vjobs:
- adm-L1
hosts:
- h0:1
adm-L1:
label: 消息分发网关
base_image: csst/adm-l1
# schedule_mode: HEAD
variables:
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# max_sleep_count: 60000
# grpc_server: 10.3.10.28:50051
# repeated: yes
parameters:
slot_on_head: yes
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
#command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
sink_jobs:
- msc-l1-mbi
- msc-l1-sls
sink_vjobs:
- msc-l1-mbi
- msc-l1-sls
environments:
- CSST_PIPELINE_GRPC_SERVER="10.3.10.28:50051"
hosts:
- h0:1
msc-l1-mbi:
label: 多色成像
base_image: cnic/csst-msc-l1-mbi
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 10.3.10.28:50051
parameters:
# start_message: 10160000068
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
sink_jobs:
- msc-l1-mbi-photmix
sink_vjobs:
- msc-l1-mbi-photmix
hosts:
# - h0:1
- c0:10
# - c1:1
# - c2:1
msc-l1-sls:
label: 无缝光谱
base_image: cnic/csst-msc-l1-sls
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
# DFS入库错误重试3次
# retry_rules: "['202:3']"
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
parameters:
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
sink_jobs:
- msc-l1-sls-extraction-zero
sink_vjobs:
- msc-l1-sls-extraction-zero
hosts:
# - h0:1
- c0:10
# - c1:3
# - c2:3
msc-l1-mbi-photmix:
label: 多色成像测光
base_image: cnic/csst-msc-l1-mbi-photmix
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
parameters:
# start_message: 10160000068
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
hosts:
- h0:5
msc-l1-sls-extraction-zero:
label: 无缝光谱抽谱
base_image: cnic/csst-msc-l1-sls-extraction-zero
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
parameters:
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
hosts:
- h0:5
\ No newline at end of file
L1/pipelines/pipeline-nao/app.yaml 0 → 100644
View file @ b5500664
name: csst-msc-l1.app.process
label: 主巡天一级流水线(mbi+sls)
comment: 主巡天一级流水线
cluster: csst-nao
parameters:
initial_status: RUNNING
jobs:
csst-msc-l1-mbi:
label: 多色成像
base_image: cnic/csst-msc-l1-mbi
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 10.3.10.28:50051
parameters:
# start_message: 10160000068
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
# sink_jobs:
# - msc-l1-mbi-photmix
# sink_vjobs:
# - msc-l1-mbi-photmix
hosts:
- h0:10
# - c0:10
# - c1:1
# - c2:1
csst-msc-l1-sls:
label: 无缝光谱
base_image: cnic/csst-msc-l1-sls
# schedule_mode: HEAD
variables:
# always_running: yes
# reserved_on_exit: yes
# DFS入库错误重试3次
# retry_rules: "['202:3']"
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
parameters:
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
environments:
- CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
- CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
- CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
- CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
- CCDS_SERVER_URL=${CCDS_SERVER_URL}
paths:
- ${CSST_AUX_ROOT}:/pipeline/aux:ro
- ${CSST_DFS_ROOT}:/dfs_root:ro
- ${CCDS_ROOT}:/ccds_root:ro
- ${CSST_AST_TEMP}:/pipeline/temp:rw
# sink_jobs:
# - msc-l1-sls-extraction-zero
# sink_vjobs:
# - msc-l1-sls-extraction-zero
hosts:
# - h0:1
- c0:20
# - c1:3
# - c2:3
\ No newline at end of file
L1/pipelines/pipeline-nao/csst-msc-l1-mbi.txt 0 → 100644
View file @ b5500664
This source diff could not be displayed because it is too large. You can view the blob instead.
L1/pipelines/pipeline-nao/csst-msc-l1-sls.txt 0 → 100644
View file @ b5500664
This source diff could not be displayed because it is too large. You can view the blob instead.
L1/pipelines/pipeline-nao/nao.env 0 → 100644
View file @ b5500664
# DFS
CSST_DFS_GATEWAY=10.3.10.28:30880
CSST_DFS_API_MODE=cluster
CSST_DFS_APP_ID=test
CSST_DFS_APP_TOKEN=test
CSST_DFS_LOGS_DIR="."
# CCDS
CCDS_SERVER_URL=http://10.3.10.28:29000
CCDS_USER=USER
CCDS_PASS=PASS
# VOLUMES
CSST_DFS_ROOT=/nfsdata/share/dfs/dfs_root # /dfs_root:ro
CCDS_ROOT=/nfsdata/share/dfs/ccds_root # /ccds_root:ro
CSST_AUX_ROOT=/nfsdata/share/pipeline-inttest/aux # /pipeline/aux:ro
CSST_AST_TEMP=/nfsdata/share/pipeline-inttest/ast_temp # /pipeline/temp:rw
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment