From e8e9440e4bc2e64b5c5a91264962120f244dca61 Mon Sep 17 00:00:00 2001 From: zxl Date: Fri, 11 Aug 2023 11:27:29 +0800 Subject: [PATCH] finish ooc-exe test. --- .../csst-ifs-l1-cube-adm/Dockerfile | 20 ------- L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile | 21 ------- .../csst-ifs-l1-cube-adm/add_task.sh | 18 ------ L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh | 26 --------- .../csst-ifs-l1-cube-adm/test/Makefile | 16 ----- .../csst-ifs-l1-cube-adm/test/add_task.sh | 18 ------ .../csst-ifs-l1-cube-adm/test/app.yaml | 28 --------- .../csst-ifs-l1-cube-adm/test/zjs.env | 3 - .../csst-ifs-l1-cube-exe/Dockerfile | 20 ------- L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile | 21 ------- .../csst-ifs-l1-cube-exe/add_task.sh | 18 ------ L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh | 24 -------- .../csst-ifs-l1-cube-exe/test/Makefile | 16 ----- .../csst-ifs-l1-cube-exe/test/add_task.sh | 18 ------ .../csst-ifs-l1-cube-exe/test/app.yaml | 28 --------- .../csst-ifs-l1-cube-exe/test/zjs.env | 3 - L1/dockerfiles/csst-msc-l1-ooc/exe/run.sh | 5 +- L1/dockerfiles/csst-msc-l1-ooc/test/Makefile | 2 +- L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml | 58 ++++++++++--------- 19 files changed, 35 insertions(+), 328 deletions(-) delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml delete mode 100644 L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile deleted file mode 100644 index 4137e3d..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM csst/csst-ifs-l1-cube-adm:latest -LABEL maintainer="Xiaoli Zhang" - -COPY run.sh /app/bin/ -RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin - -RUN mkdir /work/ -#WORKDIR /work/ -WORKDIR /L1Pipeline/workspace - -ENTRYPOINT ["goagent"] - - -# RUN mkdir -p /L1Pipeline/workspace -# WORKDIR /L1Pipeline/workspace -# COPY code/ /L1Pipeline/workspace -# RUN /app/bin/run.sh - - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile deleted file mode 100644 index 68a5f33..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -IMAGE_NAME:=csst/ifs-l1-cube-adm - -build: - docker build --network=host -t $(IMAGE_NAME) . - -dist: -#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar - -push: - docker push $(IMAGE_NAME) -run: - docker run -it --entrypoint bash $(IMAGE_NAME) -down: - docker stop $(IMAGE_NAME) -scp: - scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-adm/ - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh deleted file mode 100644 index a41e902..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh +++ /dev/null @@ -1,18 +0,0 @@ - -#!/bin/bash - -POSTGRES_USER=scalebox -POSTGRES_HOST=localhost -POSTGRES_DB=scalebox -PGPORT=5432 - -job_id=$1 - -objids=("NGC6217" "NGC6397") -for((i=0;i<${#objids[@]};i++)); do - m=${objids[$i]} - echo $m - docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ - -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" -done - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh deleted file mode 100644 index b710853..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# python helloworld.py - - -# obsid="30100000001" - -obsid=$1 - -python l1_pipeline_script_ifs_cube_adm_simdata.py $obsid > ./stdout - -exit_code=$? - -mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt - -echo finish ifs cube adm. - -cat ./stdout - -exit $exit_code - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile deleted file mode 100644 index 50026e2..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -CLUSTER=csst - -all: reset build - -run: - scalebox app create --env-file zjs.env - -reset: - cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - - -down: - make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down - -list: - make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh deleted file mode 100644 index a41e902..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh +++ /dev/null @@ -1,18 +0,0 @@ - -#!/bin/bash - -POSTGRES_USER=scalebox -POSTGRES_HOST=localhost -POSTGRES_DB=scalebox -PGPORT=5432 - -job_id=$1 - -objids=("NGC6217" "NGC6397") -for((i=0;i<${#objids[@]};i++)); do - m=${objids[$i]} - echo $m - docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ - -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" -done - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml deleted file mode 100644 index c29cee7..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: ifs-l1-cube-adm.app.process -cluster: csst -parameters: - initial_status: RUNNING - -jobs: - ifs-l1-cube-adm: - base_image: csst/ifs-l1-cube-adm - schedule_mode: HEAD - variables: - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - parameters: - # start_message: scalebox - key_group_regex: ^(.{6})(.{3})$ - key_group_seq: 1,2 - paths: - - ${AUX_DIR}:/L1Pipeline/aux - - ${CRDS_DIR}:/L1Pipeline/crds - - ${DFS_ROOT}:/L1Pipeline/dfs - # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} - # sink_jobs: - # hosts: - # - h0:1 - # - c0:2 - # - c1:2 - # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env b/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env deleted file mode 100644 index 9ff022c..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env +++ /dev/null @@ -1,3 +0,0 @@ -AUX_DIR=/sharewcl/shao/L1Pipeline/aux -CRDS_DIR=/sharewcl/shao/L1Pipeline/crds -DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile b/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile deleted file mode 100644 index ae87f6b..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM csst/csst-ifs-l1-cube-exe:latest -LABEL maintainer="Xiaoli Zhang" - -COPY run.sh /app/bin/ -RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin - -RUN mkdir /work/ -#WORKDIR /work/ -WORKDIR /L1Pipeline/workspace - -ENTRYPOINT ["goagent"] - - -# RUN mkdir -p /L1Pipeline/workspace -# WORKDIR /L1Pipeline/workspace -# COPY code/ /L1Pipeline/workspace -# RUN /app/bin/run.sh - - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile deleted file mode 100644 index 4d7449a..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -IMAGE_NAME:=csst/ifs-l1-cube-exe - -build: - docker build --network=host -t $(IMAGE_NAME) . - -dist: -#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar - ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar - ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar - -push: - docker push $(IMAGE_NAME) -run: - docker run -it --entrypoint bash $(IMAGE_NAME) -down: - docker stop $(IMAGE_NAME) -scp: - scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-exe/ - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh deleted file mode 100644 index a41e902..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh +++ /dev/null @@ -1,18 +0,0 @@ - -#!/bin/bash - -POSTGRES_USER=scalebox -POSTGRES_HOST=localhost -POSTGRES_DB=scalebox -PGPORT=5432 - -job_id=$1 - -objids=("NGC6217" "NGC6397") -for((i=0;i<${#objids[@]};i++)); do - m=${objids[$i]} - echo $m - docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ - -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" -done - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh deleted file mode 100644 index 7f03a20..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# python helloworld.py - - -# obsid="30100000001" - -obsid=$1 - -python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout - -exit_code=$? - -mkdir -p /work -# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt -# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt - -# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* - -# mv /tmp/timestamps.txt /work/timestamps.txt - -echo finish ifs cube exe. - -exit $exit_code - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile deleted file mode 100644 index 50026e2..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -CLUSTER=csst - -all: reset build - -run: - scalebox app create --env-file zjs.env - -reset: - cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd - - -down: - make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down - -list: - make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh deleted file mode 100644 index a41e902..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh +++ /dev/null @@ -1,18 +0,0 @@ - -#!/bin/bash - -POSTGRES_USER=scalebox -POSTGRES_HOST=localhost -POSTGRES_DB=scalebox -PGPORT=5432 - -job_id=$1 - -objids=("NGC6217" "NGC6397") -for((i=0;i<${#objids[@]};i++)); do - m=${objids[$i]} - echo $m - docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ - -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" -done - diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml deleted file mode 100644 index 68970d7..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: ifs-l1-cube-exe.app.process -cluster: csst -parameters: - initial_status: RUNNING - -jobs: - ifs-l1-cube-exe: - base_image: csst/ifs-l1-cube-exe - schedule_mode: HEAD - variables: - output_text_size: 100000 - text_tranc_mode: TAIL - locale_mode: NONE - parameters: - # start_message: scalebox - key_group_regex: ^(.{6})(.{3})$ - key_group_seq: 1,2 - paths: - - ${AUX_DIR}:/L1Pipeline/aux - - ${CRDS_DIR}:/L1Pipeline/crds - - ${DFS_ROOT}:/L1Pipeline/dfs - # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}} - # sink_jobs: - # hosts: - # - h0:1 - # - c0:2 - # - c1:2 - # - c2:2 \ No newline at end of file diff --git a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env b/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env deleted file mode 100644 index 9ff022c..0000000 --- a/L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env +++ /dev/null @@ -1,3 +0,0 @@ -AUX_DIR=/sharewcl/shao/L1Pipeline/aux -CRDS_DIR=/sharewcl/shao/L1Pipeline/crds -DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs \ No newline at end of file diff --git a/L1/dockerfiles/csst-msc-l1-ooc/exe/run.sh b/L1/dockerfiles/csst-msc-l1-ooc/exe/run.sh index 5b3be79..e162d86 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/exe/run.sh +++ b/L1/dockerfiles/csst-msc-l1-ooc/exe/run.sh @@ -1,9 +1,10 @@ #!/bin/bash rm -rf /pipeline/input/* /pipeline/output/* -echo input:$1* +# echo input:$1* -python /pipeline/src/exe.py $1 +# python /pipeline/src/exe.py $1 +python /pipeline/src/run.py exit_code=$? diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile index b9e5471..4cdab67 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/Makefile @@ -1,4 +1,4 @@ -CLUSTER=csst +CLUSTER=local all: reset build diff --git a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml index ba5dd02..39176d4 100644 --- a/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml +++ b/L1/dockerfiles/csst-msc-l1-ooc/test/app.yaml @@ -1,37 +1,38 @@ name: msc-l1-ooc.app.process label: 在轨定标 comment: 主巡天在轨定标一级流水线 -cluster: csst +cluster: local parameters: initial_status: RUNNING jobs: - csst-msc-l1-ooc-adm: - label: 在轨定标管理模块 - base_image: cnic/csst-msc-l1-ooc-adm:latest - schedule_mode: HEAD - variables: - # always_running: yes - output_text_size: 100000 - # DFS入库错误重试3次 - retry_rules: "['202:3']" - text_tranc_mode: TAIL - locale_mode: NONE - # grpc_server: 172.24.23.6:50051 - parameters: - start_message: ANY - environments: - - CSST_DFS_API_MODE=cluster - - CSST_DFS_GATEWAY=172.24.27.2:30880 - - CSST_DFS_APP_ID=test - - CSST_DFS_APP_TOKEN=test - paths: # change /share to /sharewcl in PML production environment - - ${CSST_AUX_DIR}:/pipeline/aux - - ${CSST_DFS_ROOT}:/dfsroot - - ${CSST_CRDS_ROOT}:/crdsroot - sink_jobs: - - csst-msc-l1-ooc-exe + # csst-msc-l1-ooc-adm: + # label: 在轨定标管理模块 + # base_image: cnic/csst-msc-l1-ooc-adm:latest + # schedule_mode: HEAD + # variables: + # # always_running: yes + # output_text_size: 100000 + # # DFS入库错误重试3次 + # retry_rules: "['202:3']" + # text_tranc_mode: TAIL + # locale_mode: NONE + # # grpc_server: 172.24.23.6:50051 + # parameters: + # start_message: ANY + # environments: + # - CSST_DFS_API_MODE=cluster + # - CSST_DFS_GATEWAY=172.24.27.2:30880 + # - CSST_DFS_APP_ID=test + # - CSST_DFS_APP_TOKEN=test + # - CRDS_SERVER_URL=http://172.24.27.2:29000 + # paths: # change /share to /sharewcl in PML production environment + # - ${CSST_AUX_DIR}:/pipeline/aux + # - ${CSST_DFS_ROOT}:/dfsroot + # - ${CSST_CRDS_ROOT}:/crdsroot + # sink_jobs: + # - csst-msc-l1-ooc-exe # command: > # bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt" @@ -39,8 +40,10 @@ jobs: label: 在轨定标执行模块 base_image: cnic/csst-msc-l1-ooc-exe:latest schedule_mode: HEAD + parameters: + start_message: ANY variables: - always_running: yes + # always_running: yes output_text_size: 100000 # DFS入库错误重试3次 retry_rules: "['202:3']" @@ -52,6 +55,7 @@ jobs: - CSST_DFS_GATEWAY=172.24.27.2:30880 - CSST_DFS_APP_ID=test - CSST_DFS_APP_TOKEN=test + - CRDS_SERVER_URL=http://172.24.27.2:29000 paths: # change /share to /sharewcl in PML production environment # output directory, omit this line in production - ${CSST_AUX_DIR}:/pipeline/aux -- GitLab