diff --git a/L1/dockerfiles/mbi/Dockerfile b/L1/dockerfiles/mbi/Dockerfile index 5a0245a3b7ef1de60313979814399db98316e8ce..c8eae2197c4c0088f67a2176a9ebcd5655464a52 100644 --- a/L1/dockerfiles/mbi/Dockerfile +++ b/L1/dockerfiles/mbi/Dockerfile @@ -1,8 +1,8 @@ -FROM csst-l1ppl:latest +FROM csst/csst-msc-l1-mbi:latest COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin RUN mkdir /work/ WORKDIR /work/ diff --git a/L1/dockerfiles/mbi/Makefile b/L1/dockerfiles/mbi/Makefile index d3ab275c5c022ce044106a314419230fef8b09da..4f26f977efcb06f1b6594fe47d18ef7e2954e595 100644 --- a/L1/dockerfiles/mbi/Makefile +++ b/L1/dockerfiles/mbi/Makefile @@ -1,14 +1,14 @@ -IMAGE_NAME:=csst/mbi +IMAGE_NAME:=cnic/csst-msc-l1-mbi build: docker build --network=host -t $(IMAGE_NAME) . dist: #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/mbi.tar - ssh c0 docker load -i /share/tmp/mbi.tar - ssh c1 docker load -i /share/tmp/mbi.tar - ssh c2 docker load -i /share/tmp/mbi.tar + docker save $(IMAGE_NAME) > /share/tmp/msc-l1-mbi.tar + ssh c0 docker load -i /share/tmp/msc-l1-mbi.tar + ssh c1 docker load -i /share/tmp/msc-l1-mbi.tar + ssh c2 docker load -i /share/tmp/msc-l1-mbi.tar push: docker push $(IMAGE_NAME) @@ -16,3 +16,5 @@ run: docker run -it --entrypoint bash $(IMAGE_NAME) down: docker stop $(IMAGE_NAME) +scp: + scp -r ./ csst-zjs:/root/csst/mbi/ \ No newline at end of file diff --git a/L1/dockerfiles/mbi/run.sh b/L1/dockerfiles/mbi/run.sh index cc03649d1dfcf588b8eb312dfa78fb583ff378e6..80a45c2d8975f4d63d24dda7081b1ccbcd868220 100644 --- a/L1/dockerfiles/mbi/run.sh +++ b/L1/dockerfiles/mbi/run.sh @@ -8,18 +8,35 @@ # obsid 10160000000 - 10160000136 obsid=$1 -#python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $obsid > /work/stdout 2> /work/stderr -l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry +# l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry +python /pipeline/src/run.py \ + --obs-id=$obsid \ + --device=cpu \ + --n-jobs=18 \ + --n-jobs-gpu=9 \ + --clean-l0 \ + --clean-l1 \ + --dfs-node=pml \ + --ver-sim=C6.2 \ + --no-astrometry \ + --no-photometry exit_code=$? echo "=====csst-l1ppl.log====" > /work/user-file.txt -cat /L1Pipeline/L1/csst-l1ppl.log >> /work/user-file.txt +cat /pipeline/output/csst-l1ppl.log >> /work/user-file.txt + + echo "======csst-l1mod.log======" >> /work/user-file.txt -cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 >> /work/user-file.txt +cat /pipeline/output/csst-l1mod.log|tail -n 100 >> /work/user-file.txt + -rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* +timefile=/pipeline/output/timestamps.txt +if test -f "$timefile"; then + echo "$timefile exist" + mv /pipeline/output/timestamps.txt /work/timestamps.txt +fi -mv /tmp/timestamps.txt /work/timestamps.txt +rm -rf /pipeline/input/* /pipeline/output/* echo finish mbi. diff --git a/L1/dockerfiles/mbi/test/add_task.sh b/L1/dockerfiles/mbi/test/add_task.sh index 15a3411d5abc6998dc3f524fb848810ee1ec6e2b..74851677432c82d593fa40d6af52e2a788ad64c7 100644 --- a/L1/dockerfiles/mbi/test/add_task.sh +++ b/L1/dockerfiles/mbi/test/add_task.sh @@ -8,7 +8,7 @@ PGPORT=5432 job_id=$1 -for m in {10160000000..10160000001}; do +for m in {10160000066..10160000068}; do echo $m docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \ -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')" diff --git a/L1/dockerfiles/mbi/test/app.yaml b/L1/dockerfiles/mbi/test/app.yaml index 3e37ae4cc812796f9a0a92f450716d3fb17cffa5..51d4be8c02993e5a501e4cf8791a9ab63f8bca91 100644 --- a/L1/dockerfiles/mbi/test/app.yaml +++ b/L1/dockerfiles/mbi/test/app.yaml @@ -1,32 +1,36 @@ -name: mbi.app.process -label: csst-mbi -cluster: local +name: msc-l1-mbi.app.process +label: 多色成像 +comment: 主巡天多色成像一级流水线 +cluster: csst parameters: initial_status: RUNNING jobs: - mbi: - base_image: csst/mbi - schedule_mode: HEAD + msc-l1-mbi: + label: 多色成像 + base_image: cnic/csst-msc-l1-mbi + # schedule_mode: HEAD variables: - # repeated: yes + # always_running: yes + # reserved_on_exit: yes output_text_size: 100000 text_tranc_mode: TAIL locale_mode: NONE parameters: + # start_message: 10160000068 key_group_regex: ^(.{6})(.{3})$ key_group_seq: 1,2 paths: - - ${AUX_DIR}:/L1Pipeline/aux + - ${AUX_DIR}:/pipeline/aux - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds - ${DFS_ROOT}:/dfsroot # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% # sink_jobs: hosts: - - h0:1 - - c0:2 - - c1:2 - - c2:2 + # - h0:1 + - c0:1 + - c1:1 + - c2:1 diff --git a/L1/dockerfiles/mbi/test/zjs.env b/L1/dockerfiles/mbi/test/zjs.env index 1f64fd8a8acb5dea29a81bb0525feb264253fc2d..91bf38556013bfd1d2853b271e4c73eb03d052db 100644 --- a/L1/dockerfiles/mbi/test/zjs.env +++ b/L1/dockerfiles/mbi/test/zjs.env @@ -1,3 +1,7 @@ -AUX_DIR=/sharewcl/L1Pipeline/aux +AUX_DIR=/sharewcl/pipeline/aux CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr DFS_ROOT=/sharewcl/dfs + +# AUX_DIR=/share/pipeline/aux +# CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr +# DFS_ROOT=/share/dfs \ No newline at end of file diff --git a/L1/dockerfiles/sls2d/Dockerfile b/L1/dockerfiles/sls2d/Dockerfile index 822654c5130d5d6d52c3a8a1eee0a8ce27cc6ab8..7c1450af778bb63229b2859df928518f3b28cd8c 100644 --- a/L1/dockerfiles/sls2d/Dockerfile +++ b/L1/dockerfiles/sls2d/Dockerfile @@ -1,8 +1,8 @@ -FROM csst-l1ppl:latest +FROM csst/csst-msc-l1-sls2d:latest COPY run.sh /app/bin/ RUN chmod +x /app/bin/run.sh -COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin +COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin RUN mkdir /work/ WORKDIR /work/ diff --git a/L1/dockerfiles/sls2d/Makefile b/L1/dockerfiles/sls2d/Makefile index 6dd57ca439e19e7bde28cfff827a2c4da3aa3274..e6571b6f2c13279c3d44c5be67d0fd2a309143be 100644 --- a/L1/dockerfiles/sls2d/Makefile +++ b/L1/dockerfiles/sls2d/Makefile @@ -1,14 +1,14 @@ -IMAGE_NAME:=csst/sls2d +IMAGE_NAME:=cnic/csst-msc-l1-sls2d build: docker build --network=host -t $(IMAGE_NAME) . -dist: build +dist: #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load' - docker save $(IMAGE_NAME) > /share/tmp/sls2d.tar - ssh c0 docker load -i /share/tmp/sls2d.tar - ssh c1 docker load -i /share/tmp/sls2d.tar - ssh c2 docker load -i /share/tmp/sls2d.tar + docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls2d.tar + ssh c0 docker load -i /share/tmp/msc-l1-sls2d.tar + ssh c1 docker load -i /share/tmp/msc-l1-sls2d.tar + ssh c2 docker load -i /share/tmp/msc-l1-sls2d.tar push: docker push $(IMAGE_NAME) run: diff --git a/L1/dockerfiles/sls2d/run.sh b/L1/dockerfiles/sls2d/run.sh index e4530b7c06ccb4db7284d4d1ee8ac8c248ef2cc2..353baa38667287df69f716d6e8a50f86b1f3d3e9 100644 --- a/L1/dockerfiles/sls2d/run.sh +++ b/L1/dockerfiles/sls2d/run.sh @@ -9,17 +9,30 @@ # obsid 10160000000 - 10160000136 obsid=$1 #python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $obsid > /work/stdout 2> /work/stderr -l1-sls2d --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry > /work/stdout 2> /work/stderr +# l1-sls2d --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry > /work/stdout 2> /work/stderr +python /pipeline/src/run.py \ + --obs-id=$obsid \ + --device=cpu \ + --n-jobs=12 \ + --n-jobs-gpu=6 \ + --clean-l0 \ + --clean-l1 \ + --dfs-node=pml \ + --ver-sim=C6.2 exit_code=$? echo "=====csst-l1ppl.log====" > /work/user-file.txt -cat /L1Pipeline/L1/csst-l1ppl.log >> /work/user-file.txt +cat /pipeline/output/csst-l1ppl.log >> /work/user-file.txt echo "======csst-l1mod.log======" >> /work/user-file.txt -cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 >> /work/user-file.txt +cat /pipeline/output/csst-l1mod.log|tail -n 100 >> /work/user-file.txt -rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/* +timefile=/pipeline/output/timestamps.txt +if test -f "$timefile"; then + echo "$timefile exist" + mv /pipeline/output/timestamps.txt /work/timestamps.txt +fi -mv /tmp/timestamps.txt /work/timestamps.txt +rm -rf /pipeline/input/* /pipeline/output/* echo finish sls2d. diff --git a/L1/dockerfiles/sls2d/test/Makefile b/L1/dockerfiles/sls2d/test/Makefile index 534943d4645ed7eb2ad10231e835d5f30b594cd8..0e709ce8a1ebf4ba2de48f361fe44c1d1624a36a 100644 --- a/L1/dockerfiles/sls2d/test/Makefile +++ b/L1/dockerfiles/sls2d/test/Makefile @@ -1,4 +1,4 @@ -CLUSTER=local +CLUSTER=csst # all: reset build diff --git a/L1/dockerfiles/sls2d/test/app.yaml b/L1/dockerfiles/sls2d/test/app.yaml index 1dbad60f9fa56d726dd0ac096f72d1dd41b094c6..d7fdb86cfac485d79c8bdd9c4589ae972a987a54 100644 --- a/L1/dockerfiles/sls2d/test/app.yaml +++ b/L1/dockerfiles/sls2d/test/app.yaml @@ -1,16 +1,20 @@ -name: sls2d.app.process -label: csst-sls2d +name: msc-l1-sls2d.app.process +label: 无缝光谱2D +comment: 主巡天无缝光谱2D一级流水线 cluster: csst parameters: initial_status: RUNNING jobs: - sls2d: + msc-l1-sls2d: label: 无缝光谱2D - base_image: csst/sls2d - schedule_mode: HEAD + base_image: cnic/csst-msc-l1-sls2d + # schedule_mode: HEAD variables: - # repeated: yes + always_running: yes + reserved_on_exit: yes + # DFS入库错误重试3次 + retry_rules: "['202:3']" output_text_size: 100000 text_tranc_mode: TAIL locale_mode: NONE @@ -18,11 +22,11 @@ jobs: key_group_regex: ^(.{6})(.{3})$ key_group_seq: 1,2 paths: - - ${AUX_DIR}:/L1Pipeline/aux + - ${AUX_DIR}:/pipeline/aux - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds - ${DFS_ROOT}:/dfsroot # sink_jobs: - # hosts: + # hosts: # - h0:1 # - c0:3 # - c1:3 diff --git a/L1/dockerfiles/sls2d/test/sls2d.yaml b/L1/dockerfiles/sls2d/test/sls2d.yaml deleted file mode 100644 index 6e43ae8bd1d1c152952e2f366956276f7ac4e32d..0000000000000000000000000000000000000000 --- a/L1/dockerfiles/sls2d/test/sls2d.yaml +++ /dev/null @@ -1,25 +0,0 @@ -name: sls2d.app.process -label: csst-sls2d -cluster: local -parameters: - initial_status: RUNNING - -jobs: - sls2d: - label: 无缝光谱 - base_image: csst/sls2d - schedule_mode: HEAD - paths: - - ${AUX_DIR}:/L1Pipeline/aux - - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds - - ${DFS_ROOT}:/dfsroot - # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE% - # sink_jobs: - #hosts: - # - h0:1 - # - c0:2 - # - c1:2 - # - c2:2 - - - diff --git a/L1/dockerfiles/sls2d/test/zjs.env b/L1/dockerfiles/sls2d/test/zjs.env index 1f64fd8a8acb5dea29a81bb0525feb264253fc2d..133c12b2c61c0a45695cc7df6819bee5204c95a0 100644 --- a/L1/dockerfiles/sls2d/test/zjs.env +++ b/L1/dockerfiles/sls2d/test/zjs.env @@ -1,3 +1,7 @@ -AUX_DIR=/sharewcl/L1Pipeline/aux -CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr -DFS_ROOT=/sharewcl/dfs +# AUX_DIR=/sharewcl/pipeline/aux +# CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr +# DFS_ROOT=/sharewcl/dfs + +AUX_DIR=/share/pipeline/aux +CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr +DFS_ROOT=/share/dfs \ No newline at end of file