From 1da6e3273020acccb897b57111c5f322b1773e57 Mon Sep 17 00:00:00 2001
From: zxl <zhangxiaoli@cnic.cn>
Date: Fri, 7 Jul 2023 17:04:41 +0800
Subject: [PATCH] Modify image's name and python running parameters.
---
L1/dockerfiles/mbi/Dockerfile | 4 ++--
L1/dockerfiles/mbi/Makefile | 12 +++++++-----
L1/dockerfiles/mbi/run.sh | 29 ++++++++++++++++++++++------
L1/dockerfiles/mbi/test/add_task.sh | 2 +-
L1/dockerfiles/mbi/test/app.yaml | 28 +++++++++++++++------------
L1/dockerfiles/mbi/test/zjs.env | 6 +++++-
L1/dockerfiles/sls2d/Dockerfile | 4 ++--
L1/dockerfiles/sls2d/Makefile | 12 ++++++------
L1/dockerfiles/sls2d/run.sh | 23 +++++++++++++++++-----
L1/dockerfiles/sls2d/test/Makefile | 2 +-
L1/dockerfiles/sls2d/test/app.yaml | 20 +++++++++++--------
L1/dockerfiles/sls2d/test/sls2d.yaml | 25 ------------------------
L1/dockerfiles/sls2d/test/zjs.env | 10 +++++++---
13 files changed, 100 insertions(+), 77 deletions(-)
delete mode 100644 L1/dockerfiles/sls2d/test/sls2d.yaml
diff --git a/L1/dockerfiles/mbi/Dockerfile b/L1/dockerfiles/mbi/Dockerfile
index 5a0245a..c8eae21 100644
--- a/L1/dockerfiles/mbi/Dockerfile
+++ b/L1/dockerfiles/mbi/Dockerfile
@@ -1,8 +1,8 @@
-FROM csst-l1ppl:latest
+FROM csst/csst-msc-l1-mbi:latest
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
RUN mkdir /work/
WORKDIR /work/
diff --git a/L1/dockerfiles/mbi/Makefile b/L1/dockerfiles/mbi/Makefile
index d3ab275..4f26f97 100644
--- a/L1/dockerfiles/mbi/Makefile
+++ b/L1/dockerfiles/mbi/Makefile
@@ -1,14 +1,14 @@
-IMAGE_NAME:=csst/mbi
+IMAGE_NAME:=cnic/csst-msc-l1-mbi
build:
docker build --network=host -t $(IMAGE_NAME) .
dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
- docker save $(IMAGE_NAME) > /share/tmp/mbi.tar
- ssh c0 docker load -i /share/tmp/mbi.tar
- ssh c1 docker load -i /share/tmp/mbi.tar
- ssh c2 docker load -i /share/tmp/mbi.tar
+ docker save $(IMAGE_NAME) > /share/tmp/msc-l1-mbi.tar
+ ssh c0 docker load -i /share/tmp/msc-l1-mbi.tar
+ ssh c1 docker load -i /share/tmp/msc-l1-mbi.tar
+ ssh c2 docker load -i /share/tmp/msc-l1-mbi.tar
push:
docker push $(IMAGE_NAME)
@@ -16,3 +16,5 @@ run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
+scp:
+ scp -r ./ csst-zjs:/root/csst/mbi/
\ No newline at end of file
diff --git a/L1/dockerfiles/mbi/run.sh b/L1/dockerfiles/mbi/run.sh
index cc03649..80a45c2 100644
--- a/L1/dockerfiles/mbi/run.sh
+++ b/L1/dockerfiles/mbi/run.sh
@@ -8,18 +8,35 @@
# obsid 10160000000 - 10160000136
obsid=$1
-#python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $obsid > /work/stdout 2> /work/stderr
-l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry
+# l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry
+python /pipeline/src/run.py \
+ --obs-id=$obsid \
+ --device=cpu \
+ --n-jobs=18 \
+ --n-jobs-gpu=9 \
+ --clean-l0 \
+ --clean-l1 \
+ --dfs-node=pml \
+ --ver-sim=C6.2 \
+ --no-astrometry \
+ --no-photometry
exit_code=$?
echo "=====csst-l1ppl.log====" > /work/user-file.txt
-cat /L1Pipeline/L1/csst-l1ppl.log >> /work/user-file.txt
+cat /pipeline/output/csst-l1ppl.log >> /work/user-file.txt
+
+
echo "======csst-l1mod.log======" >> /work/user-file.txt
-cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 >> /work/user-file.txt
+cat /pipeline/output/csst-l1mod.log|tail -n 100 >> /work/user-file.txt
+
-rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
+timefile=/pipeline/output/timestamps.txt
+if test -f "$timefile"; then
+ echo "$timefile exist"
+ mv /pipeline/output/timestamps.txt /work/timestamps.txt
+fi
-mv /tmp/timestamps.txt /work/timestamps.txt
+rm -rf /pipeline/input/* /pipeline/output/*
echo finish mbi.
diff --git a/L1/dockerfiles/mbi/test/add_task.sh b/L1/dockerfiles/mbi/test/add_task.sh
index 15a3411..7485167 100644
--- a/L1/dockerfiles/mbi/test/add_task.sh
+++ b/L1/dockerfiles/mbi/test/add_task.sh
@@ -8,7 +8,7 @@ PGPORT=5432
job_id=$1
-for m in {10160000000..10160000001}; do
+for m in {10160000066..10160000068}; do
echo $m
docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
-c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
diff --git a/L1/dockerfiles/mbi/test/app.yaml b/L1/dockerfiles/mbi/test/app.yaml
index 3e37ae4..51d4be8 100644
--- a/L1/dockerfiles/mbi/test/app.yaml
+++ b/L1/dockerfiles/mbi/test/app.yaml
@@ -1,32 +1,36 @@
-name: mbi.app.process
-label: csst-mbi
-cluster: local
+name: msc-l1-mbi.app.process
+label: 澶氳壊鎴愬儚
+comment: 涓诲贰澶╁鑹叉垚鍍忎竴绾ф祦姘寸嚎
+cluster: csst
parameters:
initial_status: RUNNING
jobs:
- mbi:
- base_image: csst/mbi
- schedule_mode: HEAD
+ msc-l1-mbi:
+ label: 澶氳壊鎴愬儚
+ base_image: cnic/csst-msc-l1-mbi
+ # schedule_mode: HEAD
variables:
- # repeated: yes
+ # always_running: yes
+ # reserved_on_exit: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
parameters:
+ # start_message: 10160000068
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
paths:
- - ${AUX_DIR}:/L1Pipeline/aux
+ - ${AUX_DIR}:/pipeline/aux
- ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
- ${DFS_ROOT}:/dfsroot
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
# sink_jobs:
hosts:
- - h0:1
- - c0:2
- - c1:2
- - c2:2
+ # - h0:1
+ - c0:1
+ - c1:1
+ - c2:1
diff --git a/L1/dockerfiles/mbi/test/zjs.env b/L1/dockerfiles/mbi/test/zjs.env
index 1f64fd8..91bf385 100644
--- a/L1/dockerfiles/mbi/test/zjs.env
+++ b/L1/dockerfiles/mbi/test/zjs.env
@@ -1,3 +1,7 @@
-AUX_DIR=/sharewcl/L1Pipeline/aux
+AUX_DIR=/sharewcl/pipeline/aux
CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
DFS_ROOT=/sharewcl/dfs
+
+# AUX_DIR=/share/pipeline/aux
+# CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+# DFS_ROOT=/share/dfs
\ No newline at end of file
diff --git a/L1/dockerfiles/sls2d/Dockerfile b/L1/dockerfiles/sls2d/Dockerfile
index 822654c..7c1450a 100644
--- a/L1/dockerfiles/sls2d/Dockerfile
+++ b/L1/dockerfiles/sls2d/Dockerfile
@@ -1,8 +1,8 @@
-FROM csst-l1ppl:latest
+FROM csst/csst-msc-l1-sls2d:latest
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
RUN mkdir /work/
WORKDIR /work/
diff --git a/L1/dockerfiles/sls2d/Makefile b/L1/dockerfiles/sls2d/Makefile
index 6dd57ca..e6571b6 100644
--- a/L1/dockerfiles/sls2d/Makefile
+++ b/L1/dockerfiles/sls2d/Makefile
@@ -1,14 +1,14 @@
-IMAGE_NAME:=csst/sls2d
+IMAGE_NAME:=cnic/csst-msc-l1-sls2d
build:
docker build --network=host -t $(IMAGE_NAME) .
-dist: build
+dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
- docker save $(IMAGE_NAME) > /share/tmp/sls2d.tar
- ssh c0 docker load -i /share/tmp/sls2d.tar
- ssh c1 docker load -i /share/tmp/sls2d.tar
- ssh c2 docker load -i /share/tmp/sls2d.tar
+ docker save $(IMAGE_NAME) > /share/tmp/msc-l1-sls2d.tar
+ ssh c0 docker load -i /share/tmp/msc-l1-sls2d.tar
+ ssh c1 docker load -i /share/tmp/msc-l1-sls2d.tar
+ ssh c2 docker load -i /share/tmp/msc-l1-sls2d.tar
push:
docker push $(IMAGE_NAME)
run:
diff --git a/L1/dockerfiles/sls2d/run.sh b/L1/dockerfiles/sls2d/run.sh
index e4530b7..353baa3 100644
--- a/L1/dockerfiles/sls2d/run.sh
+++ b/L1/dockerfiles/sls2d/run.sh
@@ -9,17 +9,30 @@
# obsid 10160000000 - 10160000136
obsid=$1
#python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $obsid > /work/stdout 2> /work/stderr
-l1-sls2d --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry > /work/stdout 2> /work/stderr
+# l1-sls2d --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry > /work/stdout 2> /work/stderr
+python /pipeline/src/run.py \
+ --obs-id=$obsid \
+ --device=cpu \
+ --n-jobs=12 \
+ --n-jobs-gpu=6 \
+ --clean-l0 \
+ --clean-l1 \
+ --dfs-node=pml \
+ --ver-sim=C6.2
exit_code=$?
echo "=====csst-l1ppl.log====" > /work/user-file.txt
-cat /L1Pipeline/L1/csst-l1ppl.log >> /work/user-file.txt
+cat /pipeline/output/csst-l1ppl.log >> /work/user-file.txt
echo "======csst-l1mod.log======" >> /work/user-file.txt
-cat /L1Pipeline/L1/csst-l1mod.log|tail -n 100 >> /work/user-file.txt
+cat /pipeline/output/csst-l1mod.log|tail -n 100 >> /work/user-file.txt
-rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
+timefile=/pipeline/output/timestamps.txt
+if test -f "$timefile"; then
+ echo "$timefile exist"
+ mv /pipeline/output/timestamps.txt /work/timestamps.txt
+fi
-mv /tmp/timestamps.txt /work/timestamps.txt
+rm -rf /pipeline/input/* /pipeline/output/*
echo finish sls2d.
diff --git a/L1/dockerfiles/sls2d/test/Makefile b/L1/dockerfiles/sls2d/test/Makefile
index 534943d..0e709ce 100644
--- a/L1/dockerfiles/sls2d/test/Makefile
+++ b/L1/dockerfiles/sls2d/test/Makefile
@@ -1,4 +1,4 @@
-CLUSTER=local
+CLUSTER=csst
# all: reset build
diff --git a/L1/dockerfiles/sls2d/test/app.yaml b/L1/dockerfiles/sls2d/test/app.yaml
index 1dbad60..d7fdb86 100644
--- a/L1/dockerfiles/sls2d/test/app.yaml
+++ b/L1/dockerfiles/sls2d/test/app.yaml
@@ -1,16 +1,20 @@
-name: sls2d.app.process
-label: csst-sls2d
+name: msc-l1-sls2d.app.process
+label: 鏃犵紳鍏夎氨2D
+comment: 涓诲贰澶╂棤缂濆厜璋�2D涓€绾ф祦姘寸嚎
cluster: csst
parameters:
initial_status: RUNNING
jobs:
- sls2d:
+ msc-l1-sls2d:
label: 鏃犵紳鍏夎氨2D
- base_image: csst/sls2d
- schedule_mode: HEAD
+ base_image: cnic/csst-msc-l1-sls2d
+ # schedule_mode: HEAD
variables:
- # repeated: yes
+ always_running: yes
+ reserved_on_exit: yes
+ # DFS鍏ュ簱閿欒閲嶈瘯3娆�
+ retry_rules: "['202:3']"
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
@@ -18,11 +22,11 @@ jobs:
key_group_regex: ^(.{6})(.{3})$
key_group_seq: 1,2
paths:
- - ${AUX_DIR}:/L1Pipeline/aux
+ - ${AUX_DIR}:/pipeline/aux
- ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
- ${DFS_ROOT}:/dfsroot
# sink_jobs:
- # hosts:
+ # hosts:
# - h0:1
# - c0:3
# - c1:3
diff --git a/L1/dockerfiles/sls2d/test/sls2d.yaml b/L1/dockerfiles/sls2d/test/sls2d.yaml
deleted file mode 100644
index 6e43ae8..0000000
--- a/L1/dockerfiles/sls2d/test/sls2d.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-name: sls2d.app.process
-label: csst-sls2d
-cluster: local
-parameters:
- initial_status: RUNNING
-
-jobs:
- sls2d:
- label: 鏃犵紳鍏夎氨
- base_image: csst/sls2d
- schedule_mode: HEAD
- paths:
- - ${AUX_DIR}:/L1Pipeline/aux
- - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
- - ${DFS_ROOT}:/dfsroot
- # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
- # sink_jobs:
- #hosts:
- # - h0:1
- # - c0:2
- # - c1:2
- # - c2:2
-
-
-
diff --git a/L1/dockerfiles/sls2d/test/zjs.env b/L1/dockerfiles/sls2d/test/zjs.env
index 1f64fd8..133c12b 100644
--- a/L1/dockerfiles/sls2d/test/zjs.env
+++ b/L1/dockerfiles/sls2d/test/zjs.env
@@ -1,3 +1,7 @@
-AUX_DIR=/sharewcl/L1Pipeline/aux
-CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
-DFS_ROOT=/sharewcl/dfs
+# AUX_DIR=/sharewcl/pipeline/aux
+# CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+# DFS_ROOT=/sharewcl/dfs
+
+AUX_DIR=/share/pipeline/aux
+CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+DFS_ROOT=/share/dfs
\ No newline at end of file
--
GitLab