Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
csst
Scalebox Apps
Commits
1da6e327
Commit
1da6e327
authored
1 year ago
by
Zhang Xiaoli
Browse files
Options
Download
Email Patches
Plain Diff
Modify image's name and python running parameters.
parent
f485ee28
Changes
13
Show whitespace changes
Inline
Side-by-side
Showing
13 changed files
with
100 additions
and
77 deletions
+100
-77
L1/dockerfiles/mbi/Dockerfile
L1/dockerfiles/mbi/Dockerfile
+2
-2
L1/dockerfiles/mbi/Makefile
L1/dockerfiles/mbi/Makefile
+7
-5
L1/dockerfiles/mbi/run.sh
L1/dockerfiles/mbi/run.sh
+23
-6
L1/dockerfiles/mbi/test/add_task.sh
L1/dockerfiles/mbi/test/add_task.sh
+1
-1
L1/dockerfiles/mbi/test/app.yaml
L1/dockerfiles/mbi/test/app.yaml
+16
-12
L1/dockerfiles/mbi/test/zjs.env
L1/dockerfiles/mbi/test/zjs.env
+5
-1
L1/dockerfiles/sls2d/Dockerfile
L1/dockerfiles/sls2d/Dockerfile
+2
-2
L1/dockerfiles/sls2d/Makefile
L1/dockerfiles/sls2d/Makefile
+6
-6
L1/dockerfiles/sls2d/run.sh
L1/dockerfiles/sls2d/run.sh
+18
-5
L1/dockerfiles/sls2d/test/Makefile
L1/dockerfiles/sls2d/test/Makefile
+1
-1
L1/dockerfiles/sls2d/test/app.yaml
L1/dockerfiles/sls2d/test/app.yaml
+12
-8
L1/dockerfiles/sls2d/test/sls2d.yaml
L1/dockerfiles/sls2d/test/sls2d.yaml
+0
-25
L1/dockerfiles/sls2d/test/zjs.env
L1/dockerfiles/sls2d/test/zjs.env
+7
-3
No files found.
L1/dockerfiles/mbi/Dockerfile
View file @
1da6e327
FROM
csst
-l1ppl
:latest
FROM
csst
/csst-msc-l1-mbi
:latest
COPY
run.sh /app/bin/
RUN
chmod
+x /app/bin/run.sh
COPY
--from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
COPY
--from=hub.cstcloud.cn/scalebox/base
:0.9
/usr/local/sbin /usr/local/sbin
RUN
mkdir
/work/
WORKDIR
/work/
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/mbi/Makefile
View file @
1da6e327
IMAGE_NAME
:=
c
sst/
mbi
IMAGE_NAME
:=
c
nic/csst-msc-l1-
mbi
build
:
docker build
--network
=
host
-t
$(IMAGE_NAME)
.
dist
:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker
save
$(IMAGE_NAME)
>
/share/tmp/mbi.tar
ssh
c0
docker
load
-i
/share/tmp/mbi.tar
ssh
c1
docker
load
-i
/share/tmp/mbi.tar
ssh
c2
docker
load
-i
/share/tmp/mbi.tar
docker
save
$(IMAGE_NAME)
>
/share/tmp/
msc-l1-
mbi.tar
ssh
c0
docker
load
-i
/share/tmp/
msc-l1-
mbi.tar
ssh
c1
docker
load
-i
/share/tmp/
msc-l1-
mbi.tar
ssh
c2
docker
load
-i
/share/tmp/
msc-l1-
mbi.tar
push
:
docker push
$(IMAGE_NAME)
...
...
@@ -16,3 +16,5 @@ run:
docker run
-it
--entrypoint
bash
$(IMAGE_NAME)
down
:
docker stop
$(IMAGE_NAME)
scp
:
scp
-r
./ csst-zjs:/root/csst/mbi/
\ No newline at end of file
This diff is collapsed.
Click to expand it.
L1/dockerfiles/mbi/run.sh
View file @
1da6e327
...
...
@@ -8,18 +8,35 @@
# obsid 10160000000 - 10160000136
obsid
=
$1
#python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $obsid > /work/stdout 2> /work/stderr
l1-mbi
--obs-id
=
$obsid
--device
=
cpu
--n-jobs
=
18
--n-jobs-gpu
=
18
--clean-l0
--clean-l1
--cleanup
--dfs-node
=
pml
--ver-sim
=
C6.2
--no-photometry
--no-astrometry
# l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry
python /pipeline/src/run.py
\
--obs-id
=
$obsid
\
--device
=
cpu
\
--n-jobs
=
18
\
--n-jobs-gpu
=
9
\
--clean-l0
\
--clean-l1
\
--dfs-node
=
pml
\
--ver-sim
=
C6.2
\
--no-astrometry
\
--no-photometry
exit_code
=
$?
echo
"=====csst-l1ppl.log===="
>
/work/user-file.txt
cat
/L1Pipeline/L1/csst-l1ppl.log
>>
/work/user-file.txt
cat
/pipeline/output/csst-l1ppl.log
>>
/work/user-file.txt
echo
"======csst-l1mod.log======"
>>
/work/user-file.txt
cat
/L1Pipeline/L1/csst-l1mod.log|tail
-n
100
>>
/work/user-file.txt
cat
/pipeline/output/csst-l1mod.log|tail
-n
100
>>
/work/user-file.txt
rm
-rf
/L1Pipeline/L0/
*
/L1Pipeline/L1/
*
timefile
=
/pipeline/output/timestamps.txt
if
test
-f
"
$timefile
"
;
then
echo
"
$timefile
exist"
mv
/pipeline/output/timestamps.txt /work/timestamps.txt
fi
mv
/tmp/timestamps.txt /work/timestamps.txt
rm
-rf
/pipeline/input/
*
/pipeline/output/
*
echo
finish mbi.
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/mbi/test/add_task.sh
View file @
1da6e327
...
...
@@ -8,7 +8,7 @@ PGPORT=5432
job_id
=
$1
for
m
in
{
101600000
00
..101600000
01
}
;
do
for
m
in
{
101600000
66
..101600000
68
}
;
do
echo
$m
docker
exec
-t
database psql
-U
${
POSTGRES_USER
}
-h
${
POSTGRES_HOST
}
-d
${
POSTGRES_DB
}
-p
${
PGPORT
}
\
-c
"INSERT INTO t_task(job,key_message) VALUES(
${
job_id
}
,'
${
m
}
')"
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/mbi/test/app.yaml
View file @
1da6e327
name
:
mbi.app.process
label
:
csst-mbi
cluster
:
local
name
:
msc-l1-mbi.app.process
label
:
多色成像
comment
:
主巡天多色成像一级流水线
cluster
:
csst
parameters
:
initial_status
:
RUNNING
jobs
:
mbi
:
base_image
:
csst/mbi
schedule_mode
:
HEAD
msc-l1-mbi
:
label
:
多色成像
base_image
:
cnic/csst-msc-l1-mbi
# schedule_mode: HEAD
variables
:
# repeated: yes
# always_running: yes
# reserved_on_exit: yes
output_text_size
:
100000
text_tranc_mode
:
TAIL
locale_mode
:
NONE
parameters
:
# start_message: 10160000068
key_group_regex
:
^(.{6})(.{3})$
key_group_seq
:
1,2
paths
:
-
${AUX_DIR}:/
L1P
ipeline/aux
-
${AUX_DIR}:/
p
ipeline/aux
-
${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-
${DFS_ROOT}:/dfsroot
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
# sink_jobs:
hosts
:
-
h0:1
-
c0:
2
-
c1:
2
-
c2:
2
#
- h0:1
-
c0:
1
-
c1:
1
-
c2:
1
This diff is collapsed.
Click to expand it.
L1/dockerfiles/mbi/test/zjs.env
View file @
1da6e327
AUX_DIR=/sharewcl/
L1P
ipeline/aux
AUX_DIR=/sharewcl/
p
ipeline/aux
CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
DFS_ROOT=/sharewcl/dfs
# AUX_DIR=/share/pipeline/aux
# CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
# DFS_ROOT=/share/dfs
\ No newline at end of file
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/Dockerfile
View file @
1da6e327
FROM
csst
-l1ppl
:latest
FROM
csst
/csst-msc-l1-sls2d
:latest
COPY
run.sh /app/bin/
RUN
chmod
+x /app/bin/run.sh
COPY
--from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
COPY
--from=hub.cstcloud.cn/scalebox/base
:0.9
/usr/local/sbin /usr/local/sbin
RUN
mkdir
/work/
WORKDIR
/work/
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/Makefile
View file @
1da6e327
IMAGE_NAME
:=
c
sst/
sls2d
IMAGE_NAME
:=
c
nic/csst-msc-l1-
sls2d
build
:
docker build
--network
=
host
-t
$(IMAGE_NAME)
.
dist
:
build
dist
:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker
save
$(IMAGE_NAME)
>
/share/tmp/sls2d.tar
ssh
c0
docker
load
-i
/share/tmp/sls2d.tar
ssh
c1
docker
load
-i
/share/tmp/sls2d.tar
ssh
c2
docker
load
-i
/share/tmp/sls2d.tar
docker
save
$(IMAGE_NAME)
>
/share/tmp/
msc-l1-
sls2d.tar
ssh
c0
docker
load
-i
/share/tmp/
msc-l1-
sls2d.tar
ssh
c1
docker
load
-i
/share/tmp/
msc-l1-
sls2d.tar
ssh
c2
docker
load
-i
/share/tmp/
msc-l1-
sls2d.tar
push
:
docker push
$(IMAGE_NAME)
run
:
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/run.sh
View file @
1da6e327
...
...
@@ -9,17 +9,30 @@
# obsid 10160000000 - 10160000136
obsid
=
$1
#python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $obsid > /work/stdout 2> /work/stderr
l1-sls2d
--obs-id
=
$obsid
--device
=
cpu
--n-jobs
=
18
--n-jobs-gpu
=
18
--clean-l0
--clean-l1
--cleanup
--dfs-node
=
pml
--ver-sim
=
C6.2
--no-photometry
--no-astrometry
>
/work/stdout 2> /work/stderr
# l1-sls2d --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry > /work/stdout 2> /work/stderr
python /pipeline/src/run.py
\
--obs-id
=
$obsid
\
--device
=
cpu
\
--n-jobs
=
12
\
--n-jobs-gpu
=
6
\
--clean-l0
\
--clean-l1
\
--dfs-node
=
pml
\
--ver-sim
=
C6.2
exit_code
=
$?
echo
"=====csst-l1ppl.log===="
>
/work/user-file.txt
cat
/
L1P
ipeline/
L1
/csst-l1ppl.log
>>
/work/user-file.txt
cat
/
p
ipeline/
output
/csst-l1ppl.log
>>
/work/user-file.txt
echo
"======csst-l1mod.log======"
>>
/work/user-file.txt
cat
/
L1P
ipeline/
L1
/csst-l1mod.log|tail
-n
100
>>
/work/user-file.txt
cat
/
p
ipeline/
output
/csst-l1mod.log|tail
-n
100
>>
/work/user-file.txt
rm
-rf
/L1Pipeline/L0/
*
/L1Pipeline/L1/
*
timefile
=
/pipeline/output/timestamps.txt
if
test
-f
"
$timefile
"
;
then
echo
"
$timefile
exist"
mv
/pipeline/output/timestamps.txt /work/timestamps.txt
fi
mv
/tmp/timestamps.txt /work/timestamps.txt
rm
-rf
/pipeline/input/
*
/pipeline/output/
*
echo
finish sls2d.
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/test/Makefile
View file @
1da6e327
CLUSTER
=
local
CLUSTER
=
csst
# all: reset build
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/test/app.yaml
View file @
1da6e327
name
:
sls2d.app.process
label
:
csst-sls2d
name
:
msc-l1-sls2d.app.process
label
:
无缝光谱2D
comment
:
主巡天无缝光谱2D一级流水线
cluster
:
csst
parameters
:
initial_status
:
RUNNING
jobs
:
sls2d
:
msc-l1-
sls2d
:
label
:
无缝光谱2D
base_image
:
c
sst/
sls2d
schedule_mode
:
HEAD
base_image
:
c
nic/csst-msc-l1-
sls2d
#
schedule_mode: HEAD
variables
:
# repeated: yes
always_running
:
yes
reserved_on_exit
:
yes
# DFS入库错误重试3次
retry_rules
:
"
['202:3']"
output_text_size
:
100000
text_tranc_mode
:
TAIL
locale_mode
:
NONE
...
...
@@ -18,7 +22,7 @@ jobs:
key_group_regex
:
^(.{6})(.{3})$
key_group_seq
:
1,2
paths
:
-
${AUX_DIR}:/
L1P
ipeline/aux
-
${AUX_DIR}:/
p
ipeline/aux
-
${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-
${DFS_ROOT}:/dfsroot
# sink_jobs:
...
...
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/test/sls2d.yaml
deleted
100644 → 0
View file @
f485ee28
name
:
sls2d.app.process
label
:
csst-sls2d
cluster
:
local
parameters
:
initial_status
:
RUNNING
jobs
:
sls2d
:
label
:
无缝光谱
base_image
:
csst/sls2d
schedule_mode
:
HEAD
paths
:
-
${AUX_DIR}:/L1Pipeline/aux
-
${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-
${DFS_ROOT}:/dfsroot
# command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
# sink_jobs:
#hosts:
# - h0:1
# - c0:2
# - c1:2
# - c2:2
This diff is collapsed.
Click to expand it.
L1/dockerfiles/sls2d/test/zjs.env
View file @
1da6e327
AUX_DIR=/sharewcl/L1Pipeline/aux
CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
DFS_ROOT=/sharewcl/dfs
# AUX_DIR=/sharewcl/pipeline/aux
# CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
# DFS_ROOT=/sharewcl/dfs
AUX_DIR=/share/pipeline/aux
CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
DFS_ROOT=/share/dfs
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment