Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
csst
Scalebox Apps
Commits
0a669f09
Commit
0a669f09
authored
Aug 11, 2023
by
Zhang Xiaoli
Browse files
add cube container, finish unit test.
parent
fb9004f3
Changes
10
Show whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
243 additions
and
0 deletions
+243
-0
L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile
L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile
+14
-0
L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile
L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile
+22
-0
L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh
L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh
+34
-0
L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile
L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile
+11
-0
L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile
L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile
+22
-0
L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh
L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh
+19
-0
L1/dockerfiles/csst-ifs-l1-cube/test/Makefile
L1/dockerfiles/csst-ifs-l1-cube/test/Makefile
+18
-0
L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml
L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml
+69
-0
L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env
L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env
+17
-0
L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env
L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env
+17
-0
No files found.
L1/dockerfiles/csst-ifs-l1-cube/adm/Dockerfile
0 → 100644
View file @
0a669f09
FROM
csst/csst-ifs-l1-cube-adm:latest
LABEL
maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
USER
root
COPY
run.sh /app/bin/
RUN
chmod
+x /app/bin/run.sh
COPY
--from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin/
RUN
mkdir
-p
/work
&&
echo
"PATH=/app/bin:
\$
{PATH}"
>>
/root/.bashrc
#WORKDIR /work/
WORKDIR
/L1Pipeline/workspace
ENTRYPOINT
[ "goagent" ]
L1/dockerfiles/csst-ifs-l1-cube/adm/Makefile
0 → 100644
View file @
0a669f09
IMAGE_NAME
:=
cnic/csst-ifs-l1-cube-adm
build
:
docker build
--network
=
host
-t
$(IMAGE_NAME)
.
dist
:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker
save
$(IMAGE_NAME)
>
/share/tmp/ifs-l1-cube-adm.tar
ssh
c0
docker
load
-i
/share/tmp/ifs-l1-cube-adm.tar
ssh
c1
docker
load
-i
/share/tmp/ifs-l1-cube-adm.tar
ssh
c2
docker
load
-i
/share/tmp/ifs-l1-cube-adm.tar
push
:
docker push
$(IMAGE_NAME)
run
:
docker run
-it
--entrypoint
bash
$(IMAGE_NAME)
down
:
docker stop
$(IMAGE_NAME)
scp
:
scp
-r
./ csst-zjs:/root/csst/csst-ifs-l1-cube/adm/
save
:
docker save
$(IMAGE_NAME)
>
/sharewcl/temp/ifs-l1-cube-adm.tar
\ No newline at end of file
L1/dockerfiles/csst-ifs-l1-cube/adm/run.sh
0 → 100644
View file @
0a669f09
#!/bin/bash
# pipeline=/pipeline/output/pipeline.log
# if test -f "$pipeline"; then
# echo "=====pipeline.log====" > /work/user-file.txt
# cat /pipeline/output/pipeline.log >> /work/user-file.txt
# fi
# timefile=/pipeline/output/timestamps.txt
# if test -f "$timefile"; then
# echo "$timefile exist"
# mv /pipeline/output/timestamps.txt /work/timestamps.txt
# fi
# obsid="30100000001"
python l1_pipeline_script_ifs_cube_adm.py
>
./stdout
exit_code
=
$?
mkdir
-p
/work
# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
# mv /tmp/timestamps.txt /work/timestamps.txt
cat
./stdout
mv
/L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt
echo
finish csst-ifs-l1-cube-adm.
exit
$exit_code
L1/dockerfiles/csst-ifs-l1-cube/exe/Dockerfile
0 → 100644
View file @
0a669f09
FROM
csst/csst-ifs-l1-cube-exe:latest
USER
root
COPY
run.sh /app/bin/
RUN
chmod
+x /app/bin/run.sh
COPY
--from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/
RUN
mkdir
-p
/work
&&
echo
"PATH=/app/bin:
\$
{PATH}"
>>
/root/.bashrc
WORKDIR
/L1Pipeline/workspace
ENTRYPOINT
[ "goagent" ]
L1/dockerfiles/csst-ifs-l1-cube/exe/Makefile
0 → 100644
View file @
0a669f09
IMAGE_NAME
:=
cnic/csst-ifs-l1-cube-exe
build
:
docker build
--network
=
host
-t
$(IMAGE_NAME)
.
dist
:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker
save
$(IMAGE_NAME)
>
/share/tmp/ifs-l1-cube-exe.tar
ssh
c0
docker
load
-i
/share/tmp/ifs-l1-cube-exe.tar
ssh
c1
docker
load
-i
/share/tmp/ifs-l1-cube-exe.tar
ssh
c2
docker
load
-i
/share/tmp/ifs-l1-cube-exe.tar
push
:
docker push
$(IMAGE_NAME)
run
:
docker run
-it
--entrypoint
bash
$(IMAGE_NAME)
down
:
docker stop
$(IMAGE_NAME)
scp
:
scp
-r
./ csst-zjs:/root/csst/ifs-l1-cube/exe/
save
:
docker save
$(IMAGE_NAME)
>
/sharewcl/temp/ifs-l1-cube-exe.tar
\ No newline at end of file
L1/dockerfiles/csst-ifs-l1-cube/exe/run.sh
0 → 100644
View file @
0a669f09
#!/bin/bash
echo
input:
$1
*
python l1_pipeline_script_ifs_cube_simdata.py
$1
>
./stdout
exit_code
=
$?
mkdir
-p
/work
# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
# mv /tmp/timestamps.txt /work/timestamps.txt
echo
finish csst-ifs-l1-cube-exe.
exit
$exit_code
L1/dockerfiles/csst-ifs-l1-cube/test/Makefile
0 → 100644
View file @
0a669f09
CLUSTER
=
local
all
:
reset build
run
:
scalebox app create
--env-file
zjs-ztm.env
reset
:
cd
${HOME}
/docker-scalebox/clusters/
$(CLUSTER)
&&
make
&&
cd
-
down
:
make
-C
${HOME}
/docker-scalebox/clusters/
$(CLUSTER)
down
list
:
make
-C
${HOME}
/docker-scalebox/clusters/
$(CLUSTER)
list
scp
:
scp
-r
../ csst-zjs:/root/csst/csst-ifs-l1-cube/
\ No newline at end of file
L1/dockerfiles/csst-ifs-l1-cube/test/app.yaml
0 → 100644
View file @
0a669f09
name
:
ifs-l1-cube.app.process
label
:
多次曝光合并
comment
:
积分场光谱仪多次曝光合并一级流水线
cluster
:
local
parameters
:
initial_status
:
RUNNING
jobs
:
csst-ifs-l1-cube-adm
:
label
:
多次曝光合并管理模块
base_image
:
cnic/csst-ifs-l1-cube-adm:latest
schedule_mode
:
HEAD
variables
:
# always_running: yes
output_text_size
:
100000
# DFS入库错误重试3次
retry_rules
:
"
['202:3']"
text_tranc_mode
:
TAIL
locale_mode
:
NONE
# grpc_server: 172.24.23.6:50051
parameters
:
start_message
:
ANY
environments
:
-
CSST_DFS_API_MODE=cluster
-
CSST_DFS_GATEWAY=172.24.27.2:30880
-
CSST_DFS_APP_ID=test
-
CSST_DFS_APP_TOKEN=test
paths
:
# change /share to /sharewcl in PML production environment
-
${CSST_AUX_DIR}:/pipeline/aux
-
${CSST_DFS_ROOT}:/dfsroot
-
${CSST_CRDS_ROOT}:/crdsroot
sink_jobs
:
-
csst-ifs-l1-cube-exe
# command: >
# bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt"
# hosts:
# - h0:1
# - c0:1
# - c1:1
# - c2:1
csst-ifs-l1-cube-exe
:
label
:
多次曝光合并执行模块
base_image
:
cnic/csst-ifs-l1-cube-exe:latest
schedule_mode
:
HEAD
variables
:
# always_running: yes
output_text_size
:
100000
# DFS入库错误重试3次
retry_rules
:
"
['202:3']"
text_tranc_mode
:
TAIL
locale_mode
:
NONE
# grpc_server: 172.24.23.6:50051
environments
:
-
CSST_DFS_API_MODE=cluster
-
CSST_DFS_GATEWAY=172.24.27.2:30880
-
CSST_DFS_APP_ID=test
-
CSST_DFS_APP_TOKEN=test
paths
:
# change /share to /sharewcl in PML production environment
-
${AUX_DIR}:/L1Pipeline/aux
-
${CRDS_DIR}:/L1Pipeline/crds
-
${DFS_ROOT}:/L1Pipeline/dfs
# command: >
# python /pipeline/src/run.py {\"brick_id\":1}
# hosts:
# - h0:1
# - c0:1
# - c1:1
# - c2:2
\ No newline at end of file
L1/dockerfiles/csst-ifs-l1-cube/test/zjs-ztm.env
0 → 100644
View file @
0a669f09
# DFS
CSST_DFS_API_MODE=cluster
CSST_DFS_GATEWAY=172.24.27.2:30880
CSST_DFS_APP_ID=test
CSST_DFS_APP_TOKEN=test
# CRDS
CRDS_SERVER_URL=http://172.24.27.2:29000
# VOLUMES ==================
CSST_DFS_ROOT=/share/dfs
CSST_CRDS_ROOT=/share/crdsdata/data
CSST_AUX_DIR=/share/pipeline/aux
# IFS
AUX_DIR=/share/shao/L1Pipeline/aux
CRDS_DIR=/share/shao/L1Pipeline/crds
DFS_ROOT=/share/dfs
\ No newline at end of file
L1/dockerfiles/csst-ifs-l1-cube/test/zjs.env
0 → 100644
View file @
0a669f09
# DFS
CSST_DFS_API_MODE=cluster
CSST_DFS_GATEWAY=172.24.27.2:30880
CSST_DFS_APP_ID=test
CSST_DFS_APP_TOKEN=test
# CRDS
CRDS_SERVER_URL=http://172.24.27.2:29000
# VOLUMES ==================
CSST_DFS_ROOT=/sharewcl/dfs
CSST_CRDS_ROOT=/sharewcl/crdsdata/data
CSST_AUX_DIR=/sharewcl/pipeline/aux
# IFS
AUX_DIR=/sharewcl/shao/L1Pipeline/aux
CRDS_DIR=/sharewcl/shao/L1Pipeline/crds
DFS_ROOT=/sharewcl/dfs
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment