Commit 0a669f09 authored by Zhang Xiaoli's avatar Zhang Xiaoli
Browse files

add cube container, finish unit test.

parent fb9004f3
FROM csst/csst-ifs-l1-cube-adm:latest
LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
USER root
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin/
RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc
#WORKDIR /work/
WORKDIR /L1Pipeline/workspace
ENTRYPOINT [ "goagent" ]
IMAGE_NAME:=cnic/csst-ifs-l1-cube-adm
build:
docker build --network=host -t $(IMAGE_NAME) .
dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar
ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar
ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar
ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar
push:
docker push $(IMAGE_NAME)
run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
scp:
scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube/adm/
save:
docker save $(IMAGE_NAME) > /sharewcl/temp/ifs-l1-cube-adm.tar
\ No newline at end of file
#!/bin/bash
# pipeline=/pipeline/output/pipeline.log
# if test -f "$pipeline"; then
# echo "=====pipeline.log====" > /work/user-file.txt
# cat /pipeline/output/pipeline.log >> /work/user-file.txt
# fi
# timefile=/pipeline/output/timestamps.txt
# if test -f "$timefile"; then
# echo "$timefile exist"
# mv /pipeline/output/timestamps.txt /work/timestamps.txt
# fi
# obsid="30100000001"
python l1_pipeline_script_ifs_cube_adm.py > ./stdout
exit_code=$?
mkdir -p /work
# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
# mv /tmp/timestamps.txt /work/timestamps.txt
cat ./stdout
mv /L1Pipeline/workspace/list_output/cube_merge_obsid_list_confirm.txt /work/messages.txt
echo finish csst-ifs-l1-cube-adm.
exit $exit_code
FROM csst/csst-ifs-l1-cube-exe:latest
USER root
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/
RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc
WORKDIR /L1Pipeline/workspace
ENTRYPOINT [ "goagent" ]
IMAGE_NAME:=cnic/csst-ifs-l1-cube-exe
build:
docker build --network=host -t $(IMAGE_NAME) .
dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar
ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar
ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar
ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar
push:
docker push $(IMAGE_NAME)
run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
scp:
scp -r ./ csst-zjs:/root/csst/ifs-l1-cube/exe/
save:
docker save $(IMAGE_NAME) > /sharewcl/temp/ifs-l1-cube-exe.tar
\ No newline at end of file
#!/bin/bash
echo input:$1*
python l1_pipeline_script_ifs_cube_simdata.py $1 > ./stdout
exit_code=$?
mkdir -p /work
# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
# mv /tmp/timestamps.txt /work/timestamps.txt
echo finish csst-ifs-l1-cube-exe.
exit $exit_code
CLUSTER=local
all: reset build
run:
scalebox app create --env-file zjs-ztm.env
reset:
cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
down:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
list:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
scp:
scp -r ../ csst-zjs:/root/csst/csst-ifs-l1-cube/
\ No newline at end of file
name: ifs-l1-cube.app.process
label: 多次曝光合并
comment: 积分场光谱仪多次曝光合并一级流水线
cluster: local
parameters:
initial_status: RUNNING
jobs:
csst-ifs-l1-cube-adm:
label: 多次曝光合并管理模块
base_image: cnic/csst-ifs-l1-cube-adm:latest
schedule_mode: HEAD
variables:
# always_running: yes
output_text_size: 100000
# DFS入库错误重试3次
retry_rules: "['202:3']"
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
parameters:
start_message: ANY
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
- ${CSST_AUX_DIR}:/pipeline/aux
- ${CSST_DFS_ROOT}:/dfsroot
- ${CSST_CRDS_ROOT}:/crdsroot
sink_jobs:
- csst-ifs-l1-cube-exe
# command: >
# bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt"
# hosts:
# - h0:1
# - c0:1
# - c1:1
# - c2:1
csst-ifs-l1-cube-exe:
label: 多次曝光合并执行模块
base_image: cnic/csst-ifs-l1-cube-exe:latest
schedule_mode: HEAD
variables:
# always_running: yes
output_text_size: 100000
# DFS入库错误重试3次
retry_rules: "['202:3']"
text_tranc_mode: TAIL
locale_mode: NONE
# grpc_server: 172.24.23.6:50051
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
- ${AUX_DIR}:/L1Pipeline/aux
- ${CRDS_DIR}:/L1Pipeline/crds
- ${DFS_ROOT}:/L1Pipeline/dfs
# command: >
# python /pipeline/src/run.py {\"brick_id\":1}
# hosts:
# - h0:1
# - c0:1
# - c1:1
# - c2:2
\ No newline at end of file
# DFS
CSST_DFS_API_MODE=cluster
CSST_DFS_GATEWAY=172.24.27.2:30880
CSST_DFS_APP_ID=test
CSST_DFS_APP_TOKEN=test
# CRDS
CRDS_SERVER_URL=http://172.24.27.2:29000
# VOLUMES ==================
CSST_DFS_ROOT=/share/dfs
CSST_CRDS_ROOT=/share/crdsdata/data
CSST_AUX_DIR=/share/pipeline/aux
# IFS
AUX_DIR=/share/shao/L1Pipeline/aux
CRDS_DIR=/share/shao/L1Pipeline/crds
DFS_ROOT=/share/dfs
\ No newline at end of file
# DFS
CSST_DFS_API_MODE=cluster
CSST_DFS_GATEWAY=172.24.27.2:30880
CSST_DFS_APP_ID=test
CSST_DFS_APP_TOKEN=test
# CRDS
CRDS_SERVER_URL=http://172.24.27.2:29000
# VOLUMES ==================
CSST_DFS_ROOT=/sharewcl/dfs
CSST_CRDS_ROOT=/sharewcl/crdsdata/data
CSST_AUX_DIR=/sharewcl/pipeline/aux
# IFS
AUX_DIR=/sharewcl/shao/L1Pipeline/aux
CRDS_DIR=/sharewcl/shao/L1Pipeline/crds
DFS_ROOT=/sharewcl/dfs
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment