Commit 65422b8c authored by Zhang Xiaoli's avatar Zhang Xiaoli
Browse files

Completed csst-msc-l2-xcat pipeline test.

parent 705b8dec
IMAGE_ADM:=cnic/csst-msc-l1-mbi
build:
make -C adm build
make -C exe build
sync:
rsync -av --del . csst-zjs:/tmp/csst-msc-l2-xcat
dist:
docker save hub.cstcloud.cn/scalebox/base:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load'
# docker save hub.cstcloud.cn/scalebox/controld:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load'
# docker save hub.cstcloud.cn/scalebox/database:latest | zstdmt | pv | ssh csst-zjs 'zstd -d | docker load'
push:
docker push $(IMAGE_NAME)
run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
FROM csst/csst-msc-l2-xcat:latest
USER root
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/
RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc
# RUN mkdir -p /work /app/bin && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc
WORKDIR /work
ENTRYPOINT [ "goagent" ]
IMAGE_NAME:=cnic/csst-msc-l2-xcat-adm
build:
docker build --network=host -t $(IMAGE_NAME) .
dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-adm.tar
ssh c0 docker load -i /share/tmp/msc-l2-xcat-adm.tar
ssh c1 docker load -i /share/tmp/msc-l2-xcat-adm.tar
ssh c2 docker load -i /share/tmp/msc-l2-xcat-adm.tar
push:
docker push $(IMAGE_NAME)
run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
scp:
scp -r ./ csst-zjs:/root/csst/csst-msc-l2-xcat/adm/
\ No newline at end of file
#!/bin/bash
python /pipeline/src/adm.py
exit_code=$?
mv /pipeline/output/msg.txt /work/messages.txt
exit $exit_code
name: msc-l2-xcat.app.process
label: 星表检索
comment: 星表检索
cluster: csst
parameters:
initial_status: RUNNING
jobs:
# msc-l1-mbi:
# label: 星表管理模块
# base_image: cnic/csst-msc-l1-mbi
# schedule_mode: HEAD
# variables:
# always_runing: yes
# output_text_size: 100000
# text_tranc_mode: TAIL
# locale_mode: NONE
# paths:
# - ${AUX_DIR}:/pipeline/aux
# - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
# - ${DFS_ROOT}:/dfsroot
csst-msc-l2-xcat-adm:
label: 星表管理模块
base_image: cnic/csst-msc-l2-xcat-adm:latest
schedule_mode: HEAD
variables:
always_running: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
parameters:
start_message: 202306212339
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
- /sharewcl/pipeline/output-csst-msc-l2-xcat-adm:/pipeline/output # output directory, omit this line in production
- /sharewcl/pipeline/aux:/pipeline/aux
- /sharewcl/dfs:/dfsroot
- /sharewcl/crdsdata/data/references:/crdsref
sink_jobs:
- csst-msc-l2-xcat-exe
# command: >
# bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt"
csst-msc-l2-xcat-exe:
base_image: cnic/csst-msc-l2-xcat-exe:latest
schedule_mode: HEAD
variables:
always_running: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
# output directory, omit this line in production
- /sharewcl/pipeline/output-csst-msc-l2-xcat-exe:/pipeline/output
- /sharewcl/pipeline/aux:/pipeline/aux
- /sharewcl/dfs:/dfsroot
- /sharewcl/crdsdata/data/references:/crdsref
# command: >
# python /pipeline/src/run.py {\"brick_id\":1}
\ No newline at end of file
FROM csst/csst-msc-l2-xcat:latest
USER root
COPY run.sh /app/bin/
RUN chmod +x /app/bin/run.sh
COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin/ /usr/local/sbin/
RUN mkdir -p /work && echo "PATH=/app/bin:\${PATH}" >> /root/.bashrc
WORKDIR /work
ENTRYPOINT [ "goagent" ]
IMAGE_NAME:=cnic/csst-msc-l2-xcat-exe
build:
docker build --network=host -t $(IMAGE_NAME) .
dist:
#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
docker save $(IMAGE_NAME) > /share/tmp/msc-l2-xcat-exe.tar
ssh c0 docker load -i /share/tmp/msc-l2-xcat-exe.tar
ssh c1 docker load -i /share/tmp/msc-l2-xcat-exe.tar
ssh c2 docker load -i /share/tmp/msc-l2-xcat-exe.tar
push:
docker push $(IMAGE_NAME)
run:
docker run -it --entrypoint bash $(IMAGE_NAME)
down:
docker stop $(IMAGE_NAME)
scp:
scp -r ./ csst-zjs:/root/csst/csst-msc-l2-xcat/exe/
\ No newline at end of file
#!/bin/bash
echo input:$1*
python /pipeline/src/exe.py $1
exit $?
CLUSTER=csst
all: reset build
run:
scalebox app create --env-file zjs.env
reset:
cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
down:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
list:
make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
scp:
scp -r ../ csst-zjs:/root/csst/csst-msc-l2-xcat/
\ No newline at end of file
name: msc-l2-xcat.app.process
label: 星表检索
comment: 星表检索
cluster: csst
parameters:
initial_status: RUNNING
jobs:
csst-msc-l2-xcat-adm:
label: 星表管理模块
base_image: cnic/csst-msc-l2-xcat-adm:latest
schedule_mode: HEAD
variables:
always_running: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
parameters:
start_message: 202306212339
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
- ${XCAT_ADM_OUTPUT}:/pipeline/output # output directory, omit this line in production
- ${AUX_DIR}:/pipeline/aux
- ${DFS_ROOT}:/dfsroot
- ${CRDS_REFERENCES}:/crdsref
sink_jobs:
- csst-msc-l2-xcat-exe
# command: >
# bash -c "python /pipeline/src/run.py && echo '---' && cat /pipeline/output/msg.txt"
csst-msc-l2-xcat-exe:
label: 星表执行模块
base_image: cnic/csst-msc-l2-xcat-exe:latest
# schedule_mode: HEAD
variables:
always_running: yes
output_text_size: 100000
text_tranc_mode: TAIL
locale_mode: NONE
environments:
- CSST_DFS_API_MODE=cluster
- CSST_DFS_GATEWAY=172.24.27.2:30880
- CSST_DFS_APP_ID=test
- CSST_DFS_APP_TOKEN=test
paths: # change /share to /sharewcl in PML production environment
# output directory, omit this line in production
- ${XCAT_EXE_OUTPUT}:/pipeline/output
- ${AUX_DIR}:/pipeline/aux
- ${DFS_ROOT}:/dfsroot
- ${CRDS_REFERENCES}:/crdsref
# command: >
# python /pipeline/src/run.py {\"brick_id\":1}
hosts:
- h0:1
- c0:1
- c1:1
- c2:1
\ No newline at end of file
AUX_DIR=/sharewcl/pipeline/aux
CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
DFS_ROOT=/sharewcl/dfs
CRDS_REFERENCES=/sharewcl/crdsdata/data/references
XCAT_ADM_OUTPUT=/sharewcl/pipeline/output-csst-msc-l2-xcat-adm
XCAT_EXE_OUTPUT=/sharewcl/pipeline/output-csst-msc-l2-xcat-exe
# AUX_DIR=/share/pipeline/aux
# CRDS_DIR=/share/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
# DFS_ROOT=/share/dfs
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment