Complete preliminary docker packaging of cube and mci

L1/dockerfiles/csst-ifs-l1-cube-adm/Dockerfile

+FROM csst/csst-ifs-l1-cube-adm:latest
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
+
+RUN mkdir /work/
+#WORKDIR /work/
+WORKDIR /L1Pipeline/workspace
+    
+ENTRYPOINT ["goagent"]
+
+
+# RUN mkdir -p /L1Pipeline/workspace
+# WORKDIR /L1Pipeline/workspace
+# COPY code/ /L1Pipeline/workspace
+# RUN /app/bin/run.sh
+
+

L1/dockerfiles/csst-ifs-l1-cube-adm/Makefile

+IMAGE_NAME:=csst/ifs-l1-cube-adm
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-adm.tar
+	ssh c0 docker load -i /share/tmp/ifs-l1-cube-adm.tar
+	ssh c1 docker load -i /share/tmp/ifs-l1-cube-adm.tar
+	ssh c2 docker load -i /share/tmp/ifs-l1-cube-adm.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-adm/
+

L1/dockerfiles/csst-ifs-l1-cube-adm/add_task.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+objids=("NGC6217" "NGC6397")
+for((i=0;i<${#objids[@]};i++)); do
+    m=${objids[$i]}
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+done
+

L1/dockerfiles/csst-ifs-l1-cube-adm/run.sh

+#!/bin/bash
+# python helloworld.py
+
+
+# obsid="30100000001"
+
+obsid=$1
+
+python l1_pipeline_script_ifs_cube_adm_simdata.py $obsid > ./stdout
+
+exit_code=$?
+
+mkdir -p /work
+# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
+# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
+
+# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
+
+# mv /tmp/timestamps.txt /work/timestamps.txt
+
+echo finish ifs cube adm.
+
+cat ./stdout
+
+exit $exit_code
+

L1/dockerfiles/csst-ifs-l1-cube-adm/test/Makefile

+CLUSTER=csst
+
+all: reset build
+
+run:
+	scalebox app create --env-file zjs.env
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/csst-ifs-l1-cube-adm/test/add_task.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+objids=("NGC6217" "NGC6397")
+for((i=0;i<${#objids[@]};i++)); do
+    m=${objids[$i]}
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+done
+

L1/dockerfiles/csst-ifs-l1-cube-adm/test/app.yaml

+name: ifs-l1-cube-adm.app.process
+cluster: csst
+parameters:
+  initial_status: RUNNING
+
+jobs:
+  ifs-l1-cube-adm:
+    base_image: csst/ifs-l1-cube-adm
+    schedule_mode: HEAD
+    variables:
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: scalebox
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    paths: 
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/crds
+      - ${DFS_ROOT}:/L1Pipeline/dfs
+    # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
+    # sink_jobs:
+    # hosts:
+    #   - h0:1
+    #   - c0:2
+    #   - c1:2
+    #   - c2:2
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-cube-adm/test/zjs.env

+AUX_DIR=/sharewcl/shao/L1Pipeline/aux
+CRDS_DIR=/sharewcl/shao/L1Pipeline/crds
+DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-cube-exe/Dockerfile

+FROM csst/csst-ifs-l1-cube-exe:latest
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
+
+RUN mkdir /work/
+#WORKDIR /work/
+WORKDIR /L1Pipeline/workspace
+    
+ENTRYPOINT ["goagent"]
+
+
+# RUN mkdir -p /L1Pipeline/workspace
+# WORKDIR /L1Pipeline/workspace
+# COPY code/ /L1Pipeline/workspace
+# RUN /app/bin/run.sh
+
+

L1/dockerfiles/csst-ifs-l1-cube-exe/Makefile

+IMAGE_NAME:=csst/ifs-l1-cube-exe
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > /share/tmp/ifs-l1-cube-exe.tar
+	ssh c0 docker load -i /share/tmp/ifs-l1-cube-exe.tar
+	ssh c1 docker load -i /share/tmp/ifs-l1-cube-exe.tar
+	ssh c2 docker load -i /share/tmp/ifs-l1-cube-exe.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-zjs:/root/csst/csst-ifs-l1-cube-exe/
+

L1/dockerfiles/csst-ifs-l1-cube-exe/add_task.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+objids=("NGC6217" "NGC6397")
+for((i=0;i<${#objids[@]};i++)); do
+    m=${objids[$i]}
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+done
+

L1/dockerfiles/csst-ifs-l1-cube-exe/run.sh

+#!/bin/bash
+# python helloworld.py
+
+
+# obsid="30100000001"
+
+obsid=$1
+
+python l1_pipeline_script_ifs_rss_simdata.py $obsid > ./stdout
+
+exit_code=$?
+
+mkdir -p /work
+# echo "=====csst-l1-ifs-rss.log====" > /work/user-file.txt
+# cat /L1Pipeline/workspace/ifs/rss/proc_data/path_RSSlog.txt >> /work/user-file.txt
+
+# rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
+
+# mv /tmp/timestamps.txt /work/timestamps.txt
+
+echo finish ifs cube exe.
+
+exit $exit_code
+

L1/dockerfiles/csst-ifs-l1-cube-exe/test/Makefile

+CLUSTER=csst
+
+all: reset build
+
+run:
+	scalebox app create --env-file zjs.env
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/csst-ifs-l1-cube-exe/test/add_task.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+objids=("NGC6217" "NGC6397")
+for((i=0;i<${#objids[@]};i++)); do
+    m=${objids[$i]}
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+done
+

L1/dockerfiles/csst-ifs-l1-cube-exe/test/app.yaml

+name: ifs-l1-cube-exe.app.process
+cluster: csst
+parameters:
+  initial_status: RUNNING
+
+jobs:
+  ifs-l1-cube-exe:
+    base_image: csst/ifs-l1-cube-exe
+    schedule_mode: HEAD
+    variables:
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+    parameters:
+      # start_message: scalebox
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    paths: 
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/crds
+      - ${DFS_ROOT}:/L1Pipeline/dfs
+    # command: docker run -d --network=host %ENVS% %VOLUMES% {{IMAGE}}
+    # sink_jobs:
+    # hosts:
+    #   - h0:1
+    #   - c0:2
+    #   - c1:2
+    #   - c2:2
\ No newline at end of file

L1/dockerfiles/csst-ifs-l1-cube-exe/test/zjs.env

+AUX_DIR=/sharewcl/shao/L1Pipeline/aux
+CRDS_DIR=/sharewcl/shao/L1Pipeline/crds
+DFS_ROOT=/sharewcl/shao/L1Pipeline/dfs
\ No newline at end of file

L1/dockerfiles/csst-mci-l1/Dockerfile

+FROM csst/csst-mci-l1:latest
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/base:0.9 /usr/local/sbin /usr/local/sbin
+
+RUN mkdir /work/
+WORKDIR /work/
+    
+ENTRYPOINT ["goagent"]
+
+

L1/dockerfiles/csst-mci-l1/Makefile

+IMAGE_NAME:=csst/mci-l1
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > /share/tmp/mci-l1.tar
+	ssh c0 docker load -i /share/tmp/mci-l1.tar
+	ssh c1 docker load -i /share/tmp/mci-l1.tar
+	ssh c2 docker load -i /share/tmp/mci-l1.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+

L1/dockerfiles/csst-mci-l1/run.sh

+#!/bin/bash
+obsid="xxxxxxxxxxxx"
+
+# python l1_pipeline_script_mci_shao_simdata.py $obsid > ./stdout
+python /L1Pipeline/workspace/helloworld.py
+
+exit_code=$?   
\ No newline at end of file

L1/dockerfiles/csst-mci-l1/test/Makefile

+CLUSTER=csst
+
+all: reset build
+
+run:
+	scalebox app create --env-file zjs.env
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+