国台环境下测试多色成像+测光

L1/dockerfiles/csst-msc-l1-mbi/Dockerfile

@@ -5,7 +5,11 @@ COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
 COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
 
-RUN mkdir /work/
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
 WORKDIR /work/
-    
+
+USER csst
+
 ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-msc-l1-mbi/Makefile

 IMAGE_NAME:=cnic/csst-msc-l1-mbi
-IMAGE_PATH:=/sharewcl/temp
+IMAGE_PATH:=/nfsdata/tmp
 build:
 	docker build --network=host -t $(IMAGE_NAME) .
 
@@ -7,15 +7,8 @@ dist: build
 #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
 	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-mbi.tar
 	ssh c0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh n0 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh n1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh w2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh w3 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh w4 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh w5 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
-	ssh w6 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+# ssh c1 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
+# ssh c2 docker load -i $(IMAGE_PATH)/msc-l1-mbi.tar
 push:
 	docker push $(IMAGE_NAME)
 run:
@@ -23,4 +16,4 @@ run:
 down:
 	docker stop $(IMAGE_NAME)
 scp:
-	scp -r ./ csst-zjs:/root/csst/csst-msc-l1-mbi/
+	scp -r ./ csst-csu:/root/csst/csst-msc-l1-mbi/

L1/dockerfiles/csst-msc-l1-mbi/run.sh

 #!/bin/bash
 
-# obsid 100000100 ~ 100000154
-##压力测，保留后三位，前六位替换回原格式100000
-#test_obsid=$1
-#obsid="100000"${test_obsid:0-3:3}
-###
-
 # obsid 10160000000 -  10160000136
-obsid=$1
-# l1-mbi --obs-id=$obsid --device=cpu --n-jobs=18 --n-jobs-gpu=18 --clean-l0 --clean-l1 --cleanup --dfs-node=pml --ver-sim=C6.2 --no-photometry --no-astrometry
-python /pipeline/src/run.py \
-        --obs-id=$obsid \
-        --device=cpu \
-        --n-jobs=18 \
-        --n-jobs-gpu=9 \
-        --clean-l0 \
-        --clean-l1 \
-        --dfs-node=pml \
-        --ver-sim=C6.2 \
-        --no-astrometry \
-        --no-photometry
+arr=($(echo $1 | tr "-" " "))
+obsid="${arr[0]}"
+detector="${arr[1]}"
+echo "obsid: "$obsid
+echo "detector: "$detector
+
+python /pipeline/app/run.py $obsid $detector
+
 exit_code=$?
 
-echo "=====csst-l1ppl.log====" > /work/user-file.txt
-cat /pipeline/output/csst-l1ppl.log >> /work/user-file.txt
+echo "=====pipeline.log====" > /work/user-file.txt
+cat /pipeline/output/pipeline.log >> /work/user-file.txt
 
 
-echo "======csst-l1mod.log======" >> /work/user-file.txt
-cat /pipeline/output/csst-l1mod.log|tail -n 100 >> /work/user-file.txt
+echo "======module.log======" >> /work/user-file.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/user-file.txt
 
 
-timefile=/pipeline/output/timestamps.txt
+timefile=/pipeline/output/timestamp.txt
 if test -f "$timefile"; then
     echo "$timefile exist"
-    mv /pipeline/output/timestamps.txt /work/timestamps.txt
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
 fi
 
 rm -rf /pipeline/input/* /pipeline/output/*
 
 echo finish mbi.
 
-if [ $exit_code -eq 0 ]; then
-    for i in 06 07 08 09 11 12 13 14 15 16 17 18 19 20 22 23 24 25 ; do
-        echo $1-$i >> /work/messages.txt
-    done
-fi
-
 exit $exit_code

L1/dockerfiles/csst-msc-l1-mbi/test.sh

+#!/bin/bash
+# 获取输出目录下的文件名
+output_files=$(ls -1 -F /pipeline/output | grep -v [$/])
+
+# 文件名存入output_files.txt文件中
+echo $output_files >> /work/output_files.txt
+
+echo $(ls -1 -F /pipeline/output | grep -v [$/])
\ No newline at end of file

L1/dockerfiles/csst-msc-l1-mbi/test/Makefile

-CLUSTER=csst
+CLUSTER=csst-nao
 
 all: reset build
 
 run:
-	scalebox app create --env-file zjs.env
-
+# PGHOST=192.168.25.27 GRPC_SERVER=192.168.25.27 scalebox app create --env-file csu.env
+	PGPORT=9090 scalebox app create --env-file nao.env
 reset:
 	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
 

L1/dockerfiles/csst-msc-l1-mbi/test/add_task.sh

@@ -8,9 +8,11 @@ PGPORT=5432
 
 job_id=$1
 
-for m in {10160000000..10160000136}; do
-    echo $m
-    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
-        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
-
+for m in {10160000000..10160000001}; do
+    for n in 06 07  08 09 11 12 13 14 15 16 17 18 19 20 22 23 24 25 ; do
+        echo $m $n
+        msg=$m-$n    
+        docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
+            -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
+    done
 done

L1/dockerfiles/csst-msc-l1-mbi/test/app.yaml

 name: msc-l1-mbi.app.process
 label: 多色成像
 comment: 主巡天多色成像一级流水线
-cluster: csst
+cluster: csst-nao
 parameters:
   initial_status: RUNNING
   
@@ -26,18 +26,19 @@ jobs:
       - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
       - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
       - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
-      - CRDS_SERVER_URL=${CRDS_SERVER_URL}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
     paths: 
-      - ${CSST_AUX_DIR}:/pipeline/aux
-      - ${CSST_DFS_ROOT}:/dfsroot
-      - ${CSST_CRDS_ROOT}:/crdsroot
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
     # sink_jobs:
     hosts:
       - h0:1
-      - c0:1
-      - c1:1
-      - c2:1
+      # - c0:1
+      # - c1:1
+      # - c2:1
       
 
       

L1/dockerfiles/csst-msc-l1-mbi/test/csu.env

+# DFS
+CSST_DFS_API_MODE=cluster
+CSST_DFS_GATEWAY=192.168.25.112:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+
+# CRDS
+CRDS_SERVER_URL=http://192.168.25.112:29000
+
+# VOLUMES
+CSST_DFS_ROOT=/obsfs/dfs
+CSST_CRDS_ROOT=/obsfs/crdsdata/data
+CSST_AUX_DIR=/obsfs/pipeline/aux

L1/dockerfiles/csst-msc-l1-mbi/test/nao.env

+# DFS
+CSST_DFS_GATEWAY=10.3.10.28:30880
+CSST_DFS_API_MODE=cluster
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+CSST_DFS_LOGS_DIR="."
+
+# CCDS
+CCDS_SERVER_URL=http://10.3.10.28:29000
+CCDS_USER=USER
+CCDS_PASS=PASS
+
+# VOLUMES
+CSST_DFS_ROOT=/nfsdata/share/dfs/dfs_root                   # /dfs_root:ro
+CCDS_ROOT=/nfsdata/share/dfs/ccds_root                      # /ccds_root:ro
+CSST_AUX_ROOT=/nfsdata/share/pipeline-inttest/aux           # /pipeline/aux:ro
+CSST_AST_TEMP=/nfsdata/share/pipeline-inttest/ast_temp      # /pipeline/temp:rw
+

L1/dockerfiles/csst-msc-l1-mbi/test/tencent.env

-AUX_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux
-CRDS_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr
-DFS_ROOT=/goosefsx/x_c60_o19xp6c1_proxy/dfs_root
+# *.env: The env variables are used for running pipeline.
+
+# TCC
+
+# ENV_FILE=tcc.env
+
+# DFS
+CSST_DFS_API_MODE=cluster
+CSST_DFS_GATEWAY=10.0.0.136:30880
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+# CRDS
+CRDS_SERVER_URL=http://10.0.0.136:29000
+
+# VOLUMES ==================
+CSST_DFS_ROOT=/goosefsx/x-c70-y4s971cs-proxy/dfs
+CSST_CRDS_ROOT=/goosefsx/x-c70-y4s971cs-proxy/crdsdata/data
+CSST_AUX_DIR=/goosefsx/x-c70-y4s971cs-proxy/pipeline/aux
+