scalebox平台更新，增加DAG逻辑、路由转发。

L1/dockerfiles/csst-msc-l1-mbi-photmix/Dockerfile

@@ -3,7 +3,7 @@ FROM 10.3.10.10:8113/csst/csst-msc-l1-mbi-photmix
 USER root
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
 
 RUN mkdir /work/ 
 RUN chown -R csst:csst /work

L1/dockerfiles/csst-msc-l1-mbi/Dockerfile

-FROM 10.3.10.10:8113/csst/csst-msc-l1-mbi
+FROM 10.3.10.10:8113/csst/csst-msc-l1-mbi:100356
 
 USER root
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh
-COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
 
 RUN mkdir /work/ 
 RUN chown -R csst:csst /work

L1/dockerfiles/csst-msc-l1-mbi/run.sh

@@ -26,8 +26,13 @@ if test -f "$timefile"; then
     mv /pipeline/output/timestamp.txt /work/timestamps.txt
 fi
 
-rm -rf /pipeline/input/* /pipeline/output/*
+if [ $exit_code -eq 0 ]; then  
+  echo "finish mbi, start sink-job."  
+  scalebox task add $1
+else  
+  echo "finish mbi, exit_code: $exit_code"  
+fi
 
-echo finish mbi.
+rm -rf /pipeline/input/* /pipeline/output/*
 
 exit $exit_code

L1/dockerfiles/csst-msc-l1-mbi/test/add_task.sh

@@ -8,8 +8,8 @@ PGPORT=5432
 
 job_id=$1
 
-
-obsid="/nfsdata/share/pipeline-inttest/run-batch-msc/msg-mbi-all.txt"
+obsid="mbi.txt"
+# obsid="/nfsdata/share/pipeline-inttest/run-batch-msc/msg-mbi-all.txt"
 while IFS= read -r line
 do
   m="$line"
@@ -20,5 +20,5 @@ do
   echo $msg
   docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
         -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
-done
+done < $obsid
 ~

L1/dockerfiles/csst-msc-l1-mbi/test/app.yaml

 name: msc-l1-mbi.app.process
-label: 多色成像
+label: 多色成像+测光
 comment: 主巡天多色成像一级流水线
 cluster: csst-nao
 parameters:
@@ -33,11 +33,44 @@ jobs:
       - ${CCDS_ROOT}:/ccds_root:ro
       - ${CSST_AST_TEMP}:/pipeline/temp:rw
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    sink_jobs:
+      - msc-l1-mbi-photmix  
     hosts:
-      - h0:10
-      - c0:20
+      - h0:1
+      # - c0:20
       # - c1:1
       # - c2:1
-      
+  msc-l1-mbi-photmix:
+    label: 多色成像测光
+    base_image: cnic/csst-msc-l1-mbi-photmix
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051      
+    parameters:
+      # start_message: 10160000068
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    hosts:
+      - h0:1
+      # - c0:10
+      # - c1:1
+      # - c2:1     
 
       

L1/dockerfiles/csst-msc-l1-qc0/Dockerfile

+FROM 10.3.10.10:8113/csst/csst-msc-l1-qc0
+
+USER root
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+RUN mkdir /work/ 
+RUN chown -R csst:csst /work
+RUN chown -R csst:csst /pipeline
+WORKDIR /work/
+
+USER csst
+
+ENTRYPOINT ["goagent"]

L1/dockerfiles/csst-msc-l1-qc0/Makefile

+IMAGE_NAME:=cnic/csst-msc-l1-qc0
+IMAGE_PATH:=/nfsdata/tmp
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist: 
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > $(IMAGE_PATH)/msc-l1-qc0.tar
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)
+scp:
+	scp -r ./ csst-csu:/root/csst/csst-msc-l1-mbi/

L1/dockerfiles/csst-msc-l1-qc0/run.sh

+#!/bin/bash
+
+# obsid 10160000000 -  10160000136
+arr=($(echo $1 | tr "-" " "))
+obsid="${arr[0]}"
+detector="${arr[1]}"
+echo "obsid: "$obsid
+echo "detector: "$detector
+
+cd /pipeline/output/
+python /pipeline/app/run.py $obsid $detector
+
+exit_code=$?
+
+echo "=====pipeline.log====" > /work/custom-out.txt
+cat /pipeline/output/pipeline.log >> /work/custom-out.txt
+
+
+echo "======module.log======" >> /work/custom-out.txt
+cat /pipeline/output/module.log|tail -n 100 >> /work/custom-out.txt
+
+
+timefile=/pipeline/output/timestamp.txt
+if test -f "$timefile"; then
+    echo "$timefile exist"
+    mv /pipeline/output/timestamp.txt /work/timestamps.txt
+fi
+
+rm -rf /pipeline/input/* /pipeline/output/*
+
+if [ $exit_code -eq 0 ]; then  
+    scalebox task add $1
+    echo "finish qc0, start sink-job."  
+else  
+    echo "finish qc0, exit_code: $exit_code"  
+fi
+
+exit $exit_code

L1/dockerfiles/csst-msc-l1-qc0/test/Makefile

+CLUSTER=csst-nao
+
+all: reset build
+
+run:
+# PGHOST=192.168.25.27 GRPC_SERVER=192.168.25.27 scalebox app create --env-file csu.env
+	PGPORT=9090 scalebox app create --env-file nao.env
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/csst-msc-l1-qc0/test/add_task.sh

+#!/bin/bash
+
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+obsid="test.txt"
+#obsid="/nfsdata/share/simulation-collection/msc/v092/raw/msg-all.txt"
+while IFS= read -r line
+do
+  m="$line"
+  arr=($(echo $m | tr " " " "))
+  obsid="${arr[0]}"
+  detector="${arr[1]}"
+  msg=$obsid-$detector
+  echo $msg
+  docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT}\
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${msg}')"
+done < $obsid
+~

L1/dockerfiles/csst-msc-l1-qc0/test/app.yaml

+name: msc-l1-qc0.app.process
+label: QC0
+comment: 主巡天QC0
+cluster: csst-nao
+parameters:
+  initial_status: RUNNING
+  
+jobs:
+  msc-l1-qc0:
+    label: QC0
+    base_image: cnic/csst-msc-l1-qc0
+    # schedule_mode: HEAD
+    variables:
+      # always_running: yes
+      # reserved_on_exit: yes
+      output_text_size: 100000
+      text_tranc_mode: TAIL
+      locale_mode: NONE
+      # grpc_server: 172.24.23.6:50051      
+    parameters:
+      # start_message: FILE:test.txt
+      key_group_regex: ^(.{6})(.{3})$
+      key_group_seq: 1,2
+    environments:
+      - CSST_DFS_API_MODE=${CSST_DFS_API_MODE}
+      - CSST_DFS_GATEWAY=${CSST_DFS_GATEWAY}
+      - CSST_DFS_APP_ID=${CSST_DFS_APP_ID}
+      - CSST_DFS_APP_TOKEN=${CSST_DFS_APP_TOKEN}
+      - CCDS_SERVER_URL=${CCDS_SERVER_URL}
+    paths: 
+      - ${CSST_AUX_ROOT}:/pipeline/aux:ro
+      - ${CSST_DFS_ROOT}:/dfs_root:ro
+      - ${CCDS_ROOT}:/ccds_root:ro
+      - ${CSST_AST_TEMP}:/pipeline/temp:rw
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    hosts:
+      - h0:1
+      # - c0:0
+      # - c1:1
+      # - c2:1
+      
+
+      

L1/dockerfiles/csst-msc-l1-qc0/test/nao.env

+# DFS
+CSST_DFS_GATEWAY=10.3.10.28:30880
+CSST_DFS_API_MODE=cluster
+CSST_DFS_APP_ID=test
+CSST_DFS_APP_TOKEN=test
+CSST_DFS_LOGS_DIR="."
+
+# CCDS
+CCDS_SERVER_URL=http://10.3.10.28:29000
+CCDS_USER=USER
+CCDS_PASS=PASS
+
+# VOLUMES
+CSST_DFS_ROOT=/nfsdata/share/dfs/dfs_root                   # /dfs_root:ro
+CCDS_ROOT=/nfsdata/share/dfs/ccds_root                      # /ccds_root:ro
+CSST_AUX_ROOT=/nfsdata/share/pipeline-inttest/aux           # /pipeline/aux:ro
+CSST_AST_TEMP=/nfsdata/share/pipeline-inttest/ast_temp      # /pipeline/temp:rw
+

L1/dockerfiles/redis-cli/Dockerfile

@@ -2,9 +2,18 @@ FROM hub.cstcloud.cn/scalebox/agent
 
 # 安装redis-cli
 RUN apt-get update \
-    && apt-get install -y redis
+    && apt-get install -y redis \
+    && apt-get install jq
+ENTRYPOINT []
+
 #ENV REDIS_SERVER=10.0.0.9
-ENV REDIS_SERVER=192.169.23.2
+ENV REDIS_SERVER=10.3.10.28
+ENV REDIS_PORT=26379
+ENV REDIS_PWD=csst__2025
+
+COPY --from=hub.cstcloud.cn/scalebox/agent /usr/local /usr/local
+
+COPY *.sh /app/bin/
+RUN chmod +x /app/bin/*.sh
 
-COPY run.sh /app/bin/
-RUN chmod +x /app/bin/run.sh
+CMD run.sh

L1/dockerfiles/redis-cli/Makefile

@@ -8,7 +8,9 @@ dist:
 
 push:
 	docker push $(IMAGE_NAME)
+run-bash:
+	docker run -it --entrypoint bash -e GRPC_SERVER=10.3.10.28 -v ${PWD}/config:/config $(IMAGE_NAME)
 run:
-	docker run -it --entrypoint bash $(IMAGE_NAME)
+	docker run -d --entrypoint -e GRPC_SERVER=10.3.10.28 -v ${PWD}/config:/config $(IMAGE_NAME)
 down:
 	docker stop $(IMAGE_NAME)

L1/dockerfiles/redis-cli/config/appconfig.txt

+202411111841239406 9
\ No newline at end of file

L1/dockerfiles/redis-cli/run.sh

 #!/bin/bash
 
+# json串
+# "{\"dag_id\": \"csst-msc-l1-mbi\", \"parameters\": {\"dag_run_id\": \"202411111505581036\", \"message\": {\"obsid\": \"11009101682009\", \"chipid\": \"24\"}}}"
 while true
 do
-    obsid=$(redis-cli -h ${REDIS_SERVER} -a 123456 rpop single-image-reduction:obsid)
-    echo "obsid : "$obsid
+    # obsid=$(redis-cli -h ${REDIS_SERVER} -p ${REDIS_PORT} rpop single-image-reduction:obsid)
+    # msg=$(redis-cli -h ${REDIS_SERVER} -p ${REDIS_PORT} -a ${REDIS_PWD} rpop csst_data_list)
+    msg=$(redis-cli -h 10.3.10.28 -a 123456 lpop csst_data_list_zxl)
+    #redis-cli -h 10.3.10.28 -p 26379 -a "csst__2025" rpop csst_data_list
+    echo "msg : "$msg
 
-    if [ -z "$obsid" ]; then
-        echo "obsid is empty"
-        exit 0
+    if [ -z "$msg" ]; then
+        echo "msg is empty"
+        exit 102
     else
-        echo "obsid is not empty"
-        send-message $obsid
+        echo "msg is not empty" $msg
+        # 提取dag_id 模块名
+        app_name=$(echo "$msg" | jq -r '.dag_id')
+        echo "app_name : "$app_name
+
+        # 提取dag_run_id 对应app_id
+        dag_run_id=$(echo "$msg" | jq -r '.parameters.dag_run_id')
+        echo "dag_run_id : "$dag_run_id
+        #检索dag_run_id与app_id对应关系的配置文件appconfig.txt
+        app_id=$(awk -v key="$dag_run_id" '$1 == key {print $2}' /config/appconfig.txt)
+    
+        # 检查是否找到app_id
+        if [ -n "$app_id" ]; then
+            echo "The appid for $dag_run_id is $app_id"
+        else
+            echo "$dag_run_id not found in appconfig.txt"
+            exit 103
+        fi
+
+        # 提取obsid
+        obsid=$(echo "$msg" | jq -r '.parameters.message.obsid')
+        echo "obsid :"$obsid
+
+        # 提取chipid
+        chipid=$(echo "$msg" | jq -r '.parameters.message.chipid')
+        echo "chipid :"$chipid
+
+        message=$obsid"-"$chipid
+        echo "message :"$message
+
+
+        #send-message $obsid
+        job_name="message-router-csst"
+        scalebox task add --app-id ${app_id} --sink-job ${job_name} ${message}
         code=$?
         #code值不为0时，退出
         if [ $code -ne 0 ]; then

L1/dockerfiles/redis-cli/test.sh

+redis-cli -h 10.3.10.28 -a 123456 lpush csst_data_list_zxl "{\"dag_id\": \"csst-msc-l1-mbi\", \"parameters\": {\"dag_run_id\": \"202411111841239406\", \"message\": {\"obsid\": \"11009101682009\", \"chipid\": \"16\"}}}"
\ No newline at end of file