增加批量集成测试方法，以及不同测试环境的环境变量文件

L1/dockerfiles/crds/test/app.yaml

@@ -23,18 +23,18 @@ jobs:
   crds:
     label: CRDS定标天区
     base_image: csst/crds
-#    schedule_mode: HEAD
+    schedule_mode: HEAD
     paths:
-      - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-      - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-      - /sharewcl/dfs:/dfsroot
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${DFS_ROOT}:/dfsroot
       - /sharewcl:/share
     # sink_jobs:
-    hosts:
+    #hosts:
     #  - h0:1
-      - c0:3
-      - c1:3
-      - c2:3
+    #  - c0:3
+    #  - c1:3
+    #  - c2:3
 
 
 

L1/dockerfiles/crds/test/tencent.env

+AUX_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux
+CRDS_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr
+DFS_ROOT=/goosefsx/x_c60_o19xp6c1_proxy/dfs_root

L1/dockerfiles/crds/test/zjs.env

+AUX_DIR=/sharewcl/L1Pipeline/aux
+CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+DFS_ROOT=/sharewcl/dfs

L1/dockerfiles/mbi/run.sh

@@ -5,20 +5,25 @@
 #rm -rf /L1Pipeline/L1/*
 
 # obsid 100000100 ~ 100000154
-python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $1 > /work/stdout 2> /work/stderr
+##压力测，保留后三位，前六位替换回原格式100000
+test_obsid=$1
+obsid="100000"${test_obsid:0-3:3}
+###
+
+python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $obsid > /work/stdout 2> /work/stderr
 
 exit_code=$?
 
 # 标准错误和标准输出，输出到文件
-echo "obsid : "$1 >> /var/log/scalebox/stdout
+echo "obsid : "$obsid >> /var/log/scalebox/stdout
 cat /work/stdout >> /var/log/scalebox/stdout
 
-echo "obsid : "$1 >> /var/log/scalebox/stderr
+echo "obsid : "$obsid >> /var/log/scalebox/stderr
 cat /work/stderr >> /var/log/scalebox/stderr
 
 rm -f /work/stdout /work/stderr
 
-obsid=$1
+#obsid=$1
 #cat /L1Pipeline/L1/csst-l1ppl.log
 #cat /L1Pipeline/L1/csst-l1ppl.log >> /var/log/scalebox/${obsid}.log
 #创建临时文件

L1/dockerfiles/mbi/test/app.yaml

 name: mbi.app.process
 label: csst-mbi
-cluster: csst
+cluster: local
 parameters:
   initial_status: RUNNING
   
 jobs:
-  redis-cli:
-    label: 消息队列接收模块
-    base_image: csst/redis-cli
-    schedule_mode: HEAD
-    parameters:
-      start_message: abcd
-    sink_jobs:
-      - mbi 
-  admL1:
-    label: 1级流水线消息网关
-    base_image: csst/adml1
-    schedule_mode: HEAD
-    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
-    sink_jobs:
-      - mbi
   mbi:
-    label: 多色成像
     base_image: csst/mbi
     schedule_mode: HEAD
-    paths: 
-      # - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-      # - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-      # - /sharewcl/dfs:/dfsroot
-      - ${AUX_DIR}:/L1Pipeline/aux
-      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
-      - ${DFS_ROOT}:/dfsroot
-    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
-    # sink_jobs:
-    hosts:
-      - h0:1
-      - c0:2
-      - c1:2
-      - c2:2
-  sls2d:
-    label: 无缝光谱
-    base_image: csst/sls2d
-    schedule_mode: HEAD
     paths: 
       - ${AUX_DIR}:/L1Pipeline/aux
       - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
       - ${DFS_ROOT}:/dfsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
     # sink_jobs:
-    hosts:
-      - h0:1
-      - c0:2
-      - c1:2
-      - c2:2
+#    hosts:
+#      - h0:1
+#      - c0:1
+#      - c1:1
+#      - c2:1
       
 
       

L1/dockerfiles/sls2d/run.sh

@@ -5,20 +5,24 @@ rm -rf /L1Pipeline/L0/*
 rm -rf /L1Pipeline/L1/*
 
 # obsid 100000100 ~ 100000154
-python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $1 > /work/stdout 2> /work/stderr
+##压力测，保留后三位，前六位替换回原格式100000
+test_obsid=$1
+obsid="100000"${test_obsid:0-3:3}
+###
+python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $obsid > /work/stdout 2> /work/stderr
 
 exit_code=$?
 
 # 标准错误和标准输出，输出到文件
-echo "obsid : "$1 >> /var/log/scalebox/stdout
+echo "obsid : "$obsid >> /var/log/scalebox/stdout
 cat /work/stdout >> /var/log/scalebox/stdout
 
-echo "obsid : "$1 >> /var/log/scalebox/stderr
+echo "obsid : "$obsid >> /var/log/scalebox/stderr
 cat /work/stderr >> /var/log/scalebox/stderr
 
 rm -f /work/stdout /work/stderr
 
-obsid=$1
+#obsid=$1
 #cat /L1Pipeline/L1/csst-l1ppl.log
 #cat /L1Pipeline/L1/csst-l1ppl.log >> /var/log/scalebox/${obsid}.log
 #创建临时文件

L1/dockerfiles/sls2d/test/Makefile

+CLUSTER=local
+
+# all: reset build
+
+run:
+	scalebox app create --env-file tencent.env
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/sls2d/test/add_task_v2.sh

+
+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=$1
+
+for m in {100000088..100000123}; do
+    echo $m
+    docker exec -t database psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} \
+        -c "INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}')"
+
+done

L1/dockerfiles/sls2d/test/app.yaml

@@ -14,12 +14,9 @@ jobs:
 #    parameters:
 #      start_message: 100000100
     paths:
-      - /goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux:/L1Pipeline/aux
-      - /goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-      - /goosefsx/x_c60_o19xp6c1_proxy/dfs_root:/dfsroot        
-   #   - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-   #   - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-   #   - /sharewcl/dfs:/dfsroot
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${DFS_ROOT}:/dfsroot
    # sink_jobs:
     #    hosts:
     #  - h0:1

L1/dockerfiles/sls2d/test/sls2d.yaml

+name: sls2d.app.process
+label: csst-sls2d
+cluster: local
+parameters:
+  initial_status: RUNNING
+  
+jobs:
+  redis-cli:
+    label: 消息队列接收模块
+    base_image: csst/redis-cli
+    schedule_mode: HEAD
+    parameters:
+      start_message: abcd
+    sink_jobs:
+      - admL1 
+  admL1:
+    label: 1级流水线消息网关
+    base_image: csst/adml1
+    schedule_mode: HEAD
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    sink_jobs:
+      - sls2d
+  sls2d:
+    label: 无缝光谱
+    base_image: csst/sls2d
+    schedule_mode: HEAD
+    paths: 
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${DFS_ROOT}:/dfsroot
+    # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:   
+    #hosts:
+    #  - h0:1
+    #  - c0:2
+    #  - c1:2
+    #  - c2:2
+      
+
+      

L1/dockerfiles/sls2d/test/tencent.env

+AUX_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux
+CRDS_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr
+DFS_ROOT=/goosefsx/x_c60_o19xp6c1_proxy/dfs_root

L1/dockerfiles/sls2d/test/zjs.env

+AUX_DIR=/sharewcl/L1Pipeline/aux
+CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+DFS_ROOT=/sharewcl/dfs