完成腾讯云单节点测试

L1/dockerfiles/crds/Dockerfile

+FROM csst-l1ppl:6.0.144
+
+COPY gen_sql.sh /app/bin/
+RUN chmod +x /app/bin/gen_sql.sh
+COPY run.sh /app/bin/
+RUN chmod +x /app/bin/run.sh
+COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
+
+RUN mkdir /work/
+WORKDIR /work/
+    
+ENTRYPOINT ["goagent"]
\ No newline at end of file

L1/dockerfiles/crds/Makefile

+IMAGE_NAME:=csst/crds
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+#docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+	docker save $(IMAGE_NAME) > /share/tmp/crds.tar
+	ssh c0 docker load -i /share/tmp/crds.tar
+	ssh c1 docker load -i /share/tmp/crds.tar
+	ssh c2 docker load -i /share/tmp/crds.tar
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)

L1/dockerfiles/crds/gen_sql.sh

+
+#!/bin/bash
+
+export JOB_ID=1
+
+for line in `cat list.txt`
+do
+    echo "INSERT INTO t_task(job,key_message) VALUES($JOB_ID,'${line}');"
+done

L1/dockerfiles/crds/run.sh

+#!/bin/bash
+
+# input=test202210/P01_N6397_030s_Astrometry-off/MSC_0000000/,test202210/P01_N6397_030s_Astrometry-off/MSC_0000000_L1/,4
+input=$1
+arr=($(echo $input | tr "," " "))
+
+echo ${arr[0]} 
+echo ${arr[1]} 
+echo ${arr[2]} 
+
+python /L1Pipeline/build/csst_l1/app/l1_mbi_local.py /share/OnOrbitCal/SimData/${arr[0]} /share/OnOrbitCal/SimData/${arr[1]} ${arr[2]} > /work/stdout 2> /work/stderr
+
+exit_code=$?
+
+# 标准错误和标准输出，输出到文件
+echo "input : " ${arr[1]} >> /var/log/scalebox/stdout
+cat /work/stdout >> /var/log/scalebox/stdout
+
+echo "input : " ${arr[1]} >> /var/log/scalebox/stderr
+cat /work/stderr >> /var/log/scalebox/stderr
+
+rm -f /work/stdout /work/stderr
+
+#创建临时文件
+touch /share/OnOrbitCal/SimData/${arr[1]}csst-l1mod-tmp.log
+#取/L1Pipeline/L1/csst-l1mod.log后100行
+cat /share/OnOrbitCal/SimData/${arr[1]}csst-l1mod.log|tail -n 100 > /share/OnOrbitCal/SimData/${arr[1]}csst-l1mod-tmp.log
+# log文件记录到标准错误或标准输出
+cat /share/OnOrbitCal/SimData/${arr[1]}csst-l1ppl.log /share/OnOrbitCal/SimData/${arr[1]}csst-l1mod-tmp.log
+cat /share/OnOrbitCal/SimData/${arr[1]}csst-l1ppl.log /share/OnOrbitCal/SimData/${arr[1]}csst-l1mod-tmp.log >> /var/log/scalebox/${arr[1]}csst_l1.log
+
+echo finish crds.
+
+exit $exit_code
\ No newline at end of file

L1/dockerfiles/crds/test/Makefile

+CLUSTER=csst
+
+all: reset build
+
+run:
+	scalebox app create
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/dockerfiles/crds/test/app.yaml

+name: crds.app.process
+label: csst-crds
+cluster: csst
+parameters:
+  initial_status: RUNNING
+
+jobs:
+  # redis-cli:
+  #   label: 消息队列接受模块
+  #   base_image: csst/redis-cli
+  #   schedule_mode: HEAD
+  #  # parameters:
+  #  #   start_message: abcd
+  #   sink_jobs:
+  #     - crds
+  # admL1:
+  #   label: 1级流水线消息网关
+  #   base_image: csst/adml1
+  #   schedule_mode: HEAD
+  #   # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+  #   sink_jobs:
+  #     - crds
+  crds:
+    label: CRDS定标天区
+    base_image: csst/crds
+#    schedule_mode: HEAD
+    paths:
+      - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
+      - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
+      - /sharewcl/dfs:/dfsroot
+      - /sharewcl:/share
+    # sink_jobs:
+    hosts:
+    #  - h0:1
+      - c0:3
+      - c1:3
+      - c2:3
+
+
+
+      

L1/dockerfiles/mbi/Dockerfile

@@ -7,4 +7,4 @@ COPY --from=hub.cstcloud.cn/scalebox/base /usr/local/sbin /usr/local/sbin
 RUN mkdir /work/
 WORKDIR /work/
     
-ENTRYPOINT ["goagent"]
\ No newline at end of file
+ENTRYPOINT ["goagent"]

L1/dockerfiles/mbi/run.sh

 #!/bin/bash
 
 # clear directories
-rm -rf /L1Pipeline/L0/*
-rm -rf /L1Pipeline/L1/*
+#rm -rf /L1Pipeline/L0/*
+#rm -rf /L1Pipeline/L1/*
 
 # obsid 100000100 ~ 100000154
-python /L1Pipeline/build/csst_l1/app/l1_mbi.py $1 > /work/stdout 2> /work/stderr
+python /L1Pipeline/build/csst_l1/app/l1_mbi_tcc.py $1 > /work/stdout 2> /work/stderr
 
 exit_code=$?
 
@@ -31,6 +31,31 @@ cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log >> /var/log/
 
 rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
 
+# do timestamps
+ts=""
+if [ -f "/tmp/timestamps.txt" ]; then
+    mapfile arr < /tmp/timestamps.txt
+    for e in ${arr[@]}; do
+       # e=${e:0:-5}
+       # e+="+08:00"
+       # e=$(echo $e|tr ',' '.')
+        ts+=\"${e}\"\,
+    done
+    # skip the last ','
+    ts=${ts:0:-1}
+    rm -f /tmp/timestamps.txt
+
+cat << EOF > /work/record.json
+{
+    "timestamps":[${ts}]
+}
+EOF
+
+fi
+
+
+
+
 echo finish mbi.
 
-exit $exit_code
\ No newline at end of file
+exit $exit_code

L1/dockerfiles/mbi/test/Makefile

@@ -3,7 +3,7 @@ CLUSTER=csst
 all: reset build
 
 run:
-	scalebox app create
+	scalebox app create --env-file tencent.env
 
 reset:
 	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -

L1/dockerfiles/mbi/test/app.yaml

@@ -25,8 +25,12 @@ jobs:
     base_image: csst/mbi
     schedule_mode: HEAD
     paths: 
-      - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-      - /sharewcl/dfs:/dfsroot
+      # - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
+      # - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
+      # - /sharewcl/dfs:/dfsroot
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${DFS_ROOT}:/dfsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
     # sink_jobs:
     hosts:
@@ -39,9 +43,9 @@ jobs:
     base_image: csst/sls2d
     schedule_mode: HEAD
     paths: 
-      - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-      - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-      - /sharewcl/dfs:/dfsroot
+      - ${AUX_DIR}:/L1Pipeline/aux
+      - ${CRDS_DIR}:/L1Pipeline/aux/C6.1_ref_crds
+      - ${DFS_ROOT}:/dfsroot
     # command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
     # sink_jobs:   
     hosts:

L1/dockerfiles/mbi/test/tencent.env

+AUX_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux
+CRDS_DIR=/goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr
+DFS_ROOT=/goosefsx/x_c60_o19xp6c1_proxy/dfs_root

L1/dockerfiles/mbi/test/zjs.env

+AUX_DIR=/sharewcl/L1Pipeline/aux
+CRDS_DIR=/sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr
+DFS_ROOT=/sharewcl/dfs

L1/dockerfiles/module2/Dockerfile

@@ -9,4 +9,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 COPY *.sh /app/bin/
-COPY *.py /app/bin/
\ No newline at end of file
+COPY *.py /app/bin/
+
+RUN cd /app/bin/ \
+    && chmod +x run.sh

L1/dockerfiles/redis-cli/Dockerfile

@@ -3,7 +3,8 @@ FROM hub.cstcloud.cn/scalebox/agent
 # 安装redis-cli
 RUN apt-get update \
     && apt-get install -y redis
-ENV REDIS_SERVER=192.169.23.2
+ENV REDIS_SERVER=10.0.0.9
+#ENV REDIS_SERVER=192.169.23.2
 
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh

L1/dockerfiles/sls2d/Dockerfile

-FROM csst-l1ppl:6.0.137
-
+FROM csst-l1ppl:latest
 
 COPY run.sh /app/bin/
 RUN chmod +x /app/bin/run.sh

L1/dockerfiles/sls2d/Makefile

 IMAGE_NAME:=csst/sls2d
 
 build:
-        docker build --network=host -t $(IMAGE_NAME) .
+	docker build --network=host -t $(IMAGE_NAME) .
 
 dist: build
         #docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
-        docker save $(IMAGE_NAME) > /share/tmp/sls2d.tar
-        ssh c0 docker load -i /share/tmp/sls2d.tar
-        ssh c1 docker load -i /share/tmp/sls2d.tar
-        ssh c2 docker load -i /share/tmp/sls2d.tar
+	docker save $(IMAGE_NAME) > /share/tmp/sls2d.tar
+	ssh c0 docker load -i /share/tmp/sls2d.tar
+	ssh c1 docker load -i /share/tmp/sls2d.tar
+	ssh c2 docker load -i /share/tmp/sls2d.tar
 push:
-        docker push $(IMAGE_NAME)
+	docker push $(IMAGE_NAME)
 run:
-        docker run -it --entrypoint bash $(IMAGE_NAME)
+	docker run -it --entrypoint bash $(IMAGE_NAME)
 down:
-        docker stop $(IMAGE_NAME)
+	docker stop $(IMAGE_NAME)

L1/dockerfiles/sls2d/run.sh

@@ -5,7 +5,7 @@ rm -rf /L1Pipeline/L0/*
 rm -rf /L1Pipeline/L1/*
 
 # obsid 100000100 ~ 100000154
-python /L1Pipeline/build/csst_l1/app/l1_sls2d.py $1 > /work/stdout 2> /work/stderr
+python /L1Pipeline/build/csst_l1/app/l1_sls2d_tcc.py $1 > /work/stdout 2> /work/stderr
 
 exit_code=$?
 
@@ -31,6 +31,28 @@ cat /L1Pipeline/L1/csst-l1ppl.log /L1Pipeline/L1/csst-l1mod-tmp.log >> /var/log/
 
 rm -rf /L1Pipeline/L0/* /L1Pipeline/L1/*
 
+# do timestamps
+ts=""
+if [ -f "/tmp/timestamps.txt" ]; then
+    mapfile arr < /tmp/timestamps.txt
+    for e in ${arr[@]}; do
+        #e=${e:0:-5}
+        #e+="+08:00"
+	#e=$(echo $e|tr ',' '.')
+        ts+=\"${e}\"\,
+    done
+    # skip the last ','
+    ts=${ts:0:-1}
+    rm -f /tmp/timestamps.txt
+
+cat << EOF > /work/record.json
+{
+    "timestamps":[${ts}]
+}
+EOF
+
+fi
+
 echo finish sls2d.
 
 exit $exit_code

L1/dockerfiles/sls2d/test/add_tasks.sh

+#!/bin/bash
+
+POSTGRES_USER=scalebox
+POSTGRES_HOST=localhost
+POSTGRES_DB=scalebox
+PGPORT=5432
+
+job_id=45
+
+for m in {100000020..100000154}; do
+    echo $m
+psql -U ${POSTGRES_USER} -h ${POSTGRES_HOST} -d ${POSTGRES_DB} -p ${PGPORT} << EOF
+INSERT INTO t_task(job,key_message) VALUES(${job_id},'${m}');
+EOF
+
+done
+

L1/dockerfiles/sls2d/test/app.yaml

@@ -9,17 +9,20 @@ jobs:
     label: 无缝光谱2D
     base_image: csst/sls2d
     schedule_mode: HEAD
-    variables:
-      repeated: yes
-    parameters:
-      start_message: 100000100
+#    variables:
+#      repeated: yes
+#    parameters:
+#      start_message: 100000100
     paths:
-      - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
-      - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
-      - /sharewcl/dfs:/dfsroot
+      - /goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux:/L1Pipeline/aux
+      - /goosefsx/x_c60_o19xp6c1_proxy/L1Pipeline/aux/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
+      - /goosefsx/x_c60_o19xp6c1_proxy/dfs_root:/dfsroot        
+   #   - /sharewcl/L1Pipeline/aux:/L1Pipeline/aux
+   #   - /sharewcl/OnOrbitCal/SimData/ref_202211/products_ref20_3hdr:/L1Pipeline/aux/C6.1_ref_crds
+   #   - /sharewcl/dfs:/dfsroot
     # sink_jobs:
-    # hosts:
+    #    hosts:
     #  - h0:1
-    #   - c0:1
-    #   - c1:1
-    #   - c2:1
+    #  - c0:3
+    #  - c1:3
+    #  - c2:3