Initial pipeline L1 & L2

L1/dockerfiles/admL1/Dockerfile

+FROM hub.cstcloud.cn/scalebox/agent:dev
+
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+# 安装python
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+#安装第三方包 grpcio、protobuf、grpcio_tools、psycopg2
+RUN pip install grpcio
+RUN pip install protobuf
+RUN pip install grpcio_tools
+RUN pip install psycopg2-binary
+
+COPY *.sh /app/bin/
+COPY *.py /app/bin/
+COPY *.proto /app/bin/
+
+#controld version 202208
+RUN cd /app/bin/ \
+    && mkdir controld \
+    && python3 -m grpc_tools.protoc  --python_out=.  --grpc_python_out=.  -I. control.proto  -I=controld \
+    && chmod +x run.sh
+
+ENV CSST_SCALEBOX_HOST=10.255.2.11 \
+    CSST_SCALEBOX_PORT=5432 \
+    CSST_SCALEBOX_USER=scalebox \
+    CSST_SCALEBOX_PWD=changeme \
+    CSST_SCALEBOX_DATABASE=scalebox \
+    CSST_PIPELINE_GRPC_SERVER="10.255.2.11:50051" \
+    CSST_ADML1_APPID="1"

L1/dockerfiles/admL1/Makefile

+IMAGE_NAME:=hub.cstcloud.cn/csst/adml1:dev
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+	docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)

L1/dockerfiles/admL1/admL1.py

+import grpc
+import control_pb2_grpc
+import control_pb2
+import psycopg2
+import os
+import sys
+#递归调用
+class admL1Api():
+    def __init__(self):
+        self.body = ""
+    
+    def adminLevelOne(self,body):
+        
+        #获取obsid
+        obsid=body.split(",",1)[0]
+        #获取moduleid
+        moduleid=body.split(",",1)[1]
+        #根据moduleid获取模块内部jobid
+        shost = os.getenv('CSST_SCALEBOX_HOST')
+        sport = os.getenv('CSST_SCALEBOX_PORT')
+        suser = os.getenv('CSST_SCALEBOX_USER')
+        spwd = os.getenv('CSST_SCALEBOX_PWD')
+        sdb = os.getenv('CSST_SCALEBOX_DATABASE')
+        # conn = psycopg2.connect(host="10.255.2.11",port=5432,user="scalebox",password="changeme",database="scalebox")
+        conn = psycopg2.connect(host=shost,port=sport,user=suser,password=spwd,database=sdb)
+        cursor = conn.cursor()
+        #获取流水线id
+        app_id=int(os.getenv('CSST_ADML1_APPID'))
+        sql ="SELECT id FROM t_job where name = %s and app= %s;"
+        params = (moduleid,app_id,)
+        cursor.execute(sql,params)
+        rows = cursor.fetchone()
+        jobId=rows[0]
+        conn.commit()
+        cursor.close()
+        conn.close()
+        #调用grpc的SendJobMessage
+        channel = grpc.insecure_channel(os.getenv('CSST_PIPELINE_GRPC_SERVER'))
+        stub = control_pb2_grpc.ControlServiceStub(channel)
+        test = control_pb2.JobKey()
+        test.cross_app_job_id = int(jobId)
+        test.key_text = obsid
+        reflag = stub.SendJobMessage(test)
+        print(reflag.value)
+        return reflag.value
+if __name__ == '__main__':
+    parameter = sys.argv
+    body=parameter[1]
+    w=admL1Api()
+    w.adminLevelOne(body)       
+
+    
\ No newline at end of file

L1/dockerfiles/admL1/control.proto

+syntax = "proto3";
+package scalebox;
+
+option go_package="./;scalebox";
+option java_multiple_files = true;
+
+// option java_outer_classname = "ControlProto";
+option java_package = "cn.scalebox.gprc";
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/empty.proto";
+import "google/protobuf/wrappers.proto";
+
+
+service ControlService {
+  // //////////////////////////////////////////////////////////////////
+  //      agent client
+  // //////////////////////////////////////////////////////////////////
+
+  //  input : slot id('ON')
+  // return : job-key of the task, id==0(NULL)
+  //  task id : 'READY/-1' -> 'QUEUED/-2'
+  rpc GetNextTask(google.protobuf.Int32Value) returns (TaskItem);
+
+  //  input : task id('QUEUED/-2')
+  //  return : ret_code : 0(OK), -1(task NOT FOUND), -2(WRONG STATUS)
+  //  task_status_code:'QUEUED'/-2 -> 'RUNNING'/-3
+  rpc SetTaskStarted(google.protobuf.Int64Value) returns (google.protobuf.Int32Value);
+
+  //  input : TaskExecMessage
+  //  return : ret_code : 0(OK), -1(task NOT FOUND), -2(task WRONG STATUS)
+  //  task_status_code:'RUNNING'/-3 -> 'OK'/0, ...
+  rpc SetTaskFinished(TaskExecMessage) returns (google.protobuf.Int32Value);
+
+  //  slot exit automatically, called in agent side
+  //  input : slot id
+  //  return : ret_code : 0(OK), -1(slot NOT FOUND), -2(slot WRONG STATUS)
+  //  slot : 'ON' -> 'OFF'
+  rpc SetSlotTerminated(google.protobuf.Int32Value) returns (google.protobuf.Int32Value);
+
+  // //////////////////////////////////////////////////////////////////
+  //      actuator client
+  // //////////////////////////////////////////////////////////////////
+
+  //  input: token_string ?
+  //  job:'RUNNING' && slot:'READY'
+  rpc GetRunnableSlotList(google.protobuf.Empty) returns (CommandList);
+
+  //  input : slot id
+  //  return : ret_code : 0(OK), -1(slot NOT FOUND), -2(slot WRONG STATUS)
+  //  slot: 'READY' -> 'ON'
+  rpc SetSlotInitialized(google.protobuf.Int32Value) returns (google.protobuf.Int32Value);
+
+  //  job:'PAUSED'/'ARCHIVED' && slot:'ON'
+  rpc GetTerminableSlotList(google.protobuf.Empty) returns (CommandList);
+
+
+  //  job:'RUNNING' && worker:'PAUSED'
+  rpc GetRunnableWorkerList(google.protobuf.Empty) returns (CommandList);
+
+  //  input : worker id
+  //  return : ret_code : 0(OK), -1(worker NOT FOUND), -2(worker WRONG STATUS)
+  //  worker: 'PAUSED' -> 'RUNNING'
+  rpc SetWorkerInitialized(google.protobuf.Int32Value) returns (google.protobuf.Int32Value);
+
+  //  job:'PAUSED' && worker:'RUNNING'
+  rpc GetTerminableWorkerList(google.protobuf.Empty) returns (CommandList);
+
+  //  input : worker id, return ret_code;
+  //  return : ret_code : 0(OK), -1(worker NOT FOUND), -2(worker WRONG STATUS)
+  //  worker : 'RUNNING' -> 'PAUSED'
+  rpc SetWorkerTerminated(google.protobuf.Int32Value) returns (google.protobuf.Int32Value);
+
+  // //////////////////////////////////////////////////////////////////
+  //      app client , called by user app.
+  // //////////////////////////////////////////////////////////////////
+
+  //  send task-key to next job in current pipeline
+  //  return : ret_code : 0(OK), -1(job NOT FOUND)
+  //  task_status_code : 'INITIAL'/-9
+  rpc SendJobMessage(JobKey) returns (google.protobuf.Int32Value);
+  // rpc SendToNextJob(JobKey) returns (google.protobuf.Int32Value);
+
+  rpc SendJobMessages(JobKeys) returns (google.protobuf.Int32Value);
+}
+
+message JobKey {
+  message JobIdRef{
+    // qualified name of job
+    string sink_job_name=1;
+    int32 current_job_id=2;
+  }
+  oneof jobId {
+    int32 cross_app_job_id=1;
+    JobIdRef builtin_job_id=2;
+  }
+  string key_text=10;
+}
+
+message JobKeys {
+  // qualified name of job
+  string next_job_name=1;
+  repeated string key_texts=2;
+  int32 current_job_id=3;
+  // OR worker_id ?
+}
+
+message TaskItem {
+  int64 id=1;
+  string key=2;
+}
+
+message Command {
+  // primary key of slot/worker table
+  int32 id=1;
+  string host=2;
+  string command_text=3;
+}
+
+message CommandList {
+  repeated Command command=1;
+}
+
+message TaskExecMessage {
+  int32 slot=1;
+  int32 status_code=2;
+  int64 task_id=3;
+  int64 input_bytes=4;
+  int64 output_bytes=5;
+  google.protobuf.Timestamp t0=6;
+  google.protobuf.Timestamp t1=7;
+  google.protobuf.Timestamp t2=8;
+  google.protobuf.Timestamp t3=9;
+  google.protobuf.Timestamp t4=10;
+  repeated google.protobuf.Timestamp time_arr=11;
+  string sys_out=12;
+  string app_out=13;
+}

L1/dockerfiles/admL1/run.sh

+#!/bin/bash
+
+python3 /app/bin/admL1.py $1
\ No newline at end of file

L1/dockerfiles/admL1/test.py

+import os
+os.environ['CSST_PIPELINE_GRPC_SERVER']="10.255.2.11:50051"
+os.environ['app_id']="1"
+
+from admL1 import admL1Api
+body="brikd,moudle1"
+admL1Api().adminLevelOne(body)

L1/dockerfiles/module1/Dockerfile

+FROM hub.cstcloud.cn/scalebox/agent
+
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+# 安装python
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY *.sh /app/bin/
+COPY *.py /app/bin/
+
+RUN cd /app/bin/ \
+    && chmod +x run.sh

L1/dockerfiles/module1/Makefile

+IMAGE_NAME:=hub.cstcloud.cn/csst/module1:dev
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+	docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)

L1/dockerfiles/module1/module1.py

+import os
+import sys
+
+if __name__ == '__main__':
+    parameter = sys.argv
+    body=parameter[1]
+    print(body)
\ No newline at end of file

L1/dockerfiles/module1/run.sh

+#!/bin/bash
+
+python /app/bin/module1.py $1
+send-message $1
\ No newline at end of file

L1/dockerfiles/module2/Dockerfile

+FROM hub.cstcloud.cn/scalebox/agent
+
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+# 安装python
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY *.sh /app/bin/
+COPY *.py /app/bin/
\ No newline at end of file

L1/dockerfiles/module2/Makefile

+IMAGE_NAME:=hub.cstcloud.cn/csst/module2:dev
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+	docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)

L1/dockerfiles/module2/module2.py

+import os
+import sys
+
+if __name__ == '__main__':
+    parameter = sys.argv
+    body=parameter[1]
+    print(body)
\ No newline at end of file

L1/dockerfiles/module2/run.sh

+#!/bin/bash
+
+python /app/bin/module2.py $1
+send-message $1
\ No newline at end of file

L1/pipelines/Makefile

+CLUSTER=csst
+
+all: reset parse up-app
+
+reset:
+	cd ${HOME}/docker-scalebox/clusters/$(CLUSTER) && make && cd -
+
+up-app:
+	docker exec -t database psql -Uscalebox -c "UPDATE t_app SET status='RUNNING' WHERE status='INITIAL'"
+
+parse:
+	CLUSTER=$(CLUSTER)  \
+	${HOME}/docker-scalebox/bin/parse.sh app.yaml
+
+start:
+	docker run -it --rm --network host -e JOB_ID=17 -e GRPC_SERVER=localhost:50051 hub.cstcloud.cn/scalebox/base bash -c ' send-job-message filelist . ' 
+
+down:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) down
+
+list:
+	make -C ${HOME}/docker-scalebox/clusters/$(CLUSTER) list
+

L1/pipelines/app.yaml

+name: csst-1.app.process
+label: csst-adminL1
+output_basedir: /data/csst-adminL1
+cluster: csst
+
+jobs:
+  admL1:
+    label: admin管理模块1
+    base_image: hub.cstcloud.cn/csst/adml1:dev
+    type: LONG-RUN-JOB
+    schedule_mode: HEAD
+    ## key_example: Dec+5841_09_04/20200817
+    key_group_regex: ^(Dec.+/[0-9]+)$
+#    environments:
+      # - RSYNC_SRC=scalebox@159.226.237.136
+#      - ROOT_PATH=${ROOT_PATH}
+      # - REGEX_FILTER=^.+_000[0-7]_2bit.fits$
+    command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    sink_jobs:
+      - module1
+      #- module2
+  module1:
+    label: 模块一
+    base_image: hub.cstcloud.cn/csst/module1:dev
+    type: LONG-RUN-JOB
+    schedule_mode: HEAD
+    ## key_example: Dec+5841_09_04/20200817
+    key_group_regex: ^(Dec.+/[0-9]+)$
+#    paths:
+#      - ${ROOT_PATH}:${ROOT_PATH}:ro
+#    environments:
+      # - RSYNC_SRC=scalebox@159.226.237.136
+#      - ROOT_PATH=${ROOT_PATH}
+      # - REGEX_FILTER=^.+_000[0-7]_2bit.fits$
+    command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+    # sink_jobs:
+    #   - admL1  
+  # module2:
+  #   label: 模块二
+  #   base_image: hub.cstcloud.cn/csst/module2:dev
+  #   type: LONG-RUN-JOB
+  #   schedule_mode: HEAD
+  #   ## key_example: Dec+5841_09_04/20200817
+  #   key_group_regex: ^(Dec.+/[0-9]+)$
+  #   paths:
+  #     - ${ROOT_PATH}:${ROOT_PATH}:ro
+  #   environments:
+  #     # - RSYNC_SRC=scalebox@159.226.237.136
+  #     - ROOT_PATH=${ROOT_PATH}
+  #     # - REGEX_FILTER=^.+_000[0-7]_2bit.fits$
+  #   command: docker run -d --network=host %ENVS% %VOLUMES% %IMAGE%
+  #   # sink_jobs:
+  #   #   - admL1        
+
+      

L2/dockerfiles/admL2/Dockerfile

+FROM hub.cstcloud.cn/scalebox/agent
+
+LABEL maintainer="Xiaoli Zhang<zhangxiaoli@cnic.cn>"
+
+# 安装python
+RUN apt-get update \
+    && apt-get install -y python3 python3-pip \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+#安装第三方包 grpcio、protobuf、grpcio_tools
+RUN pip install grpcio
+RUN pip install protobuf
+RUN pip install grpcio_tools
+
+COPY *.sh /app/bin/
+COPY *.py /app/bin/
+COPY *.proto /app/bin/
+
+#controld version 202208
+RUN cd /app/bin/ \
+    && mkdir controld \
+    && python3 -m grpc_tools.protoc  --python_out=.  --grpc_python_out=.  -I. control.proto  -I=control
+
+ENV CSST_ADML2_JOB_ID="12"  \
+    CSST_PIPELINE_GRPC_SERVER="10.255.2.11:50051"
\ No newline at end of file

L2/dockerfiles/admL2/Makefile

+IMAGE_NAME:=hub.cstcloud.cn/csst/adml2:dev
+
+build:
+	docker build --network=host -t $(IMAGE_NAME) .
+
+dist:
+	docker save $(IMAGE_NAME) | zstdmt | pv | ssh c0 'zstd -d | docker load'
+
+push:
+	docker push $(IMAGE_NAME)
+run:
+	docker run -it --entrypoint bash $(IMAGE_NAME)
+down:
+	docker stop $(IMAGE_NAME)