owlet42 · owlet42 · Jun 29, 2021 · Jul 8, 2021 · Jul 13, 2021 · Jul 19, 2021
diff --git a/docs/Use_polling.md b/docs/Use_polling.md
@@ -0,0 +1,50 @@
+# How to use KubeFATE to configure FATE polling
+
+FATE's polling is divided into server side and client side.
+
+**what's polling?**
+
+In Federated Learning scenario, a party (or site) will send and receive data from / to other parties (or sites).
+
+Normally, *Two-Way mode is recommended*. In this mode, a party (site) needs to listen to a public port and provide it to exchange or other direct connected parties (sites).
+
+In some specific cases where a party (or site) is not allowed or not willing to provide a public port, Polling mode can be used. In Polling mode, the pattern of sending data is the same as Two-Way mode. But when receiving data, this specific party (or site) will be acting as a polling client actively polls (i.e. fetches) data from the nearest RollSite.
+
+## server
+
+If you want to be the server side of polling, you can configure it like this:
+
+```bash
+# cluster.yaml
+
+...
+rollsite: 
+  ...
+  polling:
+    enabled: true
+    type: server
+    clientList:                 # polling client list
+    - partID: 9999    
+    - partID: 10000
+    concurrency: 60
+```
+
+
+
+## client
+
+If you want to be the client side of polling, you can configure it like this:
+
+```bash
+# cluster.yaml
+
+...
+rollsite: 
+  ...
+  polling:
+    enabled: true
+    type: server
+    server:
+      ip: 192.168.100.1           # polling server ip
+      port: 9370                  # polling server port
+```
diff --git a/docs/configurations/FATE_cluster_configuration.md b/docs/configurations/FATE_cluster_configuration.md
@@ -65,18 +65,24 @@
 ### rollsite mappings
 It is used to declare the `rollsite ` module in the FATE cluster to be deployed.
 
-| Name         | subitem   | Type      | Description                                                  |
-| ------------ | --------- | --------- | ------------------------------------------------------------ |
-| type         |           | scalars   | Kubernetes ServiceTypes, default is NodePort.                |
-| nodePort     |           | scalars   | The port used by `proxy` module's kubernetes service, default range: 30000-32767. |
-| partyList    |           | sequences | If this FATE cluster is exchange cluster, partyList is all party's sequences of all parties proxy address. If this FATE cluster is one of participants, delete this configuration item. |
-| partyList    | partyId   | scalars   | Participant FATE cluster party ID.                           |
-| partyList    | partyIp   | scalars   | Participant FATE cluster IP.                                 |
-| partyList    | partyPort | scalars   | Participant FATE cluster port.                               |
-| exchange     |           | mappings  | FATE cluster `exchange` module's ip and port.                |
-| exchange     | ip        | mappings  | FATE cluster `exchange` module's ip. .                       |
-| exchange     | port      | mappings  | FATE cluster `exchange` module's port.                       |
-| nodeSelector |           | mappings  | kubernetes nodeSelector.                                     |
+| Name         | subitem     | Type      | Description                                                  |
+| ------------ | ----------- | --------- | ------------------------------------------------------------ |
+| type         |             | scalars   | Kubernetes ServiceTypes, default is NodePort.                |
+| nodePort     |             | scalars   | The port used by `proxy` module's kubernetes service, default range: 30000-32767. |
+| partyList    |             | sequences | If this FATE cluster is exchange cluster, partyList is all party's sequences of all parties proxy address. If this FATE cluster is one of participants, delete this configuration item. |
+| partyList    | partyId     | scalars   | Participant FATE cluster party ID.                           |
+| partyList    | partyIp     | scalars   | Participant FATE cluster IP.                                 |
+| partyList    | partyPort   | scalars   | Participant FATE cluster port.                               |
+| exchange     |             | mappings  | FATE cluster `exchange` module's ip and port.                |
+| exchange     | ip          | mappings  | FATE cluster `exchange` module's ip. .                       |
+| exchange     | port        | mappings  | FATE cluster `exchange` module's port.                       |
+| nodeSelector |             | mappings  | kubernetes nodeSelector.                                     |
+| polling      |             |           | rollsite support polling                                     |
+| polling      | enabled     |           | enable polling                                               |
+| polling      | type        |           | polling type (server/client)                                 |
+| polling      | server      |           | if type choose client, you need a polling server.            |
+| polling      | clientList  |           | if type choose server, this rollsite serve for clientList.   |
+| polling      | concurrency |           | if type choose server, polling client concurrency.           |
 
 FATE cluster has two deployment modes: with exchange and without exchange.
 #### Exchange mode:

diff --git a/helm-charts/FATE/templates/eggroll-config.yaml b/helm-charts/FATE/templates/eggroll-config.yaml
@@ -82,3 +82,16 @@ data:
     eggroll.rollsite.push.long.retry=2
     eggroll.rollsite.push.batches.per.stream=10
     eggroll.rollsite.adapter.sendbuf.size=100000
+
+    # polling
+    # {{ .Values.modules.rollsite.polling.enabled }}
+    # {{ .Values.modules.rollsite.polling.type }}
+    # 
+    {{- if and .Values.modules.rollsite.polling.enabled ( eq (print .Values.modules.rollsite.polling.type) "client" )  }}
+    eggroll.rollsite.polling.client.enabled=true
+    {{- end }}
+
+    {{- if and .Values.modules.rollsite.polling.enabled ( eq (print .Values.modules.rollsite.polling.type) "server" )  }}
+    eggroll.rollsite.polling.server.enabled=true
+    eggroll.rollsite.polling.concurrency= {{ .Values.modules.rollsite.polling.concurrency | default 50 }}
+    {{- end }}
diff --git a/helm-charts/FATE/templates/python-spark.yaml b/helm-charts/FATE/templates/python-spark.yaml
@@ -351,9 +351,9 @@ spec:
       {{- end }}
 {{- if .Values.modules.python.serviceAccountName }}
       serviceAccountName: {{ .Values.modules.python.serviceAccountName }}
-{{- else -}}
+{{- else }}
       serviceAccountName: {{ template "serviceAccountName" . }}
-{{- end -}}
+{{- end }}
       restartPolicy: Always
       volumes:
         - name: eggroll-confs

diff --git a/helm-charts/FATE/templates/rollsite-module.yaml b/helm-charts/FATE/templates/rollsite-module.yaml
@@ -24,12 +24,23 @@ data:
             "default": {
                 "default": [
                     {
-                      {{- with .Values.exchange }}
-                        "ip": "{{ .partyIp }}",
-                        "port": {{ .partyPort }}
+                      {{- if .Values.exchange }}
+                        "ip": "{{ .Values.exchange.partyIp }}",
+                        "port": {{ .Values.exchange.partyPort }}
                       {{- else }}
+
+                        {{- if and .Values.modules.rollsite.polling.enabled ( eq (print .Values.modules.rollsite.polling.type) "client" ) }}
+
+                        {{- with .Values.modules.rollsite.polling.server }}
+                        "ip": "{{ .ip }}",
+                        "port": {{ .port }}
+                        {{- end }}
+
+                        {{- else }}
                         "ip": "rollsite",
                         "port": 9370
+                        {{- end }}
+
                       {{- end }}
                     }
                 ]
@@ -43,6 +54,23 @@ data:
                     }
                 ]
             },
+            {{- end }}
+
+            {{- if and .Values.modules.rollsite.polling.enabled ( eq (print .Values.modules.rollsite.polling.type) "server" ) }}
+
+            {{- range .Values.modules.rollsite.polling.clientList}}
+            "{{ .partID }}": 
+            {
+                "default": [
+                    {
+                        "port": -1,
+                        "ip": "",
+                        "is_polling": true
+                    }
+                ]
+            },
+            {{- end }}
+
             {{- end }}
             "{{ .Values.partyId }}": {
                 "fateflow": [

diff --git a/helm-charts/FATE/values-template-example.yaml b/helm-charts/FATE/values-template-example.yaml
@@ -54,6 +54,16 @@ backend: eggroll
   #   value: "value"
   #   effect: "NoSchedule"
   # affinity:
+  # polling:
+    # enabled: true
+    # type: server
+    # # server:
+      # # ip: 192.168.100.1
+      # # port: 9370
+    # clientList:
+    # - partID: 9999
+    # concurrency: 49
+
 
 # lbrollsite:
   # type: NodePort
@@ -170,11 +180,11 @@ backend: eggroll
   # size: 1Gi
 
 
-externalMysqlIp: mysql1
-externalMysqlPort: 33060
-externalMysqlDatabase: eggroll_meta1
-externalMysqlUser: fate1
-externalMysqlPassword: fate_dev1
+# externalMysqlIp: mysql1
+# externalMysqlPort: 33060
+# externalMysqlDatabase: eggroll_meta1
+# externalMysqlUser: fate1
+# externalMysqlPassword: fate_dev1
 
 
 # servingIp: 192.168.0.1

diff --git a/helm-charts/FATE/values-template.yaml b/helm-charts/FATE/values-template.yaml
@@ -87,6 +87,21 @@ modules:
     affinity:
 {{ toYaml . | indent 6 }}
     {{- end }}
+    {{- with .polling }}
+    polling:
+      enabled: {{ .enabled }}
+      type: {{ .type }}
+      {{- with .server }}
+      server:
+        ip: {{ .ip }}
+        port: {{ .port }}
+      {{- end }}
+      {{- with .clientList }}
+      clientList:
+{{ toYaml . | indent 6 }}
+      {{- end }}
+      concurrency: {{ .concurrency }}
+    {{- end }}
     {{- end }}
 
 

diff --git a/helm-charts/FATE/values.yaml b/helm-charts/FATE/values.yaml
@@ -51,6 +51,19 @@ modules:
     nodeSelector:
     tolerations:
     affinity:
+    polling:
+      enabled: false
+
+      # type: client
+      # server:
+        # ip: 192.168.9.1
+        # port: 9370
+
+      # type: server
+      # clientList:
+      # - partID: 9999
+      # concurrency: 50
+
   lbrollsite:
     include: true
     ip: rollsite
@@ -148,6 +161,11 @@ modules:
     nodeSelector:
     tolerations:
     affinity:
+    subPath: "client"
+    existingClaim: ""
+    storageClass: "client"
+    accessMode: ReadWriteOnce
+    size: 1Gi
   mysql: 
     include: true
     type: ClusterIP