Simplify by using HTTP based service discovery

Micah Young · Micah Young · commit a5ee709cd61f · 2017-06-27T00:36:02.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+deploy/
+quorum-examples/
diff --git a/README.md b/README.md
@@ -52,44 +52,16 @@
    cp -r quorum-pcf-7nodes/run-scripts/* deploy/
    ```
 
-1. Create an empty user-provided-service for the apps
+1. Push all apps, unstarted
    ```bash
-   cf create-user-provided-service ip-service -p '{}'
-   ```
-
-1. Deploy the bootnode
-   ```bash
-   cf push bootnode -p deploy/ -f quorum-pcf-7nodes/manifests/bootnode-manifest.yml
-   ```
-
-1. Update the service with the bootnode's IP
-   ```bash
-   cf update-user-provided-service ip-service -p "{
-     \"BOOTNODE_IP\": \"$(cf ssh bootnode -c "hostname --ip-address")\"
-   }"
-   ```
-
-1. Deploy node-1
-   ```
-   cf push node-1 -p deploy/ -f quorum-pcf-7nodes/manifests/node1-manifest.yml
-   ```
-
-1. Update the service with the bootnode's and node-1's IPs
-   ```bash
-   cf update-user-provided-service ip-service -p "{
-     \"BOOTNODE_IP\": \"$(cf ssh bootnode -c 'hostname --ip-address')\",
-     \"OTHER_NODE_IP\": \"$(cf ssh node-1 -c 'hostname --ip-address')\"
-   }"
-   ```
-
-1. Push the remaining apps
-   ```bash
-   cf push node-2 -p deploy/ -f quorum-pcf-7nodes/manifests/node2-manifest.yml
-   cf push node-3 -p deploy/ -f quorum-pcf-7nodes/manifests/node3-manifest.yml
-   cf push node-4 -p deploy/ -f quorum-pcf-7nodes/manifests/node4-manifest.yml
-   cf push node-5 -p deploy/ -f quorum-pcf-7nodes/manifests/node5-manifest.yml
-   cf push node-6 -p deploy/ -f quorum-pcf-7nodes/manifests/node6-manifest.yml
-   cf push node-7 -p deploy/ -f quorum-pcf-7nodes/manifests/node7-manifest.yml
+   cf push bootnode -p deploy/ -f quorum-pcf-7nodes/manifests/bootnode-manifest.yml --no-start
+   cf push node-1   -p deploy/ -f quorum-pcf-7nodes/manifests/node1-manifest.yml    --no-start
+   cf push node-2   -p deploy/ -f quorum-pcf-7nodes/manifests/node2-manifest.yml    --no-start
+   cf push node-3   -p deploy/ -f quorum-pcf-7nodes/manifests/node3-manifest.yml    --no-start
+   cf push node-4   -p deploy/ -f quorum-pcf-7nodes/manifests/node4-manifest.yml    --no-start
+   cf push node-5   -p deploy/ -f quorum-pcf-7nodes/manifests/node5-manifest.yml    --no-start
+   cf push node-6   -p deploy/ -f quorum-pcf-7nodes/manifests/node6-manifest.yml    --no-start
+   cf push node-7   -p deploy/ -f quorum-pcf-7nodes/manifests/node7-manifest.yml    --no-start
    ```
 
 1. Add all the container to container networking rules (copy, paste, wait...)
@@ -304,24 +276,36 @@
    cf list-access --app bootnode | wc -l #=> 32
    ```
 
+1. Start all apps
+   ```bash
+   cf start bootnode
+   cf start node-1
+   cf start node-2
+   cf start node-3
+   cf start node-4
+   cf start node-5
+   cf start node-6
+   cf start node-7
+   ```
+
 1. SSH into node-1 to run the `script1.js`
    * on your command line:
       ```bash
       cf ssh node-1
       ```
-   
+
    * ... within the container:
       ```sh
       cd app
       export PATH=$PATH:`pwd`/bin
       geth attach qdata/dd1/geth.ipc
       ```
-   
+
    * ... in the solc interpreter:
       ```js
       loadScript('script1.js')
       ```
-      
+
    * ... you should see the output if successful
       ```js
       Contract transaction send: TransactionHash: 0x541da6399119e66687fe5edada5162d586c56271800d626e33cf9e7ae811d8f6 waiting to be mined...
diff --git a/manifests/bootnode-manifest.yml b/manifests/bootnode-manifest.yml
@@ -3,9 +3,10 @@ applications:
 - name: bootnode
   buildpack: binary_buildpack
   health-check-type: process
-  no-route: true
   memory: 512MB
   command: ./bootnode-start.sh
   env:
     BOOTNODE_PORT: 33445
     BOOTNODE_KEYHEX: "77bd02ffa26e3fb8f324bda24ae588066f1873d95680104de5bc2db9e7b2e510"
+  routes:
+  - route: bootnode-ip.cfapps.io
diff --git a/manifests/node-1-manifest.yml b/manifests/node-1-manifest.yml
@@ -4,16 +4,16 @@ applications:
   buildpack: binary_buildpack
   health-check-type: process
   memory: 512MB
-  no-route: true
   command: ./primary-node-start.sh
-  services:
-  - ip-service
   env:
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    BOOTNODE_PORT: 33445
     NETID: 87234
     BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
-    BOOTNODE_PORT: 33445
     NODE_PORT: 9000
     RPC_PORT: 22000
     LISTEN_PORT: 21000
     PRIVATE_CONFIG_FILE: tm1.conf
     DATA_DIR: qdata/dd1
+  routes:
+  - route: othernode-ip.cfapps.io
diff --git a/manifests/node-2-manifest.yml b/manifests/node-2-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 1024MB
   no-route: true
   command: ./blockmaker-node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9001
     RPC_PORT: 22001
     LISTEN_PORT: 21001
diff --git a/manifests/node-3-manifest.yml b/manifests/node-3-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 512MB
   no-route: true
   command: ./node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9002
     RPC_PORT: 22002
     LISTEN_PORT: 21002
diff --git a/manifests/node-4-manifest.yml b/manifests/node-4-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 512MB
   no-route: true
   command: ./voter-node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9003
     RPC_PORT: 22003
     LISTEN_PORT: 21003
diff --git a/manifests/node-5-manifest.yml b/manifests/node-5-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 512MB
   no-route: true
   command: ./voter-node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9004
     RPC_PORT: 22004
     LISTEN_PORT: 21004
diff --git a/manifests/node-6-manifest.yml b/manifests/node-6-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 512MB
   no-route: true
   command: ./node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9005
     RPC_PORT: 22005
     LISTEN_PORT: 21005
diff --git a/manifests/node-7-manifest.yml b/manifests/node-7-manifest.yml
@@ -6,13 +6,13 @@ applications:
   memory: 512MB
   no-route: true
   command: ./node-start.sh
-  services:
-  - ip-service
   env:
-    NETID: 87234
-    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    BOOTNODE_IP_ROUTE: bootnode-ip.cfapps.io
+    OTHER_NODE_IP_ROUTE: othernode-ip.cfapps.io
     BOOTNODE_PORT: 33445
     OTHER_NODE_PORT: 9000
+    BOOTNODE_HASH: 61077a284f5ba7607ab04f33cfde2750d659ad9af962516e159cf6ce708646066cd927a900944ce393b98b95c914e4d6c54b099f568342647a1cd4a262cc0423
+    NETID: 87234
     NODE_PORT: 9006
     RPC_PORT: 22006
     LISTEN_PORT: 21006
diff --git a/run-scripts/blockmaker-node-start.sh b/run-scripts/blockmaker-node-start.sh
@@ -13,13 +13,9 @@ true ${LISTEN_PORT:?"!"}
 true ${NODE_PORT:?"!"}
 true ${VCAP_SERVICES:?"!"}
 
-while read ENV_PAIR; do export "${ENV_PAIR}"; done \
-  < <(echo $VCAP_SERVICES | jq -r '.["user-provided"] | .[].credentials | to_entries[] | "\(.key)=\(.value)"')
-
-true ${BOOTNODE_IP:?"!missing from VCAP_SERVICES"}
-true ${OTHER_NODE_IP:?"!missing from VCAP_SERVICES"}
-
-NODE_IP=$(hostname --ip-address)
+NODE_IP=$CF_INSTANCE_INTERNAL_IP
+BOOTNODE_IP=$(curl -f -s $BOOTNODE_IP_ROUTE)
+OTHER_NODE_IP=$(curl -f -s $OTHER_NODE_IP_ROUTE)
 echo "NODE_IP=$NODE_IP"
 echo "BOOTNODE_IP=$BOOTNODE_IP"
 echo "OTHER_NODE_IP=$OTHER_NODE_IP"
@@ -58,4 +54,6 @@ PRIVATE_CONFIG=$PRIVATE_CONFIG_FILE \
   --minblocktime 2 \
   --maxblocktime 5 \
   --verbosity 4 \
-;
+&
+
+wait
diff --git a/run-scripts/bootnode-start.sh b/run-scripts/bootnode-start.sh
@@ -5,10 +5,19 @@ set -e
 true ${BOOTNODE_PORT:?"!"}
 true ${BOOTNODE_KEYHEX:?"!"}
 
+NODE_IP=$CF_INSTANCE_INTERNAL_IP
+echo "NODE_IP=$NODE_IP"
+
 export PATH=$PATH:`pwd`/bin
 
+while true; do
+  nc -l $PORT < <(echo -e "HTTP/1.1 200 OK\n\n$NODE_IP") > /dev/null
+done &
+
 bootnode \
   --nodekeyhex "$BOOTNODE_KEYHEX" \
   --addr="0.0.0.0:$BOOTNODE_PORT" \
   --verbosity 9 \
-;
+&
+
+wait
diff --git a/run-scripts/node-start.sh b/run-scripts/node-start.sh
@@ -13,13 +13,9 @@ true ${LISTEN_PORT:?"!"}
 true ${NODE_PORT:?"!"}
 true ${VCAP_SERVICES:?"!"}
 
-while read ENV_PAIR; do export "${ENV_PAIR}"; done \
-  < <(echo $VCAP_SERVICES | jq -r '.["user-provided"] | .[].credentials | to_entries[] | "\(.key)=\(.value)"')
-
-true ${BOOTNODE_IP:?"!missing from VCAP_SERVICES"}
-true ${OTHER_NODE_IP:?"!missing from VCAP_SERVICES"}
-
-NODE_IP=$(hostname --ip-address)
+NODE_IP=$CF_INSTANCE_INTERNAL_IP
+BOOTNODE_IP=$(curl -f -s $BOOTNODE_IP_ROUTE)
+OTHER_NODE_IP=$(curl -f -s $OTHER_NODE_IP_ROUTE)
 echo "NODE_IP=$NODE_IP"
 echo "BOOTNODE_IP=$BOOTNODE_IP"
 echo "OTHER_NODE_IP=$OTHER_NODE_IP"
@@ -49,4 +45,6 @@ PRIVATE_CONFIG=$PRIVATE_CONFIG_FILE \
   --rpcport $RPC_PORT \
   --port $LISTEN_PORT \
   --verbosity 4 \
-;
+&
+
+wait
diff --git a/run-scripts/primary-node-start.sh b/run-scripts/primary-node-start.sh
@@ -5,25 +5,26 @@ set -e
 true ${NETID:?"!"}
 true ${BOOTNODE_HASH:?"!"}
 true ${BOOTNODE_PORT:?"!"}
+true ${BOOTNODE_IP_ROUTE:?"!"}
 true ${PRIVATE_CONFIG_FILE:?"!"}
 true ${DATA_DIR:?"!"}
 true ${RPC_PORT:?"!"}
 true ${LISTEN_PORT:?"!"}
 true ${NODE_PORT:?"!"}
-true ${VCAP_SERVICES:?"!"}
 
-while read ENV_PAIR; do export "${ENV_PAIR}"; done \
-  < <(echo $VCAP_SERVICES | jq -r '.["user-provided"] | .[].credentials | to_entries[] | "\(.key)=\(.value)"')
-
-true ${BOOTNODE_IP:?"!missing from VCAP_SERVICES"}
-
-NODE_IP=$(hostname --ip-address)
+NODE_IP=$CF_INSTANCE_INTERNAL_IP
+BOOTNODE_IP=$(curl -f -s $BOOTNODE_IP_ROUTE)
 echo "NODE_IP=$NODE_IP"
 echo "BOOTNODE_IP=$BOOTNODE_IP"
 
+while true; do
+  nc -l $PORT < <(echo -e "HTTP/1.1 200 OK\n\n$NODE_IP") > /dev/null
+done &
+
 sed -ibak "s|url = .*|url = \"http://$NODE_IP:$NODE_PORT/\"|" $PRIVATE_CONFIG_FILE
 sed -ibak "s|port = .*|port = $NODE_PORT|" $PRIVATE_CONFIG_FILE
 
+
 export PATH=$PATH:`pwd`/bin
 export LD_LIBRARY_PATH=`pwd`/bin
 
@@ -49,4 +50,6 @@ PRIVATE_CONFIG=$PRIVATE_CONFIG_FILE \
   --unlock 0 \
   --password passwords.txt \
   --verbosity 4 \
-;
+&
+
+wait
diff --git a/run-scripts/voter-node-start.sh b/run-scripts/voter-node-start.sh
@@ -13,13 +13,9 @@ true ${LISTEN_PORT:?"!"}
 true ${NODE_PORT:?"!"}
 true ${VCAP_SERVICES:?"!"}
 
-while read ENV_PAIR; do export "${ENV_PAIR}"; done \
-  < <(echo $VCAP_SERVICES | jq -r '.["user-provided"] | .[].credentials | to_entries[] | "\(.key)=\(.value)"')
-
-true ${BOOTNODE_IP:?"!missing from VCAP_SERVICES"}
-true ${OTHER_NODE_IP:?"!missing from VCAP_SERVICES"}
-
-NODE_IP=$(hostname --ip-address)
+NODE_IP=$CF_INSTANCE_INTERNAL_IP
+BOOTNODE_IP=$(curl -f -s $BOOTNODE_IP_ROUTE)
+OTHER_NODE_IP=$(curl -f -s $OTHER_NODE_IP_ROUTE)
 echo "NODE_IP=$NODE_IP"
 echo "BOOTNODE_IP=$BOOTNODE_IP"
 echo "OTHER_NODE_IP=$OTHER_NODE_IP"
@@ -53,4 +49,6 @@ PRIVATE_CONFIG=$PRIVATE_CONFIG_FILE \
   --voteaccount $VOTE_ACCOUNT \
   --votepassword "" \
   --verbosity 4 \
-;
+&
+
+wait
