CORE-36133 Adding Error handling for failures in quick start script | updated README

Author: Harry S

README.md

@@ -1,8 +1,9 @@
-# What is Kafka Connect?
-
+\
 [![Docker Hub](https://img.shields.io/badge/docker-latest-blue.svg)](https://hub.docker.com/r/adobe/experience-platform-streaming-connect/)
 [![Build Status](https://travis-ci.com/adobe/experience-platform-streaming-connect.svg?branch=master)](https://travis-ci.com/adobe/experience-platform-streaming-connect)
 
+# What is Kafka Connect?
+
 "[Kafka Connect](https://docs.confluent.io/current/connect/index.html)", an open source component of Apache Kafka, is a framework for connecting Kafka with external systems such as databases, key-value stores, search indexes, and file systems.
 
 Kafka Connect is a framework which enables connectors developed by the open source community around Apache Kafka. It allows developers to easily import data from their data sources directly into Kafka, and then take that data from Kafka and then feed it into other systems like Elastic Search.
@@ -100,16 +101,26 @@ Enter Client Secret
 ***
 Enter JWT Token
 ***
+Enter Schema Name: [default: Streaming_Connect_Schema_20191014074347]
 
-Making call to create schema https://platform.adobe.io/ with name Streaming_test_profile_api-20190922211238
-Schema ID: https://ns.adobe.com/<tenant>/schemas/090d01896b3cbd72dc7defff1290eb99
-Data Set: ["@/dataSets/5d86d1a29ba7e11648cc3afb"]
-Streaming Connection: https://dcs.adobedc.net/collection/1e58b84cb62853b333b54980c45bdb40fc3bf80bc47022da0f76eececb2f9237
-AEP Sink Connector aep-sink-connector-20190922211238
+Making call to create schema to https://platform.adobe.io/ with name Streaming_Connect_Schema_20191014074347
+Schema ID: https://ns.adobe.com/<tenant>/schemas/<schema ID>
+Enter Dataset Name: [default: Streaming_Ingest_Test_20191014074347]
+
+Making call to create dataset to https://platform.adobe.io/ with name Streaming_Ingest_Test_20191014074347
+Data Set: ["@/dataSets/<Dataset ID>"]
+Enter Streaming Connection Name: [default: My Streaming Connection-20191014074347]
+
+Enter Streaming Connection Source: [default: My Streaming Source-20191014074347]
+
+Making call to create streaming connection to https://platform.adobe.io/ with name My Streaming Connection-20191014074347 and source My Streaming Source-20191014074347
+Streaming Connection: https://dcs.adobedc.net/collection/<Streaming Connection ID>
+AEP Sink Connector aep-sink-connector-20191014074347
 Enter the number of Experience events to publish
-5
-Publishing 5 messages for Data set 5d86d1a29ba7e11648cc3afb and schema https://ns.adobe.com/<tenant>/schemas/090d01896b3cbd72dc7defff1290eb99
-Published 5 messages
+100
+Publishing 100 messages for Data set <Dataset ID> and schema https://ns.adobe.com/<tenant>/schemas/<schema ID>
+Published 100 messages
+
 ```
 
 The quick-start script will save values for newly created resources like schema and dataset in application.conf

docker-compose.yml

@@ -10,7 +10,7 @@
  # governing permissions and limitations under the License.
 ##
 
-version: "2.1"
+version: "3"
 services:
   zoo1:
     image: zookeeper:3.4.9

setup.sh

@@ -41,25 +41,33 @@ if [ -z "${JWT_TOKEN}" ]; then
 fi
 
 # Fetch Access token
-access_token=$(curl -X POST \
+ims_response=$(curl -i -o - --silent -X POST \
   https://${IMS_HOST}/ims/exchange/jwt/ \
   -H "Content-Type: multipart/form-data" \
   -F client_id=${CLIENT_ID} \
   -F client_secret=${CLIENT_SECRET} \
-  -F jwt_token=${JWT_TOKEN} 2>/dev/null | jq -r ".access_token")
+  -F jwt_token=${JWT_TOKEN} 2>/dev/null)
+
+ims_response_code=$(echo "$ims_response" | grep -v '100 Continue' | grep HTTP |  awk '{print $2}')
+if [[ "${ims_response_code}" -ge "400" ]]; then
+  echo "Error: Unable to fetch access token from IMS, response code: ${ims_response_code}";
+  exit 1;
+fi
+
+access_token=$(echo "${ims_response}" | grep 'access_token' | jq -r ".access_token");
 
 dateString=`date +%Y%m%d%H%M%S`
 
 DEFAULT_SCHEMA_NAME="Streaming_Connect_Schema_${dateString}"
-if [ -z "${SCHEMA_NAME}" ]; then
+if [[ -z "${SCHEMA_NAME}" ]]; then
   echo "Enter Schema Name: [default: ${DEFAULT_SCHEMA_NAME}]"
   read SCHEMA_NAME
 fi
 
 SCHEMA_NAME=${SCHEMA_NAME:-${DEFAULT_SCHEMA_NAME}}
 echo "Making call to create schema to ${PLATFORM_GATEWAY} with name ${SCHEMA_NAME}"
 
-schema=$(curl -X POST \
+schema_response=$(curl -i -o - --silent -X POST \
   ${PLATFORM_GATEWAY}data/foundation/schemaregistry/tenant/schemas \
   -H "Authorization: Bearer ${access_token}" \
   -H "Content-Type: application/json" \
@@ -92,13 +100,20 @@ schema=$(curl -X POST \
     "meta:immutableTags": [
       "union"
     ]
-}' 2>/dev/null | jq -r '.["$id"]')
+}' 2>/dev/null)
+
+schema_response_code=$(echo "$schema_response" | grep -v '100 Continue' | grep HTTP |  awk '{print $2}')
+if [[ "${schema_response_code}" -ge "400" ]]; then
+  echo "Error: Unable to create schema, response code: ${schema_response_code}";
+  exit 1;
+fi
 
+schema=$(echo "${schema_response}" | grep 'meta:resourceType' | jq -r '.["$id"]')
 echo "Schema ID: ${schema}"
 
 # Create a dataset for the schema
 DEFAULT_DATASET_NAME="Streaming_Ingest_Test_${dateString}"
-if [ -z "${DATASET_NAME}" ]; then
+if [[ -z "${DATASET_NAME}" ]]; then
   echo "Enter Dataset Name: [default: ${DEFAULT_DATASET_NAME}]"
   read DATASET_NAME
 fi
@@ -143,15 +158,15 @@ fi
 INLET_NAME=${STREAMING_CONNECTION_NAME:-${DEFAULT_INLET_NAME}}
 
 DEFAULT_INLET_SOURCE="My Streaming Source-${dateString}"
-if [ -z "${STREAMING_CONNECTION_SOURCE}" ]; then
+if [[ -z "${STREAMING_CONNECTION_SOURCE}" ]]; then
   echo "Enter Streaming Connection Source: [default: ${DEFAULT_INLET_SOURCE}]"
   read STREAMING_CONNECTION_SOURCE
 fi
 INLET_SOURCE=${STREAMING_CONNECTION_SOURCE:-${DEFAULT_INLET_SOURCE}}
 
 echo "Making call to create streaming connection to ${PLATFORM_GATEWAY} with name ${INLET_NAME} and source ${INLET_SOURCE}"
 
-streamingEndpoint=$(curl POST \
+inlet_response=$(curl -i -o - --silent -X POST \
   ${PLATFORM_GATEWAY}data/core/edge/inlet \
   -H "Authorization: Bearer ${access_token}" \
   -H "Content-Type: application/json" \
@@ -162,7 +177,14 @@ streamingEndpoint=$(curl POST \
     "description" : "Collects streaming data from my website",
     "sourceId" : "'"${INLET_SOURCE}"'",
     "dataType": "xdm"
-}' 2> /dev/null | jq -r ".inletUrl" )
+}' 2> /dev/null)
+
+inlet_response_code=$(echo "$inlet_response" | grep -v '100 Continue' | grep HTTP  |  awk '{print $2}')
+if [[ "${inlet_response_code}" -ge "400" ]]; then
+  echo "Error: Unable to create schema, response code: ${inlet_response_code}";
+  exit 1;
+fi
+streamingEndpoint=$(echo "${inlet_response}" | grep 'inletUrl' | jq -r ".inletUrl")
 
 echo "Streaming Connection: "${streamingEndpoint}
 
@@ -175,7 +197,7 @@ ${KAFKA_HOME}/bin/kafka-topics.sh \
  --topic ${connectTopicName}
 
 aemSinkConnectorName="aep-sink-connector-${dateString}"
-aemSinkConnector=$(curl -s -X POST \
+aem_connector_response=$(curl -i -o - --silent -X POST \
   http://kafka-connect:8083/connectors \
   -H "Content-Type: application/json" \
   -d '{
@@ -189,6 +211,12 @@ aemSinkConnector=$(curl -s -X POST \
     }
 }')
 
+aem_connector_response_code=$(echo "$aem_connector_response" | grep -v '100 Continue'| grep HTTP |  awk '{print $2}')
+if [[ "${aem_connector_response_code}" -ge "400" ]]; then
+  echo "Error: Unable to create streaming connector, response code: ${aem_connector_response_code}";
+  exit 1;
+fi
+
 echo "AEP Sink Connector ${aemSinkConnectorName}"
 
 datasetId=`echo ${dataSet} | cut -d'"' -f2 | cut -d'/' -f3`