Merge pull request #45 from santhh/gcs-s3-inspect

solution publish update

Author: theodoresiu

README.md

@@ -81,7 +81,7 @@ dependencies {
 	compile group: 'org.slf4j', name: 'slf4j-jdk14', version: '1.7.5'
 	compile 'com.google.cloud:google-cloud-kms:0.70.0-beta'   
     compile 'com.google.guava:guava:27.0-jre'
-	compile group: 'com.google.cloud', name: 'google-cloud-dlp', version: '0.100.0-beta'
+	compile group: 'com.google.cloud', name: 'google-cloud-dlp', version: '0.99.0-beta'
     compile 'com.google.api-client:google-api-client:1.27.0'
     compile group: 'com.google.apis', name: 'google-api-services-cloudkms', version: 'v1-rev53-1.23.0'
 	compile group: 'org.apache.beam', name: 'beam-sdks-java-io-amazon-web-services', version: dataflowBeamVersion

build.gradle

@@ -51,29 +51,29 @@ INSPECT_TEMPLATE_NAME=$(jq -c '.name' ${INSPECT_TEMPLATE_OUTPUT})
 DYNAMIC_TEMPLATE_BUCKET_SPEC=gs://dynamic-template/dynamic_template_dlp_inspect.json
 JOB_NAME="dlp-inspect-pipeline-`date +%Y%m%d-%H%M%S-%N`"
 echo $JOB_NAME
-GCS_STAGING_LOCATION=gs://dynamic-template/log
-TEMP_LOCATION=gs://dynamic-template/temp
+GCS_STAGING_LOCATION=$GCS_BUCKET_URL/log
+TEMP_LOCATION=$GCS_BUCKET_URL/temp
 PARAMETERS_CONFIG='{  
    "jobName":"'$JOB_NAME'",
    "parameters":{  
       "streaming":"true",
 	  "enableStreamingEngine":"true",
 	  "autoscalingAlgorithm":"NONE",
       "workerMachineType": "n1-standard-8",
-      "numWorkers":"50",
-      "maxNumWorkers":"50",
+      "numWorkers":"3",
+      "maxNumWorkers":"3",
       "awsAccessKey":"'$AWS_ACCESS_KEY'",
 	  "awsSecretKey":"'$AWS_SECRET_KEY'",
 	  "s3BucketUrl":"'$S3_BUCKET_URL'",
-	  "gcsBucketUrl":"'$GCS_BUCKET_URL'",
+	  "gcsBucketUrl":"'$GCS_BUCKET_URL'/*.csv",
 	  "inspectTemplateName":'$INSPECT_TEMPLATE_NAME',
 	  "s3ThreadPoolSize":"1000",
 	  "maxConnections":"1000000",
 	  "socketTimeout":"100",
 	  "connectionTimeout":"100",
 	  "tempLocation":"'$TEMP_LOCATION'",
 	  "awsRegion":"'$AWS_REGION'",
-	  "dataSetId":"'$BQ_DATASET'",	  
+	  "dataSetId":"'$BQ_DATASET'"	  
 	}
 }'
 DF_API_ROOT_URL="https://dataflow.googleapis.com"

create-df-template.sh

@@ -12,20 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.#!/usr/bin/env bash
-# Copyright 2019 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
- 
+
 set -x 
 PROJECT_ID=$1
 KEY_RING_NAME=$2

create-kek.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.#!/usr/bin/env bash
+
+# please make sure you have owner permission in your project 
+set -x 
+# export some env variables
+export PROJECT_ID=$(gcloud config get-value project)
+export DATA_STORAGE_BUCKET=${PROJECT_ID}-demo-data 
+export TEK=$(openssl rand -base64 32)
+export KEY_RING_NAME=demo-key-ring
+export KEY_NAME=demo-key
+export KEK_FILE_NAME=kek.json
+export PROJECT_NUMBER=$(gcloud projects list --filter=${PROJECT_ID} --format="value(PROJECT_NUMBER)") 
+export SERVICE_ACCOUNT_NAME=demo-service-account
+export REGION=us-central1
+export BQ_DATASET_NAME=demo_dataset
+# enable the required APIs
+gcloud services enable dlp.googleapis.com
+gcloud services enable cloudkms.googleapis.com
+gcloud services enable bigquery
+gcloud services enable storage_component
+gcloud services enable dataflow
+gcloud services enable cloudbuild.googleapis.com
+# create BQ dataset. Table will be dynamically generated from dataflow pipeline
+bq --location=US mk -d --description "De-Identified PII Dataset" ${BQ_DATASET_NAME}
+# create a data bucket to store the PII data
+gsutil mb -c standard -l ${REGION} gs://${DATA_STORAGE_BUCKET}
+# allow some additional access to cloud build service account
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com --role roles/cloudkms.cryptoKeyEncrypter
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com --role roles/cloudkms.admin
+# trigger the first cloud build script to create the KEK
+gcloud builds submit . --config dlp-demo-part-1-crypto-key.yaml --substitutions _GCS_BUCKET_NAME=gs://${DATA_STORAGE_BUCKET},_KEY_RING_NAME=${KEY_RING_NAME},_KEY_NAME=${KEY_NAME},_TEK=${TEK},_KEK=${KEK_FILE_NAME},_API_KEY=$(gcloud auth print-access-token)
+# DLP requires a service account to be used for API call
+gcloud iam service-accounts create ${SERVICE_ACCOUNT_NAME} --display-name "DLP Demo Service Account"
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member serviceAccount:${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com --role roles/editor
+gcloud projects add-iam-policy-binding ${PROJECT_ID} --member serviceAccount:${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com --role roles/storage.admin
+gcloud iam service-accounts keys create --iam-account ${SERVICE_ACCOUNT_NAME}@${PROJECT_ID}.iam.gserviceaccount.com demo_key.json --user-output-enabled
+gcloud auth activate-service-account --key-file demo_key.json
+# trigger the cloud build script to create DLP templates
+gcloud builds submit . --config dlp-demo-part-2-dlp-template.yaml --substitutions _KEK_CONFIG_FILE=gs://${DATA_STORAGE_BUCKET}/${KEK_FILE_NAME},_GCS_BUCKET_NAME=gs://${DATA_STORAGE_BUCKET},_API_KEY=$(gcloud auth print-access-token)
+# download the json file to parse template name using jq
+gsutil cp gs://${DATA_STORAGE_BUCKET}/deid-template.json .
+gsutil cp gs://${DATA_STORAGE_BUCKET}/inspect-template.json .
+export DEID_TEMPLATE_NAME=$(jq -r '.name' deid-template.json)
+export INSPECT_TEMPLATE_NAME=$(jq -r '.name' inspect-template.json)
+# trigger the dataflow pipeline
+export jobId="demo-dlp-deid-pipeline-`date +%Y%m%d-%H%M%S`"
+gcloud dataflow jobs run ${jobId} --gcs-location gs://dataflow-templates/latest/Stream_DLP_GCS_Text_to_BigQuery --parameters --region=us-central1,inputFilePattern=gs://${DATA_STORAGE_BUCKET}/CCRecords_1564602825.csv,dlpProjectId=${PROJECT_ID},deidentifyTemplateName=${DEID_TEMPLATE_NAME},inspectTemplateName=${INSPECT_TEMPLATE_NAME},datasetName=${BQ_DATASET_NAME},batchSize=500
+
+

deploy-data-tokeninzation-solution.sh

@@ -0,0 +1,30 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ 
+set -x 
+export PROJECT_ID=$(gcloud config get-value project)
+gcloud services enable dlp.googleapis.com
+gcloud services enable cloudkms.googleapis.com
+gcloud services enable bigquery
+gcloud services enable storage_component
+gcloud services enable dataflow
+gcloud services enable cloudbuild.googleapis.com
+export AWS_ACCESS_KEY="<access_key>"
+export AWS_SECRET_KEY="<secret_key>"
+export S3_BUCKET_URL="s3://<path>"
+export GCS_BUCKET_URL="gs://<path>"
+export AWS_REGION="<region>"
+export BQ_DATASET="dlp_inspection"
+
+gcloud builds submit . --config dlp-demo-s3-gcs-inspect.yaml --substitutions _AWS_ACCESS_KEY=$AWS_ACCESS_KEY,_API_KEY=$(gcloud auth print-access-token),_AWS_SECRET_KEY=$AWS_SECRET_KEY,_S3_BUCKET_URL=$S3_BUCKET_URL,_GCS_BUCKET_URL=$GCS_BUCKET_URL,_AWS_REGION=$AWS_REGION,_BQ_DATASET=$BQ_DATASET

deploy-s3-inspect-solution.sh

@@ -19,7 +19,7 @@
     args: ['kms', 'keys', 'create' ,'${_KEY_NAME}', '--location=global','--purpose=encryption','--keyring=${_KEY_RING_NAME}']
   - name: 'gcr.io/cloud-builders/docker'
     entrypoint: 'bash'
-    args: ['-c', 'sh create-kek.sh ${PROJECT_ID} ${_KEY_RING_NAME} ${_KEY_NAME} ${_TEK} ${_KEK} ${_API_KEY}'] 
+    args: ['-c', 'sh create-kek.sh ${PROJECT_ID} ${_KEY_RING_NAME} ${_KEY_NAME} ${_TEK} ${_KEK} ${_API_KEY}']
   - name: gcr.io/cloud-builders/curl
     args: ['http://storage.googleapis.com/dataflow-dlp-solution-sample-data/sample_data_scripts.tar.gz', '-o', 'sample_data_scripts.tar.gz']
   - name: 'gcr.io/cloud-builders/docker'
@@ -28,4 +28,4 @@
   - name: gcr.io/cloud-builders/gsutil
     args: ['cp', '${_KEK}', '${_GCS_BUCKET_NAME}']
   - name: gcr.io/cloud-builders/gsutil
-    args: ['-m', 'cp', './solution-test/*.csv', '${_GCS_BUCKET_NAME}']  
+    args: ['-m', 'cp', './solution-test/*.csv', '${_GCS_BUCKET_NAME}']

diagrams/ref_arch_solution.png

@@ -24,4 +24,4 @@ steps:
 - name: gcr.io/cloud-builders/gsutil
   args: ['cp', 'deid-template.json','${_GCS_BUCKET_NAME}/deid-template.json'] 
 - name: gcr.io/cloud-builders/gsutil
-  args: ['cp', 'inspect-template.json','${_GCS_BUCKET_NAME}/inspect-template.json']       
+  args: ['cp', 'inspect-template.json','${_GCS_BUCKET_NAME}/inspect-template.json']     

dlp-demo-part-1-crypto-key.yaml

@@ -13,5 +13,7 @@
  # See the License for the specific language governing permissions and
  # limitations under the License. 
 steps:
+- name: gcr.io/cloud-solutions-images/bq
+  args: ['mk', '-d', '--description', 'DLP Inspection', '--location=US', '${_BQ_DATASET}']
 - name: 'ubuntu'
   args: ['bash', '-c','apt-get -q update && apt-get install -qqy curl && apt-get install -qqy jq;sh create-df-template.sh ${PROJECT_ID} ${_AWS_ACCESS_KEY} ${_API_KEY} ${_AWS_SECRET_KEY} ${_S3_BUCKET_URL} ${_GCS_BUCKET_URL} ${_AWS_REGION} ${_BQ_DATASET}']