Issue #SB-30224 merge: Merge pull request #284 from project-sunbird/release-4.8.5

Issue #SB-30224 merge: merge release-4.8.5 to master

Author: vinukumar-vs

.circleci/config.yml

@@ -2,23 +2,34 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: node:6-alpine
+      - image: circleci/node:12.13.0 
+    parallelism: 2
+    working_directory: ~/project/src 
     steps:
       - checkout
-      - run: apk add zip git
-      - run: apk add python make g++
+      - run: npm cache clean --force 
       - run: git submodule update --init --recursive src/libs
       - run: mkdir -p opt/content && cp -r src/* opt/content/
       - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
+          key: dependency-cache-{{ checksum "src/package.json" }}
       - run: cd opt/content/ && npm install --unsafe-perm
       - save_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-          paths: ./node_modules
+          key: dependency-cache-{{ checksum "src/package.json" }}
+          paths: ./node_modules     
       - run: cd opt && zip -r content.zip content
       - store_artifacts:
           path: opt/content.zip 
-          destination: content.zip
+          destination: content.zip  
+      - run:
+          name: Run test cases
+          command: 'cd src && npm install --unsafe-perm && npm run coverage'     
+      - run:
+          name: install sonar scanner
+          command: sudo npm install -g sonarqube-scanner@2.8.0
+      - run:
+          name: Run SonarScan
+          command: sonar-scanner
+                  
 workflows:
   version: 2.1
   build_and_test:

.gitignore

@@ -1,4 +1,6 @@
 /src/node_modules/
 /src/npm-debug.log
 /src/coverage/
-/src/**/*.log
+/src/**/*.log
+src/.env
+/src/.nyc_output

Jenkinsfile

@@ -15,27 +15,17 @@ node('build-slave') {
                 else
                     println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
                 cleanWs()
-                if (params.github_release_tag == "") {
-                    checkout scm
-                    commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
-                    branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim()
-                    build_tag = branch_name + "_" + commit_hash
-                    println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL)
-                }
-                else {
-                    def scmVars = checkout scm
-                    checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
-                    build_tag = params.github_release_tag
-                    println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag specified, building from github_release_tag: " + params.github_release_tag + ANSI_NORMAL)
-                }
+                checkout scm
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                build_tag = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
                 echo "build_tag: " + build_tag
             }
 
             stage('Build') {
                 env.NODE_ENV = "build"
                 print "Environment will be : ${env.NODE_ENV}"
                 sh('git submodule update --init')
-                sh('git submodule update --init --recursive --remote')
+                // sh('git submodule update --init --recursive --remote')
                 sh('chmod 777 build.sh')
                 sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
             }

README.md

@@ -36,10 +36,13 @@ The code in this repository is licensed under MIT unless otherwise noted. Please
 * sunbird_telemetry_sync_batch_size e.g: 20
 * sunbird_learner_service_local_base_url e.g: 'http://learner-service:9000'
 * sunbird_content_service_local_base_url e.g: 'http://localhost:5000'
+* sunbird_content_upload_data_limit: Content upload data limit e.g.: 50mb (string)
 
 ## Setup Instructions
 * Clone the project.eg .(git clone --recursive  url)
-* Run "git submodule foreach git pull origin master" to pull the latest sunbird-js-utils sub module
+* Run "git submodule init"
+* Run "git submodule update"
+* Run "git submodule update --init --recursive" to pull the latest sunbird-js-utils sub module (if you want to pull from specific branch, "git submodule foreach git pull origin 'branch_name'" )
 * Change to src folder
 * Run `npm install`
 * Run `node app.js`

auto_build_deploy

@@ -0,0 +1,56 @@
+@Library('deploy-conf') _ 
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            stage('Checkout') {
+                tag_name = env.JOB_NAME.split("/")[-1]
+                pre_checks()
+                if (!env.hub_org) {
+                    println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
+                    error 'Please resolve the errors and rerun..'
+                }
+                else
+                    println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
+                cleanWs()
+                def scmVars = checkout scm
+                checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$tag_name"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
+                build_tag = tag_name + "_" + env.BUILD_NUMBER
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                artifact_version = tag_name + "_" + commit_hash
+                echo "build_tag: " + build_tag
+                
+            }
+
+    //        stage Build
+                env.NODE_ENV = "build"
+                print "Environment will be : ${env.NODE_ENV}"
+                sh('git submodule update --init')
+                // sh('git submodule update --init --recursive --remote')
+                sh('chmod 777 build.sh')
+                sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
+            
+ //           stage ArchiveArtifacts
+                archiveArtifacts "metadata.json"
+                currentBuild.description = "${build_tag}"
+            
+        }
+        currentBuild.result = "SUCCESS"    
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()  
+        auto_build_deploy()
+
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()
+        throw err
+    }
+
+}

build.sh

@@ -2,7 +2,7 @@
 # Build script
 set -eo pipefail
 build_tag=$1
-name=content-service
+name=knowledge-mw-service
 node=$2
 org=$3
 

setup.md

@@ -25,7 +25,8 @@
 
 ##Setup Instructions
 * Clone the project.
-* Run "git submodule foreach git pull origin master" to pull the latest sunbird-js-utils sub module
+* RUN "git submodule init" followed by "git submodule update" to load submodules to the latest version.
+* Run "git submodule foreach git pull origin master" to pull the latest sunbird-js-utils sub module. Rename `master` to specific remote branch name if you want to load from specific branch.
 * Change to src folder
 * Run `npm install`
 * Run `node app.js`

sonar-project.properties

@@ -0,0 +1,8 @@
+sonar.projectName=sunbird-content-service
+sonar.language=js
+sonar.projectKey=project-sunbird_sunbird-content-service
+sonar.host.url=https://sonarcloud.io
+sonar.exclusions=src/test/**, **/node_modules/**
+sonar.javascript.lcov.reportPaths=src/coverage/lcov.info
+sonar.organization=project-sunbird
+sonar.projectVersion=1.0

src/app.js

@@ -10,6 +10,7 @@ var fs = require('fs')
 var configUtil = require('sb-config-util')
 var _ = require('lodash')
 var logger = require('sb_logger_util_v2')
+var ApiInterceptor = require('sb_api_interceptor')
 
 const contentProvider = require('sb_content_provider_util')
 var contentMetaProvider = require('./contentMetaFilter')
@@ -19,7 +20,7 @@ const contentProviderConfigPath = path.join(__dirname, '/config/contentProviderA
 var contentProviderApiConfig = JSON.parse(fs.readFileSync(contentProviderConfigPath))
 const telemtryEventConfig = JSON.parse(fs.readFileSync(path.join(__dirname, 'config/telemetryEventConfig.json')))
 
-var reqDataLimitOfContentUpload = '50mb'
+var reqDataLimitOfContentUpload = process.env.sunbird_content_upload_data_limit || '50mb'
 
 const port = process.env.sunbird_content_service_port ? process.env.sunbird_content_service_port : 5000
 const defaultChannel = process.env.sunbird_default_channel || 'sunbird'
@@ -29,6 +30,11 @@ globalEkstepProxyBaseUrl = process.env.sunbird_content_plugin_base_url ? process
 const contentRepoBaseUrl = process.env.sunbird_content_repo_api_base_url || 'https://qa.ekstep.in/api'
 const contentRepoApiKey = process.env.sunbird_content_repo_api_key
 
+const contentServiceBaseUrl = process.env.sunbird_contnet_service_base_url || 'http://content-service:9000'
+const contentServiceAuthToken = process.env.sunbird_content_service_auth_token
+
+const assessmentServiceBaseUrl = process.env.sunbird_assessment_service_base_url || 'http://assessment-service:9000'
+
 const learnerServiceLocalBaseUrl = process.env.sunbird_learner_service_local_base_url
   ? process.env.sunbird_learner_service_local_base_url
   : 'http://learner-service:9000'
@@ -50,10 +56,14 @@ const producerId = process.env.sunbird_environment + '.' + process.env.sunbird_i
 const sunbirdPortalBaseUrl = process.env.sunbird_portal_base_url || 'https://staging.open-sunbird.org'
 const lockExpiryTime = process.env.sunbird_lock_expiry_time || 3600
 const isHealthCheckEnabled = process.env.sunbird_health_check_enable || 'true'
-const contentServiceLocalBaseUrl = process.env.sunbird_content_service_local_base_url ? process.env.sunbird_content_service_local_base_url : 'http://content-service:5000'
+const contentServiceLocalBaseUrl = process.env.sunbird_content_service_local_base_url ? process.env.sunbird_content_service_local_base_url : 'http://knowledge-mw-service:5000'
 const sunbirdGzipEnable = process.env.sunbird_gzip_enable || 'true'
+const kidTokenPublicKeyBasePath = process.env.sunbird_kid_public_key_base_path || '/keys/'
 
 configUtil.setContentProviderApi(contentProviderApiConfig.API)
+configUtil.setConfig('CONTENT_SERVICE_BASE_URL', contentServiceBaseUrl)
+configUtil.setConfig('CONTENT_SERVICE_AUTH_TOKEN', contentServiceAuthToken)
+configUtil.setConfig('ASSESSMENT_SERVICE_BASE_URL', assessmentServiceBaseUrl)
 configUtil.setConfig('CONTENT_REPO_BASE_URL', contentRepoBaseUrl)
 configUtil.setConfig('TELEMETRY_BASE_URL', telemetryBaseUrl)
 configUtil.setConfig('CONTENT_REPO_AUTHORIZATION_TOKEN', 'Bearer ' + contentRepoApiKey)
@@ -148,7 +158,7 @@ app.use(function (req, res, next) {
     res.sendStatus(200)
   } else {
     next()
-  };
+  }
 })
 
 require('./routes/healthCheckRoutes')(app)
@@ -167,10 +177,34 @@ require('./routes/externalUrlMetaRoute')(app)
 require('./routes/pluginsRoutes')(app)
 require('./routes/collaborationRoutes')(app)
 require('./routes/lockRoutes')(app)
+require('./routes/questionRoutes')(app)
 // this middleware route add after all the routes
 require('./middlewares/proxy.middleware')(app)
 
-function startServer () {
+async function startServer (cb) {
+  var keyCloakConfig = {
+    'authServerUrl': process.env.sunbird_keycloak_auth_server_url ? process.env.sunbird_keycloak_auth_server_url : 'https://staging.open-sunbird.org/auth',
+    'realm': process.env.sunbird_keycloak_realm ? process.env.sunbird_keycloak_realm : 'sunbird',
+    'clientId': process.env.sunbird_keycloak_client_id ? process.env.sunbird_keycloak_client_id : 'portal',
+    'public': process.env.sunbird_keycloak_public ? process.env.sunbird_keycloak_public : true,
+    'realmPublicKey': process.env.sunbird_keycloak_public_key
+  }
+  logger.info({ msg: 'keyCloakConfig', keyCloakConfig })
+
+  var cacheConfig = {
+    store: process.env.sunbird_cache_store ? process.env.sunbird_cache_store : 'memory',
+    ttl: process.env.sunbird_cache_ttl ? process.env.sunbird_cache_ttl : 1800
+  }
+
+  var apiInterceptor = new ApiInterceptor(keyCloakConfig, cacheConfig)
+
+  await apiInterceptor.loadTokenPublicKeys(path.join(__dirname, kidTokenPublicKeyBasePath))
+
+  if (this.server) {
+    cb && cb()
+    return
+  }
+
   this.server = http.createServer(app).listen(port, function () {
     logger.info({ msg: `server running at PORT ${port}` })
     logger.debug({ msg: `server started at ${new Date()}` })
@@ -186,8 +220,9 @@ function startServer () {
       logger.fatal({ msg: 'error in getting meta filters', err })
       process.exit(1)
     })
+    cb && cb()
   })
-  this.server.keepAliveTimeout = 60000 * 5;
+  this.server.keepAliveTimeout = 30000 * 5
 }
 
 // Create server
@@ -206,12 +241,6 @@ if (defaultChannel) {
   startServer()
 }
 
-// Close server, when we start for test cases
-exports.close = function () {
-  logger.debug({ msg: `server stopped at ${new Date()}` })
-  this.server.close()
-}
-
 // Telemetry initialization
 const telemetryBatchSize = parseInt(process.env.sunbird_telemetry_sync_batch_size, 10) || 20
 telemtryEventConfig.pdata.id = producerId
@@ -226,3 +255,15 @@ const telemetryConfig = {
 
 logger.debug({ msg: 'Telemetry is initialized.' })
 telemetry.init(telemetryConfig)
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Kp-mw Unhandled Rejection', p, reason)
+  logger.error({ msg: 'Kp-mw Unhandled Rejection', p, reason })
+})
+process.on('uncaughtException', (err) => {
+  console.log('Kp-mw Uncaught Exception', err)
+  logger.error({ msg: 'Kp-mw Uncaught Exception', err })
+  process.exit(1)
+})
+
+exports.start = startServer
+exports.close = (cb) => { this.server.close(cb) }

src/config/constants.json

@@ -1,5 +1,8 @@
 {
     "orgCacheExpiryTime": 1800,
     "orgfieldsAllowedToSend": ["email","orgName"],
-    "orgCacheKeyNamePrefix" : "orgdata-"
+    "orgCacheKeyNamePrefix" : "orgdata-",
+    "readableLicenseFields": ["name", "url", "description"],
+    "licenseCacheKeyNamePrefix": "license-",
+    "licenseCacheExpiryTime": 1800
 }