Merge branch 'apache:master' into ASYNC_IMPORT

Author: DishaTalreja3

.asf.yaml

@@ -32,12 +32,6 @@ github:
     squash:  true
     merge:   false
     rebase:  false
-  notifications:
-    commits: [email protected]
-    issues: [email protected]
-    pullrequests: [email protected]
-    pullrequests_comment: [email protected]
-    jira_options: worklog link label
   protected_branches:
     master: {}
   collaborators:
@@ -48,3 +42,10 @@ github:
     - pinal-shah
     - rkundam
     - sarathsubramanian
+
+notifications:
+  commits: [email protected]
+  issues: [email protected]
+  pullrequests: [email protected]
+  pullrequests_comment: [email protected]
+  jira_options: worklog link label

addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaLineageHook.java

@@ -107,6 +107,7 @@ public void process(ImpalaQuery lineageQuery) throws Exception {
                     case CREATETABLE_AS_SELECT:
                     case ALTERVIEW_AS:
                     case QUERY:
+                    case QUERY_WITH_CLAUSE:
                         event = new CreateImpalaProcess(context);
                         break;
                 default:

addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/ImpalaOperationParser.java

@@ -41,6 +41,9 @@ public class ImpalaOperationParser {
     private static final Pattern INSERT_SELECT_FROM_PATTERN =
             Pattern.compile("^[ ]*\\binsert\\b.*\\b(into|overwrite)\\b.*\\bselect\\b.*\\bfrom\\b.*", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
 
+    private static final Pattern WITH_CLAUSE_INSERT_SELECT_FROM_PATTERN =
+            Pattern.compile("^[ ]*(\\bwith\\b.*)?\\s*\\binsert\\b.*\\b(into|overwrite)\\b.*\\bselect\\b.*\\bfrom\\b.*", Pattern.DOTALL | Pattern.CASE_INSENSITIVE);
+    
     public ImpalaOperationParser() {
     }
 
@@ -55,6 +58,9 @@ public static ImpalaOperationType getImpalaOperationType(String queryText) {
             return ImpalaOperationType.ALTERVIEW_AS;
         } else if (doesMatch(queryTextWithNoComments, INSERT_SELECT_FROM_PATTERN)) {
             return ImpalaOperationType.QUERY;
+        } else if (doesMatch(queryTextWithNoComments, WITH_CLAUSE_INSERT_SELECT_FROM_PATTERN)) {
+            return ImpalaOperationType.QUERY_WITH_CLAUSE;
+            
         }
 
         return ImpalaOperationType.UNKNOWN;

addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/events/BaseImpalaEvent.java

@@ -208,8 +208,8 @@ protected String getQualifiedName(List<AtlasEntity> inputs, List<AtlasEntity> ou
             }
         }
 
-        if (operation != ImpalaOperationType.QUERY) {
-            String errorMessage = String.format("Expect operation to be QUERY, but get unexpected operation type {}", operation.name());
+        if (operation != ImpalaOperationType.QUERY && operation != ImpalaOperationType.QUERY_WITH_CLAUSE) {
+            String errorMessage = String.format("Expect operation to be QUERY or QUERY_WITH_CLAUSE, but get unexpected operation type {}", operation.name());
             LOG.error(errorMessage);
             throw new IllegalArgumentException(errorMessage);
         }

addons/impala-bridge/src/main/java/org/apache/atlas/impala/model/ImpalaOperationType.java

@@ -23,6 +23,7 @@ public enum ImpalaOperationType{
     CREATETABLE_AS_SELECT ("CREATETABLE_AS_SELECT"),
     ALTERVIEW_AS ("ALTERVIEW_AS"),
     QUERY ("QUERY"),
+    QUERY_WITH_CLAUSE ("QUERY_WITH_CLAUSE"),
 
     // sub operation type, which is associated with output
     INSERT ("INSERT"),

addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookTest.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.impala.hook;
+
+import org.apache.atlas.impala.model.ImpalaOperationType;
+import org.apache.commons.lang.RandomStringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+public class ImpalaLineageHookTest {
+    private static final Logger LOG = LoggerFactory.getLogger(ImpalaLineageHookTest.class);
+
+ @Test(dataProvider = "queryDataProvider")
+    public void testAllImpalaOperationTypes(String query, ImpalaOperationType expectedOperationType) {
+        try {
+            ImpalaOperationType operationType = ImpalaOperationParser.getImpalaOperationType(query);
+            assertEquals(operationType, expectedOperationType);
+        } catch (Exception e) {
+            fail("Query processing failed for query: " + query + " due to exception: " + e.getMessage());
+        }
+    }
+
+    @DataProvider(name = "queryDataProvider")
+    public Object[][] provideTestData() {
+        String table1 = "table_" + random();
+        String table2 = "table_" + random();
+
+        return new Object[][] {
+                { "CREATE VIEW my_view AS SELECT id, name FROM " + table1, ImpalaOperationType.CREATEVIEW },
+                { "CREATE TABLE " + table1 + " AS SELECT id, name FROM " + table1, ImpalaOperationType.CREATETABLE_AS_SELECT },
+                { "ALTER VIEW my_view AS SELECT id, name FROM " + table1, ImpalaOperationType.ALTERVIEW_AS },
+                { "INSERT INTO " + table1 + " SELECT id, name FROM " + table1, ImpalaOperationType.QUERY },
+                { "WITH filtered_data AS (SELECT id, name, amount FROM " + table1 + " WHERE amount > 100) " +
+                        "INSERT INTO " + table2 + " SELECT id, name, amount FROM filtered_data", ImpalaOperationType.QUERY_WITH_CLAUSE }
+        };
+    }
+
+    private String random() {
+        return RandomStringUtils.randomAlphanumeric(10);
+    }
+}

build-tools/src/main/resources/ui-dist/index.js

@@ -51,7 +51,19 @@
         window.ui = ui;
 
         atlasLogo = gatewayUrl + "/img/atlas_logo.svg";
-        $('#swagger-ui img').attr("src", atlasLogo);
+        setTimeout(() => {
+            const logoAnchor = document.querySelector(".swagger-ui .topbar a");
+            
+            if (logoAnchor) {
+              const svgLogo = logoAnchor.querySelector("svg");
+              if (svgLogo) svgLogo.remove();
+              const img = document.createElement("img");
+              img.src = atlasLogo;
+              img.alt = "Atlas Logo";
+              img.style.height = "40px";
+              logoAnchor.appendChild(img);
+            }
+        }, 500);
 
         fetchCsrfHeader();
     }

build-tools/src/main/resources/ui-dist/swagger-ui-bundle.js

@@ -497,10 +497,11 @@ protected <T> T callAPIWithResource(API api, WebResource resource, Object reques
                 } catch (ClientHandlerException e) {
                     throw new AtlasServiceException(api, e);
                 }
-            } else if (clientResponse.getStatus() != ClientResponse.Status.SERVICE_UNAVAILABLE.getStatusCode()) {
+            } else if (clientResponse.getStatus() != ClientResponse.Status.SERVICE_UNAVAILABLE.getStatusCode() &&
+                    clientResponse.getStatus() != ClientResponse.Status.INTERNAL_SERVER_ERROR.getStatusCode()) {
                 break;
             } else {
-                LOG.error("Got a service unavailable when calling: {}, will retry..", resource);
+                LOG.error("attempt #{}: error {} when calling: {}, will retry..", (i + 1), clientResponse.getStatus(), resource);
 
                 sleepBetweenRetries();
             }

build-tools/src/main/resources/ui-dist/swagger-ui-standalone-preset.js

@@ -247,7 +247,7 @@ module.exports = function(grunt) {
                 files: [{
                     expand: true,
                     cwd: distPath + '/js',
-                    src: ['**/*.js', '!libs/**', '!external_lib/**/purify.min.js', '!external_lib/**'],
+                    src: ['**/*.js', '!libs/**', '!external_lib/**'],
                     dest: distPath + '/js'
                 }]
             }

build-tools/src/main/resources/ui-dist/swagger-ui.css

@@ -253,7 +253,7 @@ module.exports = function(grunt) {
                 files: [{
                     expand: true,
                     cwd: distPath + '/js',
-                    src: ['**/*.js', '!libs/**', '!external_lib/**/purify.min.js', '!external_lib/**'],
+                    src: ['**/*.js', '!libs/**', '!external_lib/**'],
                     dest: distPath + '/js'
                 }]
             }