-
Notifications
You must be signed in to change notification settings - Fork 2.7k
Core, Hive: Double check commit status in case of commit conflict for NoLock #12637
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
6efb3f8
ed60da3
3fe077a
991a783
bf871bc
f5dae6b
5c5f672
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -268,16 +268,6 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { | |
throw e; | ||
|
||
} catch (Throwable e) { | ||
if (e.getMessage() != null | ||
&& e.getMessage() | ||
.contains( | ||
"The table has been modified. The parameter value for key '" | ||
+ HiveTableOperations.METADATA_LOCATION_PROP | ||
+ "' is")) { | ||
throw new CommitFailedException( | ||
e, "The table %s.%s has been modified concurrently", database, tableName); | ||
} | ||
|
||
if (e.getMessage() != null | ||
&& e.getMessage().contains("Table/View 'HIVE_LOCKS' does not exist")) { | ||
throw new RuntimeException( | ||
|
@@ -287,15 +277,33 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) { | |
e); | ||
} | ||
|
||
LOG.error( | ||
"Cannot tell if commit to {}.{} succeeded, attempting to reconnect and check.", | ||
database, | ||
tableName, | ||
e); | ||
commitStatus = BaseMetastoreOperations.CommitStatus.UNKNOWN; | ||
commitStatus = | ||
BaseMetastoreOperations.CommitStatus.valueOf( | ||
checkCommitStatus(newMetadataLocation, metadata).name()); | ||
if (e.getMessage() != null | ||
&& e.getMessage() | ||
.contains( | ||
"The table has been modified. The parameter value for key '" | ||
+ HiveTableOperations.METADATA_LOCATION_PROP | ||
+ "' is")) { | ||
// It's possible the HMS client incorrectly retries a successful operation, due to network | ||
// issue for example, and triggers this exception. So we need double-check to make sure | ||
// this is really a concurrent modification. Hitting this exception means no pending | ||
// requests, if any, can succeed later, so it's safe to check status in strict mode | ||
commitStatus = checkCommitStatusStrict(newMetadataLocation, metadata); | ||
if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) { | ||
throw new CommitFailedException( | ||
e, "The table %s.%s has been modified concurrently", database, tableName); | ||
} | ||
} else { | ||
LOG.error( | ||
"Cannot tell if commit to {}.{} succeeded, attempting to reconnect and check.", | ||
database, | ||
tableName, | ||
e); | ||
commitStatus = | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why do we need this weird casting? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I believe it was introduced in #10001 when we had two enums: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I updated to address this issue. At least I think the new |
||
BaseMetastoreOperations.CommitStatus.valueOf( | ||
checkCommitStatus(newMetadataLocation, metadata).name()); | ||
} | ||
|
||
switch (commitStatus) { | ||
case SUCCESS: | ||
break; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -52,9 +52,9 @@ CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCH | |
|
||
CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER" VARCHAR(128)); | ||
|
||
CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); | ||
CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); | ||
|
||
CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(767) NOT NULL, "TYPE_NAME" CLOB, "INTEGER_IDX" INTEGER NOT NULL); | ||
CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(767) NOT NULL, "TYPE_NAME" VARCHAR(32672), "INTEGER_IDX" INTEGER NOT NULL); | ||
|
||
CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); | ||
|
||
|
@@ -130,15 +130,15 @@ CREATE TABLE "APP"."TAB_COL_STATS"( | |
"BIT_VECTOR" BLOB | ||
); | ||
|
||
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we want to change the other There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add more changes to be consistent with HIVE-16667 and HIVE-25574 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FYI CLOB handling was fixed in https://github.com/apache/hive/pull/5386/files#diff-bcca13f6cc251df321e8fe80568ef0334a1d44f7e5e7ff2fcaa06ab4f05bbdf9R3387 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good to know the direct SQL is made more robust. Maybe we can revisit this change when we upgrade our hive dependency. |
||
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); | ||
|
||
CREATE TABLE "APP"."BUCKETING_COLS" ("SD_ID" BIGINT NOT NULL, "BUCKET_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); | ||
|
||
CREATE TABLE "APP"."TYPE_FIELDS" ("TYPE_NAME" BIGINT NOT NULL, "COMMENT" VARCHAR(256), "FIELD_NAME" VARCHAR(128) NOT NULL, "FIELD_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); | ||
|
||
CREATE TABLE "APP"."NUCLEUS_TABLES" ("CLASS_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(128) NOT NULL, "TYPE" VARCHAR(4) NOT NULL, "OWNER" VARCHAR(2) NOT NULL, "VERSION" VARCHAR(20) NOT NULL, "INTERFACE_NAME" VARCHAR(256) DEFAULT NULL); | ||
|
||
CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); | ||
CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); | ||
|
||
CREATE TABLE "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID" BIGINT NOT NULL); | ||
|
||
|
@@ -218,7 +218,7 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" ( | |
"CAT_NAME" VARCHAR(256) NOT NULL, | ||
"DB_NAME" VARCHAR(128) NOT NULL, | ||
"TBL_NAME" VARCHAR(256) NOT NULL, | ||
"TXN_LIST" CLOB, | ||
"TXN_LIST" VARCHAR(32672), | ||
"MATERIALIZATION_TIME" BIGINT NOT NULL | ||
); | ||
|
||
|
Uh oh!
There was an error while loading. Please reload this page.