From ec4953ba26eca99cc7b31709d0bd503cefbb8f9b Mon Sep 17 00:00:00 2001 From: lw112 <131352377+felixwluo@users.noreply.github.com> Date: Fri, 7 Nov 2025 11:36:49 +0800 Subject: [PATCH] [fix](catalog) fix iceberg catalog NPE issue (#57696) ### What problem does this PR solve? Problem Summary: error message: ``` 2025-11-04 15:33:05,766 WARN (mysql-nio-pool-196|193) [StmtExecutor.executeByNereids():853] Nereids plan query failed: select * from iceberg_catalog.bss.bss_customer limit 1 java.lang.RuntimeException: NullPointerException: Cannot invoke "org.apache.iceberg.catalog.Catalog.loadTable(org.apache.iceberg.catalog.TableIdentifier)" because "this.catalog" is null at org.apache.doris.datasource.iceberg.IcebergUtils.getSchema(IcebergUtils.java:591) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.hive.HMSExternalTable.getIcebergSchema(HMSExternalTable.java:554) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.hive.HMSExternalTable.initSchema(HMSExternalTable.java:545) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.hive.HMSExternalTable.initSchemaAndUpdateTime(HMSExternalTable.java:529) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.hive.HMSExternalTable.initSchemaAndUpdateTime(HMSExternalTable.java:517) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalCatalog.getSchema(ExternalCatalog.java:595) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalSchemaCache.loadSchema(ExternalSchemaCache.java:81) ~[doris-fe.jar:1.2-SNAPSHOT] at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:145) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916) ~[?:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2404) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2387) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:56) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at org.apache.doris.datasource.ExternalSchemaCache.getSchemaValue(ExternalSchemaCache.java:94) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalSchemaCache.getSchemaValue(ExternalSchemaCache.java:90) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalTable.getFullSchema(ExternalTable.java:171) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalTable.getBaseSchema(ExternalTable.java:177) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation.computeOutput(LogicalCatalogRelation.java:103) ~[doris-fe.jar:1.2-SNAPSHOT] at com.google.common.base.Suppliers$NonSerializableMemoizingSupplier.get(Suppliers.java:186) ~[guava-33.2.1-jre.jar:?] at org.apache.doris.nereids.properties.LogicalProperties.getOutput(LogicalProperties.java:104) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.rules.analysis.BindRelation.lambda$build$0(BindRelation.java:123) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.pattern.PatternMatcher$1.transform(PatternMatcher.java:92) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteJob.rewrite(PlanTreeRewriteJob.java:57) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob.rewriteThis(PlanTreeRewriteBottomUpJob.java:91) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob.execute(PlanTreeRewriteBottomUpJob.java:75) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler.executeJobPool(SimpleJobScheduler.java:44) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.execute(RootPlanTreeRewriteJob.java:66) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor.execute(AbstractBatchJobExecutor.java:139) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.jobs.executor.Analyzer.analyze(Analyzer.java:83) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.lambda$analyze$4(NereidsPlanner.java:365) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.keepOrShowPlanProcess(NereidsPlanner.java:898) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.analyze(NereidsPlanner.java:365) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.planWithoutLock(NereidsPlanner.java:250) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.planWithLock(NereidsPlanner.java:224) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:145) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:847) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:625) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:587) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:572) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:375) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:256) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:229) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:258) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:438) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: NullPointerException: Cannot invoke "org.apache.iceberg.catalog.Catalog.loadTable(org.apache.iceberg.catalog.TableIdentifier)" because "this.catalog" is null at org.apache.doris.datasource.iceberg.IcebergMetadataCache.loadTable(IcebergMetadataCache.java:117) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.lambda$new$1(IcebergMetadataCache.java:68) ~[doris-fe.jar:1.2-SNAPSHOT] at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:145) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916) ~[?:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2404) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2387) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:56) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.getIcebergTable(IcebergMetadataCache.java:83) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getIcebergTableInternal(IcebergUtils.java:570) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getIcebergTable(IcebergUtils.java:549) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.lambda$getSchema$1(IcebergUtils.java:579) ~[doris-fe.jar:1.2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899) ~[hadoop-common-3.3.6.jar:?] at org.apache.doris.common.security.authentication.HadoopAuthenticator.doAs(HadoopAuthenticator.java:32) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.common.security.authentication.PreExecutionAuthenticator.execute(PreExecutionAuthenticator.java:71) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getSchema(IcebergUtils.java:578) ~[doris-fe.jar:1.2-SNAPSHOT] ... 48 more Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.iceberg.catalog.Catalog.loadTable(org.apache.iceberg.catalog.TableIdentifier)" because "this.catalog" is null at org.apache.doris.datasource.iceberg.IcebergMetadataOps.loadTable(IcebergMetadataOps.java:278) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.lambda$loadTable$2(IcebergMetadataCache.java:115) ~[doris-fe.jar:1.2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899) ~[hadoop-common-3.3.6.jar:?] at org.apache.doris.common.security.authentication.HadoopAuthenticator.doAs(HadoopAuthenticator.java:32) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.common.security.authentication.PreExecutionAuthenticator.execute(PreExecutionAuthenticator.java:71) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.loadTable(IcebergMetadataCache.java:114) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.lambda$new$1(IcebergMetadataCache.java:68) ~[doris-fe.jar:1.2-SNAPSHOT] at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:145) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916) ~[?:?] at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2404) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2387) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:56) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at org.apache.doris.datasource.iceberg.IcebergMetadataCache.getIcebergTable(IcebergMetadataCache.java:83) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getIcebergTableInternal(IcebergUtils.java:570) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getIcebergTable(IcebergUtils.java:549) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.lambda$getSchema$1(IcebergUtils.java:579) ~[doris-fe.jar:1.2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1899) ~[hadoop-common-3.3.6.jar:?] at org.apache.doris.common.security.authentication.HadoopAuthenticator.doAs(HadoopAuthenticator.java:32) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.common.security.authentication.PreExecutionAuthenticator.execute(PreExecutionAuthenticator.java:71) ~[fe-common-1.2-SNAPSHOT.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.iceberg.IcebergUtils.getSchema(IcebergUtils.java:578) ~[doris-fe.jar:1.2-SNAPSHOT] ... 48 more ``` --- .../apache/doris/datasource/hive/HMSExternalCatalog.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index ceb4755b2c5f12..0711775963e7ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -155,9 +155,6 @@ protected void initLocalObjectsImpl() { @Override public synchronized void resetToUninitialized(boolean invalidCache) { super.resetToUninitialized(invalidCache); - if (metadataOps != null) { - metadataOps.close(); - } } @Override @@ -166,8 +163,13 @@ public void onClose() { if (null != fileSystemExecutor) { fileSystemExecutor.shutdown(); } + if (null != metadataOps) { + metadataOps.close(); + metadataOps = null; + } if (null != icebergMetadataOps) { icebergMetadataOps.close(); + icebergMetadataOps = null; } }