From 7411b1014c7232b22b62864cc1a4139ccbd3ebcb Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 20 Mar 2025 17:55:15 +0800 Subject: [PATCH 01/54] Finish window function plan prototype. --- .../function/WindowFunctionFactory.java | 10 + .../window/partition/frame/FrameInfo.java | 58 +++ .../window/partition/frame/RowsFrame.java | 24 +- .../process/window/utils/RowComparator.java | 1 + .../plan/planner/TableOperatorGenerator.java | 187 ++++++++ .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/relational/analyzer/Analysis.java | 122 ++++- .../analyzer/ExpressionAnalyzer.java | 274 ++++++++++- .../analyzer/ExpressionTreeUtils.java | 11 +- .../analyzer/StatementAnalyzer.java | 278 +++++++---- .../plan/relational/planner/QueryPlanner.java | 451 ++++++++++++++++++ .../relational/planner/RelationPlanner.java | 16 +- .../TableDistributedPlanGenerator.java | 18 + .../relational/planner/node/WindowNode.java | 340 +++++++++++++ .../planner/optimizations/SymbolMapper.java | 43 ++ .../UnaliasSymbolReferences.java | 12 + .../plan/relational/sql/ast/AstVisitor.java | 12 + .../plan/relational/sql/ast/FrameBound.java | 85 ++++ .../plan/relational/sql/ast/FunctionCall.java | 26 + .../plan/relational/sql/ast/WindowFrame.java | 86 ++++ .../sql/ast/WindowSpecification.java | 96 ++++ .../relational/sql/parser/AstBuilder.java | 243 ++++------ .../sql/util/ExpressionFormatter.java | 127 ++--- .../utils/DataOrganizationSpecification.java | 55 +++ .../relational/grammar/sql/RelationalSql.g4 | 37 +- 25 files changed, 2285 insertions(+), 332 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java new file mode 100644 index 0000000000000..29ff5376493fe --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java @@ -0,0 +1,10 @@ +package org.apache.iotdb.db.queryengine.execution.operator.process.window.function; + +import java.util.List; + +public class WindowFunctionFactory { + public static WindowFunction createBuiltinWindowFunction( + String functionName, List argumentChannels) { + return null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameInfo.java index eac6126e49811..78cb7d2b9e4db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameInfo.java @@ -20,8 +20,66 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.frame; import org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; + +import java.util.Optional; public class FrameInfo { + public FrameInfo( + WindowFrame.Type frameType, + FrameBound.Type startType, + Optional startOffsetChannel, + FrameBound.Type endType, + Optional endOffsetChannel, + Optional sortChannel, + Optional sortOrder) { + this.frameType = convertFrameType(frameType); + this.startType = convertFrameBoundType(startType); + this.startOffsetChannel = startOffsetChannel.orElse(-1); + this.endType = convertFrameBoundType(endType); + this.endOffsetChannel = endOffsetChannel.orElse(-1); + + if (sortChannel.isPresent()) { + assert sortOrder.isPresent(); + this.sortChannel = sortChannel.get(); + this.sortOrder = sortOrder.get(); + } else { + this.sortChannel = -1; + this.sortOrder = SortOrder.ASC_NULLS_FIRST; + } + } + + private FrameType convertFrameType(WindowFrame.Type frameType) { + switch (frameType) { + case RANGE: + return FrameType.RANGE; + case ROWS: + return FrameType.ROWS; + case GROUPS: + return FrameType.GROUPS; + default: + throw new IllegalArgumentException("Unsupported frame bound type: " + frameType); + } + } + + private FrameBoundType convertFrameBoundType(FrameBound.Type frameBoundType) { + switch (frameBoundType) { + case UNBOUNDED_PRECEDING: + return FrameBoundType.UNBOUNDED_PRECEDING; + case UNBOUNDED_FOLLOWING: + return FrameBoundType.UNBOUNDED_FOLLOWING; + case CURRENT_ROW: + return FrameBoundType.CURRENT_ROW; + case PRECEDING: + return FrameBoundType.PRECEDING; + case FOLLOWING: + return FrameBoundType.FOLLOWING; + default: + throw new IllegalArgumentException("Unsupported frame bound type: " + frameBoundType); + } + } + public enum FrameType { RANGE, ROWS, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java index 2177bb68c8e3e..ef9fa4b7baf9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java @@ -43,8 +43,6 @@ public RowsFrame(Partition partition, FrameInfo frameInfo, int partitionStart, i @Override public Range getRange( int currentPosition, int currentGroup, int peerGroupStart, int peerGroupEnd) { - int posInPartition = currentPosition - partitionStart; - int offset; int frameStart; switch (frameInfo.getStartType()) { @@ -52,15 +50,17 @@ public Range getRange( frameStart = 0; break; case PRECEDING: - offset = (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); - frameStart = posInPartition - offset; + offset = + (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition + partitionStart); + frameStart = currentPosition - offset; break; case CURRENT_ROW: - frameStart = posInPartition; + frameStart = currentPosition; break; case FOLLOWING: - offset = (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); - frameStart = posInPartition + offset; + offset = + (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition + partitionStart); + frameStart = currentPosition + offset; break; default: // UNBOUND_FOLLOWING is not allowed in frame start @@ -70,15 +70,15 @@ public Range getRange( int frameEnd; switch (frameInfo.getEndType()) { case PRECEDING: - offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition); - frameEnd = posInPartition - offset; + offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition + partitionStart); + frameEnd = currentPosition - offset; break; case CURRENT_ROW: - frameEnd = posInPartition; + frameEnd = currentPosition; break; case FOLLOWING: - offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition); - frameEnd = posInPartition + offset; + offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition + partitionStart); + frameEnd = currentPosition + offset; break; case UNBOUNDED_FOLLOWING: frameEnd = partitionSize - 1; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index 768eeaed85429..5823a83875be8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -87,6 +87,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse } break; case TEXT: + case STRING: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 860005397dbd7..b05774c58cbd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -69,6 +69,12 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.join.SimpleNestedLoopCrossJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparatorFactory; import org.apache.iotdb.db.queryengine.execution.operator.process.last.LastQueryUtil; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunctionFactory; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.AggregationWindowFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.WindowAggregator; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.frame.FrameInfo; import org.apache.iotdb.db.queryengine.execution.operator.schema.CountMergeOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaCountOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaQueryScanOperator; @@ -111,9 +117,11 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionKind; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.planner.CastToBlobLiteralVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.CastToBooleanLiteralVisitor; @@ -154,6 +162,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryScanNode; @@ -230,6 +239,7 @@ import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.InformationSchemaContentSupplierFactory.getSupplier; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.constructAlignedPath; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createAccumulator; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createBuiltinAccumulator; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createGroupedAccumulator; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.hash.GroupByHash.DEFAULT_GROUP_NUMBER; import static org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.convertPredicateToFilter; @@ -2448,4 +2458,181 @@ public Operator visitMarkDistinct(MarkDistinctNode node, LocalExecutionPlanConte node.getDistinctSymbols().stream().map(childLayout::get).collect(Collectors.toList()), Optional.empty()); } + + @Override + public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext context) { + TypeProvider typeProvider = context.getTypeProvider(); + Operator child = node.getChild().accept(this, context); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + TableWindowOperator.class.getSimpleName()); + + Map childLayout = + makeLayoutFromOutputSymbols(node.getChild().getOutputSymbols()); + + // Partition channel + List partitionBySymbols = node.getSpecification().getPartitionBy(); + List partitionChannels = + ImmutableList.copyOf(getChannelsForSymbols(partitionBySymbols, childLayout)); + + // Sort channel + List sortChannels = ImmutableList.of(); + // List sortOrder = ImmutableList.of(); + if (node.getSpecification().getOrderingScheme().isPresent()) { + OrderingScheme orderingScheme = node.getSpecification().getOrderingScheme().get(); + sortChannels = getChannelsForSymbols(orderingScheme.getOrderBy(), childLayout); + // sortOrder = orderingScheme.getOrderingList(); + } + + // Output channel + ImmutableList.Builder outputChannels = ImmutableList.builder(); + List outputDataTypes = new ArrayList<>(); + List inputDataTypes = + getOutputColumnTypes(node.getChild(), context.getTypeProvider()); + for (int i = 0; i < inputDataTypes.size(); i++) { + outputChannels.add(i); + outputDataTypes.add(inputDataTypes.get(i)); + } + + // Window functions + List frameInfoList = new ArrayList<>(); + List windowFunctions = new ArrayList<>(); + List windowFunctionOutputSymbols = new ArrayList<>(); + List windowFunctionOutputDataTypes = new ArrayList<>(); + for (Map.Entry entry : node.getWindowFunctions().entrySet()) { + // Create FrameInfo + WindowNode.Frame frame = entry.getValue().getFrame(); + + Optional frameStartChannel = Optional.empty(); + if (frame.getStartValue().isPresent()) { + frameStartChannel = Optional.ofNullable(childLayout.get(frame.getStartValue().get())); + } + // Optional sortKeyChannelForStartComparison = Optional.empty(); + // if (frame.getSortKeyCoercedForFrameStartComparison().isPresent()) { + // sortKeyChannelForStartComparison = + // Optional.ofNullable(childLayout.get(frame.getSortKeyCoercedForFrameStartComparison().get())); + // } + Optional frameEndChannel = Optional.empty(); + if (frame.getEndValue().isPresent()) { + frameEndChannel = Optional.ofNullable(childLayout.get(frame.getEndValue().get())); + } + // Optional sortKeyChannelForEndComparison = Optional.empty(); + // if (frame.getSortKeyCoercedForFrameEndComparison().isPresent()) { + // sortKeyChannelForEndComparison = + // Optional.ofNullable(childLayout.get(frame.getSortKeyCoercedForFrameEndComparison().get())); + // } + Optional sortKeyChannel = Optional.empty(); + Optional ordering = Optional.empty(); + if (node.getSpecification().getOrderingScheme().isPresent()) { + sortKeyChannel = Optional.of(sortChannels.get(0)); + // if (sortOrder.get(0).isNullsFirst()) { + // if (sortOrder.get(0).isAscending()) { + // ordering = Optional.of(ASC_NULLS_FIRST); + // } else { + // ordering = Optional.of(DESC_NULLS_FIRST); + // } + // } else { + // if (sortOrder.get(0).isAscending()) { + // ordering = Optional.of(ASC_NULLS_LAST); + // } else { + // ordering = Optional.of(DESC_NULLS_LAST); + // } + // } + } + FrameInfo frameInfo = + new FrameInfo( + frame.getType(), + frame.getStartType(), + frameStartChannel, + frame.getEndType(), + frameEndChannel, + sortKeyChannel, + ordering); + frameInfoList.add(frameInfo); + + // Arguments + WindowNode.Function function = entry.getValue(); + ResolvedFunction resolvedFunction = function.getResolvedFunction(); + List argumentChannels = new ArrayList<>(); + for (Expression argument : function.getArguments()) { + Symbol argumentSymbol = Symbol.from(argument); + argumentChannels.add(childLayout.get(argumentSymbol)); + } + + // Return value + Type returnType = resolvedFunction.getSignature().getReturnType(); + windowFunctionOutputDataTypes.add(getTSDataType(returnType)); + + // Window function + Symbol symbol = entry.getKey(); + WindowFunction windowFunction; + FunctionKind functionKind = resolvedFunction.getFunctionKind(); + if (functionKind == FunctionKind.AGGREGATE) { + WindowAggregator tableWindowAggregator = + buildWindowAggregator(symbol, function, typeProvider, argumentChannels); + windowFunction = new AggregationWindowFunction(tableWindowAggregator); + } else if (functionKind == FunctionKind.WINDOW) { + String functionName = function.getResolvedFunction().getSignature().getName(); + windowFunction = + WindowFunctionFactory.createBuiltinWindowFunction(functionName, argumentChannels); + } else { + throw new UnsupportedOperationException("Unsupported function kind: " + functionKind); + } + + windowFunctions.add(windowFunction); + windowFunctionOutputSymbols.add(symbol); + } + + // Compute layout + ImmutableMap.Builder outputMappings = ImmutableMap.builder(); + for (Symbol symbol : node.getChild().getOutputSymbols()) { + outputMappings.put(symbol, childLayout.get(symbol)); + } + int channel = inputDataTypes.size(); + + for (Symbol symbol : windowFunctionOutputSymbols) { + outputMappings.put(symbol, channel); + channel++; + } + + outputDataTypes.addAll(windowFunctionOutputDataTypes); + return new TableWindowOperator( + operatorContext, + child, + inputDataTypes, + outputDataTypes, + outputChannels.build(), + windowFunctions, + frameInfoList, + partitionChannels, + sortChannels); + } + + private WindowAggregator buildWindowAggregator( + Symbol symbol, + WindowNode.Function function, + TypeProvider typeProvider, + List argumentChannels) { + // Create accumulator first + String functionName = function.getResolvedFunction().getSignature().getName(); + List originalArgumentTypes = + function.getResolvedFunction().getSignature().getArgumentTypes().stream() + .map(InternalTypeManager::getTSDataType) + .collect(Collectors.toList()); + TableAccumulator accumulator = + createBuiltinAccumulator( + getAggregationTypeByFuncName(functionName), + originalArgumentTypes, + function.getArguments(), + Collections.emptyMap(), + true); + + // Create aggregator by accumulator + return new WindowAggregator( + accumulator, getTSDataType(typeProvider.getTableModelType(symbol)), argumentChannels); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 8660e6145407f..6b5d7e8b73044 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -134,6 +134,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; @@ -800,4 +801,8 @@ public R visitTreeNonAlignedDeviceViewScan(TreeNonAlignedDeviceViewScanNode node public R visitMarkDistinct(MarkDistinctNode node, C context) { return visitSingleChildProcess(node, context); } + + public R visitWindowFunction(WindowNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 1f3873067b4d7..ab1b62dfd887e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -62,6 +62,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import com.google.common.collect.ArrayListMultimap; @@ -153,6 +154,12 @@ public class Analysis implements IAnalysis { private final Map, Type> coercions = new LinkedHashMap<>(); private final Set> typeOnlyCoercions = new LinkedHashSet<>(); + private final Map, Type> sortKeyCoercionsForFrameBoundCalculation = + new LinkedHashMap<>(); + private final Map, Type> sortKeyCoercionsForFrameBoundComparison = + new LinkedHashMap<>(); + private final Map, ResolvedFunction> frameBoundCalculations = + new LinkedHashMap<>(); private final Map, List> relationCoercions = new LinkedHashMap<>(); private final Map, RoutineEntry> resolvedFunctions = new LinkedHashMap<>(); @@ -186,6 +193,10 @@ public class Analysis implements IAnalysis { private final Map> tableColumnSchemas = new HashMap<>(); + private final Map, ResolvedWindow> windows = new LinkedHashMap<>(); + private final Map, List> windowFunctions = + new LinkedHashMap<>(); + private DataPartition dataPartition; // only be used in write plan and won't be used in query @@ -399,9 +410,31 @@ public void addCoercion(Expression expression, Type type, boolean isTypeOnlyCoer } public void addCoercions( - Map, Type> coercions, Set> typeOnlyCoercions) { + Map, Type> coercions, + Set> typeOnlyCoercions, + Map, Type> sortKeyCoercionsForFrameBoundCalculation, + Map, Type> sortKeyCoercionsForFrameBoundComparison) { this.coercions.putAll(coercions); this.typeOnlyCoercions.addAll(typeOnlyCoercions); + this.sortKeyCoercionsForFrameBoundCalculation.putAll(sortKeyCoercionsForFrameBoundCalculation); + this.sortKeyCoercionsForFrameBoundComparison.putAll(sortKeyCoercionsForFrameBoundComparison); + } + + public Type getSortKeyCoercionForFrameBoundCalculation(Expression frameOffset) { + return sortKeyCoercionsForFrameBoundCalculation.get(NodeRef.of(frameOffset)); + } + + public Type getSortKeyCoercionForFrameBoundComparison(Expression frameOffset) { + return sortKeyCoercionsForFrameBoundComparison.get(NodeRef.of(frameOffset)); + } + + public void addFrameBoundCalculations( + Map, ResolvedFunction> frameBoundCalculations) { + this.frameBoundCalculations.putAll(frameBoundCalculations); + } + + public ResolvedFunction getFrameBoundCalculation(Expression frameOffset) { + return frameBoundCalculations.get(NodeRef.of(frameOffset)); } public Set> getTypeOnlyCoercions() { @@ -1157,6 +1190,93 @@ public Optional getSubqueryCoercion() { } } + public void setWindow(Node node, ResolvedWindow window) { + windows.put(NodeRef.of(node), window); + } + + public ResolvedWindow getWindow(Node node) { + return windows.get(NodeRef.of(node)); + } + + public void setWindowFunctions(QuerySpecification node, List functions) { + windowFunctions.put(NodeRef.of(node), ImmutableList.copyOf(functions)); + } + + public List getWindowFunctions(QuerySpecification query) { + return windowFunctions.get(NodeRef.of(query)); + } + + public static class ResolvedWindow { + private final List partitionBy; + private final Optional orderBy; + private final Optional frame; + private final boolean partitionByInherited; + private final boolean orderByInherited; + private final boolean frameInherited; + + public ResolvedWindow( + List partitionBy, + Optional orderBy, + Optional frame, + boolean partitionByInherited, + boolean orderByInherited, + boolean frameInherited) { + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null"); + this.orderBy = requireNonNull(orderBy, "orderBy is null"); + this.frame = requireNonNull(frame, "frame is null"); + this.partitionByInherited = partitionByInherited; + this.orderByInherited = orderByInherited; + this.frameInherited = frameInherited; + } + + public List getPartitionBy() { + return partitionBy; + } + + public Optional getOrderBy() { + return orderBy; + } + + public Optional getFrame() { + return frame; + } + + public boolean isPartitionByInherited() { + return partitionByInherited; + } + + public boolean isOrderByInherited() { + return orderByInherited; + } + + public boolean isFrameInherited() { + return frameInherited; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ResolvedWindow that = (ResolvedWindow) o; + return partitionByInherited == that.partitionByInherited + && orderByInherited == that.orderByInherited + && frameInherited == that.frameInherited + && partitionBy.equals(that.partitionBy) + && orderBy.equals(that.orderBy) + && frame.equals(that.frame); + } + + @Override + public int hashCode() { + return Objects.hash( + partitionBy, orderBy, frame, partitionByInherited, orderByInherited, frameInherited); + } + } + public static class FillAnalysis { protected final FillPolicy fillMethod; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 1798a0ed0b7ef..9e06bb781157f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; @@ -68,22 +69,26 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StackableAstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; import com.google.common.collect.Multimap; import org.apache.tsfile.read.common.type.RowType; @@ -116,7 +121,16 @@ import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isNumericType; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isTwoTypeComparable; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression.isQualifiedAllFieldsReference; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.CURRENT_ROW; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.FOLLOWING; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.PRECEDING; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.UNBOUNDED_FOLLOWING; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.UNBOUNDED_PRECEDING; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.GROUPS; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.RANGE; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.ROWS; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.NodeUtils.getSortItemsFromOrderBy; import static org.apache.tsfile.read.common.type.BlobType.BLOB; import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE; @@ -147,6 +161,7 @@ public class ExpressionAnalyzer { private final Set> quantifiedComparisons = new LinkedHashSet<>(); + private final Set> windowFunctions = new LinkedHashSet<>(); private final Multimap tableColumnReferences = HashMultimap.create(); // Track referenced fields from source relation node @@ -166,6 +181,8 @@ public class ExpressionAnalyzer { private final Map, LabelPrefixedReference> labelDereferences = new LinkedHashMap<>(); + private final Function getResolvedWindow; + private static final String SUBQUERY_COLUMN_NUM_CHECK = "Subquery must return only one column for now. Row Type is not supported for now."; @@ -189,7 +206,8 @@ private ExpressionAnalyzer( types, analysis.getParameters(), warningCollector, - analysis::getType); + analysis::getType, + analysis::getWindow); } ExpressionAnalyzer( @@ -201,7 +219,8 @@ private ExpressionAnalyzer( TypeProvider symbolTypes, Map, Expression> parameters, WarningCollector warningCollector, - Function getPreanalyzedType) { + Function getPreanalyzedType, + Function getResolvedWindow) { this.metadata = requireNonNull(metadata, "metadata is null"); this.context = requireNonNull(context, "context is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); @@ -211,6 +230,7 @@ private ExpressionAnalyzer( this.symbolTypes = requireNonNull(symbolTypes, "symbolTypes is null"); this.parameters = requireNonNull(parameters, "parameters is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); + this.getResolvedWindow = requireNonNull(getResolvedWindow, "getResolvedWindow is null"); this.getPreanalyzedType = requireNonNull(getPreanalyzedType, "getPreanalyzedType is null"); } @@ -777,8 +797,16 @@ protected Type visitFunctionCall( } }); - if (node.isDistinct() && !isAggregation) { - throw new SemanticException("DISTINCT is not supported for non-aggregation functions"); + if (node.getWindow().isPresent()) { + Analysis.ResolvedWindow window = getResolvedWindow.apply(node); + checkState(window != null, "no resolved window for: " + node); + + analyzeWindow(window, context, (Node) node.getWindow().get()); + windowFunctions.add(NodeRef.of(node)); + } else { + if (node.isDistinct() && !isAggregation) { + throw new SemanticException("DISTINCT is not supported for non-aggregation functions"); + } } List argumentTypes = getCallArgumentTypes(node.getArguments(), context); @@ -1062,6 +1090,238 @@ private Type analyzeSubquery( return getOnlyElement(fields.build().stream().iterator()).getType(); } + private void analyzeWindow( + Analysis.ResolvedWindow window, + StackableAstVisitor.StackableAstVisitorContext context, + Node originalNode) { + // check no nested window functions + ImmutableList.Builder childNodes = ImmutableList.builder(); + if (!window.isPartitionByInherited()) { + childNodes.addAll(window.getPartitionBy()); + } + if (!window.isOrderByInherited()) { + window.getOrderBy().ifPresent(orderBy -> childNodes.addAll(orderBy.getSortItems())); + } + if (!window.isFrameInherited()) { + window.getFrame().ifPresent(childNodes::add); + } + + if (!window.isPartitionByInherited()) { + for (Expression expression : window.getPartitionBy()) { + process(expression, context); + Type type = getExpressionType(expression); + if (!type.isComparable()) { + throw new SemanticException( + String.format( + "%s is not comparable, and therefore cannot be used in window function PARTITION BY", + type)); + } + } + } + + if (!window.isOrderByInherited()) { + for (SortItem sortItem : getSortItemsFromOrderBy(window.getOrderBy())) { + process(sortItem.getSortKey(), context); + Type type = getExpressionType(sortItem.getSortKey()); + if (!type.isOrderable()) { + throw new SemanticException( + String.format( + "%s is not orderable, and therefore cannot be used in window function ORDER BY", + type)); + } + } + } + + if (window.getFrame().isPresent() && !window.isFrameInherited()) { + WindowFrame frame = window.getFrame().get(); + + // validate frame start and end types + FrameBound.Type startType = frame.getStart().getType(); + FrameBound.Type endType = + frame.getEnd().orElse(new FrameBound(null, CURRENT_ROW)).getType(); + if (startType == UNBOUNDED_FOLLOWING) { + throw new SemanticException("Window frame start cannot be UNBOUNDED FOLLOWING"); + } + if (endType == UNBOUNDED_PRECEDING) { + throw new SemanticException("Window frame end cannot be UNBOUNDED PRECEDING"); + } + if ((startType == CURRENT_ROW) && (endType == PRECEDING)) { + throw new SemanticException( + "Window frame starting from CURRENT ROW cannot end with PRECEDING"); + } + if ((startType == FOLLOWING) && (endType == PRECEDING)) { + throw new SemanticException( + "Window frame starting from FOLLOWING cannot end with PRECEDING"); + } + if ((startType == FOLLOWING) && (endType == CURRENT_ROW)) { + throw new SemanticException( + "Window frame starting from FOLLOWING cannot end with CURRENT ROW"); + } + + // analyze frame offset values + if (frame.getType() == ROWS) { + if (frame.getStart().getValue().isPresent()) { + Expression startValue = frame.getStart().getValue().get(); + Type type = process(startValue, context); + if (!isExactNumericWithScaleZero(type)) { + throw new SemanticException( + String.format( + "Window frame ROWS start value type must be exact numeric type with scale 0 (actual %s)", + type)); + } + } + if (frame.getEnd().isPresent() && frame.getEnd().get().getValue().isPresent()) { + Expression endValue = frame.getEnd().get().getValue().get(); + Type type = process(endValue, context); + if (!isExactNumericWithScaleZero(type)) { + throw new SemanticException( + String.format( + "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", + type)); + } + } + } else if (frame.getType() == RANGE) { + if (frame.getStart().getValue().isPresent()) { + Expression startValue = frame.getStart().getValue().get(); + analyzeFrameRangeOffset( + startValue, frame.getStart().getType(), context, window, originalNode); + } + if (frame.getEnd().isPresent() && frame.getEnd().get().getValue().isPresent()) { + Expression endValue = frame.getEnd().get().getValue().get(); + analyzeFrameRangeOffset( + endValue, frame.getEnd().get().getType(), context, window, originalNode); + } + } else if (frame.getType() == GROUPS) { + if (frame.getStart().getValue().isPresent()) { + if (!window.getOrderBy().isPresent()) { + throw new SemanticException( + "Window frame of type GROUPS PRECEDING or FOLLOWING requires ORDER BY"); + } + Expression startValue = frame.getStart().getValue().get(); + Type type = process(startValue, context); + if (!isExactNumericWithScaleZero(type)) { + throw new SemanticException( + String.format( + "Window frame GROUPS start value type must be exact numeric type with scale 0 (actual %s)", + type)); + } + } + if (frame.getEnd().isPresent() && frame.getEnd().get().getValue().isPresent()) { + if (!window.getOrderBy().isPresent()) { + throw new SemanticException( + "Window frame of type GROUPS PRECEDING or FOLLOWING requires ORDER BY"); + } + Expression endValue = frame.getEnd().get().getValue().get(); + Type type = process(endValue, context); + if (!isExactNumericWithScaleZero(type)) { + throw new SemanticException( + String.format( + "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", + type)); + } + } + } else { + throw new SemanticException("Unsupported frame type: " + frame.getType()); + } + } + } + + private void analyzeFrameRangeOffset( + Expression offsetValue, + FrameBound.Type boundType, + StackableAstVisitorContext context, + Analysis.ResolvedWindow window, + Node originalNode) { + OrderBy orderBy = + window + .getOrderBy() + .orElseThrow( + () -> + new SemanticException( + "Window frame of type RANGE PRECEDING or FOLLOWING requires ORDER BY")); + if (orderBy.getSortItems().size() != 1) { + throw new SemanticException( + "Window frame of type RANGE PRECEDING or FOLLOWING requires single sort item in ORDER BY (actual: %s)", + orderBy.getSortItems().size()); + } + Expression sortKey = Iterables.getOnlyElement(orderBy.getSortItems()).getSortKey(); + Type sortKeyType; + if (window.isOrderByInherited()) { + sortKeyType = getPreanalyzedType.apply(sortKey); + } else { + sortKeyType = getExpressionType(sortKey); + } + if (!isNumericType(sortKeyType)) { + throw new SemanticException( + String.format( + "Window frame of type RANGE PRECEDING or FOLLOWING requires that sort item type be numeric, datetime or interval (actual: %s)", + sortKeyType)); + } + + Type offsetValueType = process(offsetValue, context); + + if (isNumericType(sortKeyType)) { + if (!isNumericType(offsetValueType)) { + throw new SemanticException( + String.format( + "Window frame RANGE value type (%s) not compatible with sort item type (%s)", + offsetValueType, sortKeyType)); + } + } + + // // resolve function to calculate frame boundary value (add / subtract offset from + // sortKey) + // SortItem.Ordering ordering = + // Iterables.getOnlyElement(orderBy.getSortItems()).getOrdering(); + // OperatorType operatorType; + // ResolvedFunction function; + // if ((boundType == PRECEDING && ordering == ASCENDING) || (boundType == FOLLOWING && + // ordering == DESCENDING)) { + // operatorType = SUBTRACT; + // } + // else { + // operatorType = ADD; + // } + // try { + // function = metadata.resolveOperator(operatorType, ImmutableList.of(sortKeyType, + // offsetValueType)); + // } + // catch (TrinoException e) { + // ErrorCode errorCode = e.getErrorCode(); + // if (errorCode.equals(OPERATOR_NOT_FOUND.toErrorCode())) { + // throw semanticException(TYPE_MISMATCH, offsetValue, "Window frame RANGE value type + // (%s) not compatible with sort item type (%s)", offsetValueType, sortKeyType); + // } + // throw e; + // } + // BoundSignature signature = function.getSignature(); + // Type expectedSortKeyType = signature.getArgumentTypes().get(0); + // if (!expectedSortKeyType.equals(sortKeyType)) { + // if (!typeCoercion.canCoerce(sortKeyType, expectedSortKeyType)) { + // throw semanticException(TYPE_MISMATCH, sortKey, "Sort key must evaluate to a %s + // (actual: %s)", expectedSortKeyType, sortKeyType); + // } + // sortKeyCoercionsForFrameBoundCalculation.put(NodeRef.of(offsetValue), + // expectedSortKeyType); + // } + // Type expectedOffsetValueType = signature.getArgumentTypes().get(1); + // if (!expectedOffsetValueType.equals(offsetValueType)) { + // coerceType(offsetValue, offsetValueType, expectedOffsetValueType, format("Function + // %s argument 1", function)); + // } + // Type expectedFunctionResultType = signature.getReturnType(); + // if (!expectedFunctionResultType.equals(sortKeyType)) { + // if (!typeCoercion.canCoerce(sortKeyType, expectedFunctionResultType)) { + // throw semanticException(TYPE_MISMATCH, sortKey, "Sort key must evaluate to a %s + // (actual: %s)", expectedFunctionResultType, sortKeyType); + // } + // sortKeyCoercionsForFrameBoundComparison.put(NodeRef.of(offsetValue), + // expectedFunctionResultType); + // } + // + // frameBoundCalculations.put(NodeRef.of(offsetValue), function); + } + @Override protected Type visitExists(ExistsPredicate node, StackableAstVisitorContext context) { StatementAnalyzer analyzer = @@ -1476,7 +1736,8 @@ public static void analyzeExpressionWithoutSubqueries( TypeProvider.empty(), analysis.getParameters(), warningCollector, - analysis::getType); + analysis::getType, + analysis::getWindow); analyzer.analyze(expression, scope, correlationSupport); updateAnalysis(analysis, analyzer, session, accessControl); @@ -1558,6 +1819,9 @@ public static ExpressionAnalyzer createWithoutSubqueries( warningCollector, expression -> { throw new IllegalStateException("Cannot access preanalyzed types"); + }, + functionCall -> { + throw new IllegalStateException("Cannot access resolved windows"); }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java index f8cec46dfd19b..e84a8a87113aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java @@ -65,8 +65,17 @@ private static List extractExpressions( .collect(toImmutableList()); } + public static List extractWindowFunctions(Iterable nodes) { + return extractExpressions(nodes, FunctionCall.class, ExpressionTreeUtils::isWindowFunction); + } + + private static boolean isWindowFunction(FunctionCall functionCall) { + return functionCall.getWindow().isPresent(); + } + private static boolean isAggregation(FunctionCall functionCall) { - return isAggregationFunction(functionCall.getName().toString()); + return isAggregationFunction(functionCall.getName().toString()) + && !functionCall.getWindow().isPresent(); } private static List linearizeNodes(Node node) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index edf7f9980c038..7f6f2450cd1f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -41,104 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.TranslationMap; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractQueryDeviceWithCache; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AbstractTraverseDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateOrUpdateDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTopic; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTopic; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingElement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Insert; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRow; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetOperation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowSubscriptions; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Union; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Update; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UpdateAssignment; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.*; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -192,6 +95,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.CanonicalizationAware.canonicalizationAwareKey; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.asQualifiedName; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractAggregateFunctions; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractWindowFunctions; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.Scope.BasisType.TABLE; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.MetadataUtil.createQualifiedObjectName; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isTimestampType; @@ -913,6 +817,7 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope sourceScope = analyzeFrom(node, scope); + resolveFunctionCallAndMeasureWindows(node); node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); @@ -969,6 +874,7 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional analyzeAggregations( node, sourceScope, orderByScope, groupByAnalysis, sourceExpressions, orderByExpressions); + analyzeWindowFunctionsAndMeasures(node, outputExpressions, orderByExpressions); if (analysis.isAggregation(node) && node.getOrderBy().isPresent()) { ImmutableList.Builder aggregates = @@ -991,6 +897,182 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional return outputScope; } + private void analyzeWindowFunctionsAndMeasures( + QuerySpecification node, + List outputExpressions, + List orderByExpressions) { + analysis.setWindowFunctions(node, analyzeWindowFunctions(node, outputExpressions)); + // if (node.getOrderBy().isPresent()) { + // OrderBy orderBy = node.getOrderBy().get(); + // analysis.setOrderByWindowFunctions(orderBy, analyzeWindowFunctions(node, + // orderByExpressions)); + // analysis.setOrderByWindowMeasures(orderBy, + // extractWindowMeasures(orderByExpressions)); + // } + } + + private List analyzeWindowFunctions( + QuerySpecification node, List expressions) { + List windowFunctions = extractWindowFunctions(expressions); + + // for (FunctionCall windowFunction : windowFunctions) { + // List nestedWindowExpressions = + // extractWindowExpressions(windowFunction.getArguments()); + // if (!nestedWindowExpressions.isEmpty()) { + // throw semanticException(NESTED_WINDOW, nestedWindowExpressions.get(0), "Cannot + // nest window functions or row pattern measures inside window function arguments"); + // } + // + // if (windowFunction.isDistinct()) { + // throw semanticException(NOT_SUPPORTED, node, "DISTINCT in window function + // parameters not yet supported: %s", windowFunction); + // } + // + // Analysis.ResolvedWindow window = analysis.getWindow(windowFunction); + // // TODO get function requirements from window function metadata when we have it + // String name = windowFunction.getName().toString().toLowerCase(ENGLISH); + // if (name.equals("lag") || name.equals("lead")) { + // if (!window.getOrderBy().isPresent()) { + // throw semanticException(MISSING_ORDER_BY, (Node) + // windowFunction.getWindow().orElseThrow(), "%s function requires an ORDER BY window clause", + // windowFunction.getName()); + // } + // if (window.getFrame().isPresent()) { + // throw semanticException(INVALID_WINDOW_FRAME, window.getFrame().get(), "Cannot + // specify window frame for %s function", windowFunction.getName()); + // } + // } + // + // if (!WINDOW_VALUE_FUNCTIONS.contains(name) && + // windowFunction.getNullTreatment().isPresent()) { + // throw semanticException(NULL_TREATMENT_NOT_ALLOWED, windowFunction, "Cannot + // specify null treatment clause for %s function", windowFunction.getName()); + // } + // + // List argumentTypes = mappedCopy(windowFunction.getArguments(), + // analysis::getType); + // + // ResolvedFunction resolvedFunction = functionResolver.resolveFunction(session, + // windowFunction.getName(), fromTypes(argumentTypes), accessControl); + // FunctionKind kind = resolvedFunction.getFunctionKind(); + // if (kind != AGGREGATE && kind != WINDOW) { + // throw semanticException(FUNCTION_NOT_WINDOW, node, "Not a window function: %s", + // windowFunction.getName()); + // } + // } + + return windowFunctions; + } + + private void resolveFunctionCallAndMeasureWindows(QuerySpecification querySpecification) { + ImmutableList.Builder expressions = ImmutableList.builder(); + + // SELECT expressions and ORDER BY expressions can contain window functions + for (SelectItem item : querySpecification.getSelect().getSelectItems()) { + if (item instanceof AllColumns) { + ((AllColumns) item).getTarget().ifPresent(expressions::add); + } else if (item instanceof SingleColumn) { + expressions.add(((SingleColumn) item).getExpression()); + } + } + for (SortItem sortItem : getSortItemsFromOrderBy(querySpecification.getOrderBy())) { + expressions.add(sortItem.getSortKey()); + } + + for (FunctionCall windowFunction : extractWindowFunctions(expressions.build())) { + Analysis.ResolvedWindow resolvedWindow = + resolveWindowSpecification(querySpecification, windowFunction.getWindow().get()); + analysis.setWindow(windowFunction, resolvedWindow); + } + } + + private Analysis.ResolvedWindow resolveWindowSpecification( + QuerySpecification querySpecification, WindowSpecification window) { + // if (window instanceof WindowReference windowReference) { + // CanonicalizationAware canonicalName = + // canonicalizationAwareKey(windowReference.getName()); + // ResolvedWindow referencedWindow = analysis.getWindowDefinition(querySpecification, + // canonicalName); + // if (referencedWindow == null) { + // throw semanticException(INVALID_WINDOW_REFERENCE, windowReference.getName(), + // "Cannot resolve WINDOW name %s", windowReference.getName()); + // } + // + // return new ResolvedWindow( + // referencedWindow.getPartitionBy(), + // referencedWindow.getOrderBy(), + // referencedWindow.getFrame(), + // !referencedWindow.getPartitionBy().isEmpty(), + // referencedWindow.getOrderBy().isPresent(), + // referencedWindow.getFrame().isPresent()); + // } + + WindowSpecification windowSpecification = (WindowSpecification) window; + + // if (windowSpecification.getExistingWindowName().isPresent()) { + // Identifier referencedName = windowSpecification.getExistingWindowName().get(); + // CanonicalizationAware canonicalName = + // canonicalizationAwareKey(referencedName); + // Analysis.ResolvedWindow referencedWindow = + // analysis.getWindowDefinition(querySpecification, canonicalName); + // if (referencedWindow == null) { + // throw semanticException(INVALID_WINDOW_REFERENCE, referencedName, "Cannot resolve + // WINDOW name %s", referencedName); + // } + // + // // analyze dependencies between this window specification and referenced window + // specification + // if (!windowSpecification.getPartitionBy().isEmpty()) { + // throw semanticException(INVALID_PARTITION_BY, + // windowSpecification.getPartitionBy().get(0), "WINDOW specification with named WINDOW + // reference cannot specify PARTITION BY"); + // } + // if (windowSpecification.getOrderBy().isPresent() && + // referencedWindow.getOrderBy().isPresent()) { + // throw semanticException(INVALID_ORDER_BY, windowSpecification.getOrderBy().get(), + // "Cannot specify ORDER BY if referenced named WINDOW specifies ORDER BY"); + // } + // if (referencedWindow.getFrame().isPresent()) { + // throw semanticException(INVALID_WINDOW_REFERENCE, + // windowSpecification.getExistingWindowName().get(), "Cannot reference named WINDOW + // containing frame specification"); + // } + // + // // resolve window + // Optional orderBy = windowSpecification.getOrderBy(); + // boolean orderByInherited = false; + // if (!orderBy.isPresent() && referencedWindow.getOrderBy().isPresent()) { + // orderBy = referencedWindow.getOrderBy(); + // orderByInherited = true; + // } + // + // List partitionBy = windowSpecification.getPartitionBy(); + // boolean partitionByInherited = false; + // if (!referencedWindow.getPartitionBy().isEmpty()) { + // partitionBy = referencedWindow.getPartitionBy(); + // partitionByInherited = true; + // } + // + // Optional windowFrame = windowSpecification.getFrame(); + // boolean frameInherited = false; + // if (!windowFrame.isPresent() && referencedWindow.getFrame().isPresent()) { + // windowFrame = referencedWindow.getFrame(); + // frameInherited = true; + // } + // + // return new Analysis.ResolvedWindow(partitionBy, orderBy, windowFrame, + // partitionByInherited, orderByInherited, frameInherited); + // } + + return new Analysis.ResolvedWindow( + windowSpecification.getPartitionBy(), + windowSpecification.getOrderBy(), + windowSpecification.getFrame(), + false, + false, + false); + } + private Scope analyzeFrom(QuerySpecification node, Optional scope) { if (node.getFrom().isPresent()) { return process(node.getFrom().get(), scope); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 747288a48c875..6dabd2b8e1e5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis.GroupingSetAnalysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.FieldId; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GapFillStartAndEndTimeExtractVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation; @@ -40,20 +41,27 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; +import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -85,6 +93,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isNumericType; import static org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingTranslator.sortItemToSortOrder; import static org.apache.iotdb.db.queryengine.plan.relational.planner.PlanBuilder.newPlanBuilder; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ScopeAware.scopeAwareKey; @@ -94,10 +103,19 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.groupingSets; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleAggregation; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.singleGroupingSet; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.GREATER_THAN_OR_EQUAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.GROUPS; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.RANGE; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.ROWS; +import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toSqlType; +import static org.apache.iotdb.db.queryengine.plan.relational.utils.NodeUtils.getSortItemsFromOrderBy; +import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN; public class QueryPlanner { private final Analysis analysis; private final SymbolAllocator symbolAllocator; + private final QueryId idAllocator; private final MPPQueryContext queryContext; private final QueryId queryIdAllocator; private final SessionInfo session; @@ -111,6 +129,7 @@ public class QueryPlanner { public QueryPlanner( Analysis analysis, SymbolAllocator symbolAllocator, + QueryId idAllocator, MPPQueryContext queryContext, Optional outerContext, SessionInfo session, @@ -124,6 +143,7 @@ public QueryPlanner( this.analysis = analysis; this.symbolAllocator = symbolAllocator; + this.idAllocator = idAllocator; this.queryContext = queryContext; this.queryIdAllocator = queryContext.getQueryId(); this.session = session; @@ -182,6 +202,8 @@ public RelationPlan plan(QuerySpecification node) { } builder = aggregate(builder, node); builder = filter(builder, analysis.getHaving(node), node); + builder = + planWindowFunctions(node, builder, ImmutableList.copyOf(analysis.getWindowFunctions(node))); if (gapFillColumn != null) { if (wherePredicate == null) { @@ -275,6 +297,390 @@ public RelationPlan plan(QuerySpecification node) { builder.getRoot(), analysis.getScope(node), computeOutputs(builder, outputs), outerContext); } + private PlanBuilder planWindowFunctions( + Node node, PlanBuilder subPlan, List windowFunctions) { + if (windowFunctions.isEmpty()) { + return subPlan; + } + + Map> functions = + scopeAwareDistinct(subPlan, windowFunctions).stream() + .collect(Collectors.groupingBy(analysis::getWindow)); + + for (Map.Entry> entry : functions.entrySet()) { + Analysis.ResolvedWindow window = entry.getKey(); + List functionCalls = entry.getValue(); + + // Pre-project inputs. + // Predefined window parts (specified in WINDOW clause) can only use source symbols, and no + // output symbols. + // It matters in case when this window planning takes place in ORDER BY clause, where both + // source and output + // symbols are visible. + // This issue is solved by analyzing window definitions in the source scope. After analysis, + // the expressions + // are recorded as belonging to the source scope, and consequentially source symbols will be + // used to plan them. + ImmutableList.Builder inputsBuilder = + ImmutableList.builder() + .addAll(window.getPartitionBy()) + .addAll( + getSortItemsFromOrderBy(window.getOrderBy()).stream() + .map(SortItem::getSortKey) + .iterator()); + + if (window.getFrame().isPresent()) { + WindowFrame frame = window.getFrame().get(); + frame.getStart().getValue().ifPresent(inputsBuilder::add); + + if (frame.getEnd().isPresent()) { + frame.getEnd().get().getValue().ifPresent(inputsBuilder::add); + } + } + + for (FunctionCall windowFunction : functionCalls) { + inputsBuilder.addAll(new ArrayList<>(windowFunction.getArguments())); + } + + List inputs = inputsBuilder.build(); + + subPlan = subqueryPlanner.handleSubqueries(subPlan, inputs, analysis.getSubqueries(node)); + subPlan = subPlan.appendProjections(inputs, symbolAllocator, queryContext); + + // Add projection to coerce inputs to their site-specific types. + // This is important because the same lexical expression may need to be coerced + // in different ways if it's referenced by multiple arguments to the window function. + // For example, given v::integer, + // avg(v) OVER (ORDER BY v) + // Needs to be rewritten as + // avg(CAST(v AS double)) OVER (ORDER BY v) + PlanAndMappings coercions = coerce(subPlan, inputs, analysis, idAllocator, symbolAllocator); + subPlan = coercions.getSubPlan(); + + // For frame of type RANGE, append casts and functions necessary for frame bound calculations + Optional frameStart = Optional.empty(); + Optional frameEnd = Optional.empty(); + Optional sortKeyCoercedForFrameStartComparison = Optional.empty(); + Optional sortKeyCoercedForFrameEndComparison = Optional.empty(); + + if (window.getFrame().isPresent() && window.getFrame().get().getType() == RANGE) { + Optional startValue = window.getFrame().get().getStart().getValue(); + Optional endValue = + window.getFrame().get().getEnd().flatMap(FrameBound::getValue); + // record sortKey coercions for reuse + Map sortKeyCoercions = new HashMap<>(); + + // process frame start + FrameBoundPlanAndSymbols plan = + planFrameBound(subPlan, coercions, startValue, window, sortKeyCoercions); + subPlan = plan.getSubPlan(); + frameStart = plan.getFrameBoundSymbol(); + sortKeyCoercedForFrameStartComparison = plan.getSortKeyCoercedForFrameBoundComparison(); + + // process frame end + plan = planFrameBound(subPlan, coercions, endValue, window, sortKeyCoercions); + subPlan = plan.getSubPlan(); + frameEnd = plan.getFrameBoundSymbol(); + sortKeyCoercedForFrameEndComparison = plan.getSortKeyCoercedForFrameBoundComparison(); + } else if (window.getFrame().isPresent() + && (window.getFrame().get().getType() == ROWS + || window.getFrame().get().getType() == GROUPS)) { + Optional startValue = window.getFrame().get().getStart().getValue(); + Optional endValue = + window.getFrame().get().getEnd().flatMap(FrameBound::getValue); + + // process frame start + FrameOffsetPlanAndSymbol plan = planFrameOffset(subPlan, startValue.map(coercions::get)); + subPlan = plan.getSubPlan(); + frameStart = plan.getFrameOffsetSymbol(); + + // process frame end + plan = planFrameOffset(subPlan, endValue.map(coercions::get)); + subPlan = plan.getSubPlan(); + frameEnd = plan.getFrameOffsetSymbol(); + } else if (window.getFrame().isPresent()) { + throw new IllegalArgumentException( + "unexpected window frame type: " + window.getFrame().get().getType()); + } + + subPlan = + planWindow( + subPlan, + functionCalls, + window, + coercions, + frameStart, + sortKeyCoercedForFrameStartComparison, + frameEnd, + sortKeyCoercedForFrameEndComparison); + } + + return subPlan; + } + + private PlanBuilder planWindow( + PlanBuilder subPlan, + List windowFunctions, + Analysis.ResolvedWindow window, + PlanAndMappings coercions, + Optional frameStartSymbol, + Optional sortKeyCoercedForFrameStartComparison, + Optional frameEndSymbol, + Optional sortKeyCoercedForFrameEndComparison) { + WindowFrame.Type frameType = WindowFrame.Type.RANGE; + FrameBound.Type frameStartType = FrameBound.Type.UNBOUNDED_PRECEDING; + FrameBound.Type frameEndType = FrameBound.Type.CURRENT_ROW; + + Optional frameStartExpression = Optional.empty(); + Optional frameEndExpression = Optional.empty(); + + if (window.getFrame().isPresent()) { + WindowFrame frame = window.getFrame().get(); + frameType = frame.getType(); + + frameStartType = frame.getStart().getType(); + frameStartExpression = frame.getStart().getValue(); + + if (frame.getEnd().isPresent()) { + frameEndType = frame.getEnd().get().getType(); + frameEndExpression = frame.getEnd().get().getValue(); + } + } + + DataOrganizationSpecification specification = + planWindowSpecification(window.getPartitionBy(), window.getOrderBy(), coercions::get); + + // Rewrite frame bounds in terms of pre-projected inputs + WindowNode.Frame frame = + new WindowNode.Frame( + frameType, + frameStartType, + frameStartSymbol, + sortKeyCoercedForFrameStartComparison, + frameEndType, + frameEndSymbol, + sortKeyCoercedForFrameEndComparison, + frameStartExpression, + frameEndExpression); + + ImmutableMap.Builder, Symbol> mappings = ImmutableMap.builder(); + ImmutableMap.Builder functions = ImmutableMap.builder(); + + for (FunctionCall windowFunction : windowFunctions) { + Symbol newSymbol = + symbolAllocator.newSymbol(windowFunction, analysis.getType(windowFunction)); + + WindowNode.Function function = + new WindowNode.Function( + analysis.getResolvedFunction(windowFunction), + windowFunction.getArguments().stream() + .map(argument -> coercions.get(argument).toSymbolReference()) + .collect(toImmutableList()), + frame, + // TODO: remove ignore null + false); + + functions.put(newSymbol, function); + mappings.put(scopeAwareKey(windowFunction, analysis, subPlan.getScope()), newSymbol); + } + + // create window node + return new PlanBuilder( + subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), + new WindowNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + specification, + functions.buildOrThrow(), + Optional.empty(), + ImmutableSet.of(), + 0)); + } + + public static DataOrganizationSpecification planWindowSpecification( + List partitionBy, + Optional orderBy, + Function expressionRewrite) { + // Rewrite PARTITION BY + ImmutableList.Builder partitionBySymbols = ImmutableList.builder(); + for (Expression expression : partitionBy) { + partitionBySymbols.add(expressionRewrite.apply(expression)); + } + + // Rewrite ORDER BY + LinkedHashMap orderings = new LinkedHashMap<>(); + for (SortItem item : getSortItemsFromOrderBy(orderBy)) { + Symbol symbol = expressionRewrite.apply(item.getSortKey()); + // don't override existing keys, i.e. when "ORDER BY a ASC, a DESC" is specified + orderings.putIfAbsent(symbol, sortItemToSortOrder(item)); + } + + Optional orderingScheme = Optional.empty(); + if (!orderings.isEmpty()) { + orderingScheme = + Optional.of(new OrderingScheme(ImmutableList.copyOf(orderings.keySet()), orderings)); + } + + return new DataOrganizationSpecification(partitionBySymbols.build(), orderingScheme); + } + + private FrameBoundPlanAndSymbols planFrameBound( + PlanBuilder subPlan, + PlanAndMappings coercions, + Optional frameOffset, + Analysis.ResolvedWindow window, + Map sortKeyCoercions) { + Optional frameBoundCalculationFunction = + frameOffset.map(analysis::getFrameBoundCalculation); + + // Empty frameBoundCalculationFunction indicates that frame bound type is CURRENT ROW or + // UNBOUNDED. + // Handling it doesn't require any additional symbols. + if (!frameBoundCalculationFunction.isPresent()) { + return new FrameBoundPlanAndSymbols(subPlan, Optional.empty(), Optional.empty()); + } + + // Present frameBoundCalculationFunction indicates that frame bound type is + // PRECEDING or FOLLOWING. + // It requires adding certain projections to the plan so that the operator can determine frame + // bounds. + + // First, append filter to validate offset values. They mustn't be negative or null. + Symbol offsetSymbol = coercions.get(frameOffset.get()); + Expression zeroOffset = zeroOfType(symbolAllocator.getTypes().getTableModelType(offsetSymbol)); + Expression predicate = + new IfExpression( + new ComparisonExpression( + GREATER_THAN_OR_EQUAL, offsetSymbol.toSymbolReference(), zeroOffset), + TRUE_LITERAL, + new Cast(new NullLiteral(), toSqlType(BOOLEAN))); + subPlan = + subPlan.withNewRoot( + new FilterNode(idAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); + + // Then, coerce the sortKey so that we can add / subtract the offset. + // Note: for that we cannot rely on the usual mechanism of using the coerce() method. The + // coerce() method can only handle one coercion for a node, + // while the sortKey node might require several different coercions, e.g. one for frame start + // and one for frame end. + Expression sortKey = + Iterables.getOnlyElement(window.getOrderBy().get().getSortItems()).getSortKey(); + Symbol sortKeyCoercedForFrameBoundCalculation = coercions.get(sortKey); + Optional coercion = frameOffset.map(analysis::getSortKeyCoercionForFrameBoundCalculation); + if (coercion.isPresent()) { + Type expectedType = coercion.get(); + Symbol alreadyCoerced = sortKeyCoercions.get(expectedType); + if (alreadyCoerced != null) { + sortKeyCoercedForFrameBoundCalculation = alreadyCoerced; + } else { + Expression cast = + new Cast( + coercions.get(sortKey).toSymbolReference(), + toSqlType(expectedType), + false, + analysis.getType(sortKey).equals(expectedType)); + sortKeyCoercedForFrameBoundCalculation = symbolAllocator.newSymbol(cast, expectedType); + sortKeyCoercions.put(expectedType, sortKeyCoercedForFrameBoundCalculation); + subPlan = + subPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + Assignments.builder() + .putIdentities(subPlan.getRoot().getOutputSymbols()) + .put(sortKeyCoercedForFrameBoundCalculation, cast) + .build())); + } + } + + // Next, pre-project the function which combines sortKey with the offset. + // Note: if frameOffset needs a coercion, it was added before by a call to coerce() method. + ResolvedFunction function = frameBoundCalculationFunction.get(); + Expression functionCall = + new FunctionCall( + function.toQualifiedName(), + ImmutableList.of( + sortKeyCoercedForFrameBoundCalculation.toSymbolReference(), + offsetSymbol.toSymbolReference())); + Symbol frameBoundSymbol = + symbolAllocator.newSymbol(functionCall, function.getSignature().getReturnType()); + subPlan = + subPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + Assignments.builder() + .putIdentities(subPlan.getRoot().getOutputSymbols()) + .put(frameBoundSymbol, functionCall) + .build())); + + // Finally, coerce the sortKey to the type of frameBound so that the operator can perform + // comparisons on them + Optional sortKeyCoercedForFrameBoundComparison = Optional.of(coercions.get(sortKey)); + coercion = frameOffset.map(analysis::getSortKeyCoercionForFrameBoundComparison); + if (coercion.isPresent()) { + Type expectedType = coercion.get(); + Symbol alreadyCoerced = sortKeyCoercions.get(expectedType); + if (alreadyCoerced != null) { + sortKeyCoercedForFrameBoundComparison = Optional.of(alreadyCoerced); + } else { + Expression cast = + new Cast( + coercions.get(sortKey).toSymbolReference(), + toSqlType(expectedType), + false, + // TODO: type coercion + analysis.getType(sortKey).equals(expectedType)); + Symbol castSymbol = symbolAllocator.newSymbol(cast, expectedType); + sortKeyCoercions.put(expectedType, castSymbol); + subPlan = + subPlan.withNewRoot( + new ProjectNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + Assignments.builder() + .putIdentities(subPlan.getRoot().getOutputSymbols()) + .put(castSymbol, cast) + .build())); + sortKeyCoercedForFrameBoundComparison = Optional.of(castSymbol); + } + } + + return new FrameBoundPlanAndSymbols( + subPlan, Optional.of(frameBoundSymbol), sortKeyCoercedForFrameBoundComparison); + } + + private FrameOffsetPlanAndSymbol planFrameOffset( + PlanBuilder subPlan, Optional frameOffset) { + if (!frameOffset.isPresent()) { + return new FrameOffsetPlanAndSymbol(subPlan, Optional.empty()); + } + + Symbol offsetSymbol = frameOffset.get(); + Type offsetType = symbolAllocator.getTypes().getTableModelType(offsetSymbol); + + // Append filter to validate offset values. They mustn't be negative or null. + Expression zeroOffset = zeroOfType(offsetType); + Expression predicate = + new IfExpression( + new ComparisonExpression( + GREATER_THAN_OR_EQUAL, offsetSymbol.toSymbolReference(), zeroOffset), + TRUE_LITERAL, + new Cast(new NullLiteral(), toSqlType(BOOLEAN))); + subPlan = + subPlan.withNewRoot( + new FilterNode(idAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); + + return new FrameOffsetPlanAndSymbol(subPlan, Optional.of(offsetSymbol)); + } + + private static Expression zeroOfType(Type type) { + if (isNumericType(type)) { + return new Cast(new LongLiteral("0"), toSqlType(type)); + } + throw new IllegalArgumentException("unexpected type: " + type); + } + private static boolean hasExpressionsToUnfold(List selectExpressions) { return selectExpressions.stream() .map(Analysis.SelectExpression::getUnfoldedExpressions) @@ -974,4 +1380,49 @@ public Aggregation getRewritten() { return aggregation; } } + + private static class FrameBoundPlanAndSymbols { + private final PlanBuilder subPlan; + private final Optional frameBoundSymbol; + private final Optional sortKeyCoercedForFrameBoundComparison; + + public FrameBoundPlanAndSymbols( + PlanBuilder subPlan, + Optional frameBoundSymbol, + Optional sortKeyCoercedForFrameBoundComparison) { + this.subPlan = subPlan; + this.frameBoundSymbol = frameBoundSymbol; + this.sortKeyCoercedForFrameBoundComparison = sortKeyCoercedForFrameBoundComparison; + } + + public PlanBuilder getSubPlan() { + return subPlan; + } + + public Optional getFrameBoundSymbol() { + return frameBoundSymbol; + } + + public Optional getSortKeyCoercedForFrameBoundComparison() { + return sortKeyCoercedForFrameBoundComparison; + } + } + + private static class FrameOffsetPlanAndSymbol { + private final PlanBuilder subPlan; + private final Optional frameOffsetSymbol; + + public FrameOffsetPlanAndSymbol(PlanBuilder subPlan, Optional frameOffsetSymbol) { + this.subPlan = subPlan; + this.frameOffsetSymbol = frameOffsetSymbol; + } + + public PlanBuilder getSubPlan() { + return subPlan; + } + + public Optional getFrameOffsetSymbol() { + return frameOffsetSymbol; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index aa346416bbaab..a8a971f7669f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -153,7 +153,13 @@ public RelationPlanner( @Override protected RelationPlan visitQuery(final Query node, final Void context) { return new QueryPlanner( - analysis, symbolAllocator, queryContext, outerContext, sessionInfo, recursiveSubqueries) + analysis, + symbolAllocator, + idAllocator, + queryContext, + outerContext, + sessionInfo, + recursiveSubqueries) .plan(node); } @@ -245,7 +251,13 @@ protected RelationPlan visitTable(final Table table, final Void context) { protected RelationPlan visitQuerySpecification( final QuerySpecification node, final Void context) { return new QueryPlanner( - analysis, symbolAllocator, queryContext, outerContext, sessionInfo, recursiveSubqueries) + analysis, + symbolAllocator, + idAllocator, + queryContext, + outerContext, + sessionInfo, + recursiveSubqueries) .plan(node); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index fb479eb447484..939e7c47f650c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.AbstractTableDeviceQueryNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceQueryCountNode; @@ -1241,6 +1242,23 @@ public List visitTableDeviceFetch( } } + @Override + public List visitWindowFunction(WindowNode node, PlanContext context) { + List childrenNodes = node.getChild().accept(this, context); + OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); + Optional ordering = node.getSpecification().getOrderingScheme(); + if (childOrdering != null) { + nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + } + + // TODO: multi-child + if (childrenNodes.size() == 1) { + node.setChild(childrenNodes.get(0)); + } + + return Collections.singletonList(node); + } + public static class PlanContext { final Map nodeDistributionMap; boolean hasExchangeNode = false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java new file mode 100644 index 0000000000000..b2a1f7a82d3d8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java @@ -0,0 +1,340 @@ +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; +import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.errorprone.annotations.Immutable; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Iterables.concat; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.CURRENT_ROW; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound.Type.UNBOUNDED_PRECEDING; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame.Type.RANGE; + +public class WindowNode extends SingleChildProcessNode { + private final Set prePartitionedInputs; + private final DataOrganizationSpecification specification; + private final int preSortedOrderPrefix; + private final Map windowFunctions; + private final Optional hashSymbol; + + public WindowNode( + PlanNodeId id, + PlanNode child, + DataOrganizationSpecification specification, + Map windowFunctions, + Optional hashSymbol, + Set prePartitionedInputs, + int preSortedOrderPrefix) { + super(id, child); + // Make the defensive copy eagerly, so it can be used for both the validation checks and + // assigned directly to the field afterwards + prePartitionedInputs = ImmutableSet.copyOf(prePartitionedInputs); + + ImmutableSet partitionBy = ImmutableSet.copyOf(specification.getPartitionBy()); + Optional orderingScheme = specification.getOrderingScheme(); + checkArgument( + partitionBy.containsAll(prePartitionedInputs), + "prePartitionedInputs must be contained in partitionBy"); + checkArgument( + preSortedOrderPrefix == 0 + || (orderingScheme.isPresent() + && preSortedOrderPrefix <= orderingScheme.get().getOrderBy().size()), + "Cannot have sorted more symbols than those requested"); + checkArgument( + preSortedOrderPrefix == 0 || partitionBy.equals(prePartitionedInputs), + "preSortedOrderPrefix can only be greater than zero if all partition symbols are pre-partitioned"); + + this.prePartitionedInputs = prePartitionedInputs; + this.specification = specification; + this.windowFunctions = ImmutableMap.copyOf(windowFunctions); + this.hashSymbol = hashSymbol; + this.preSortedOrderPrefix = preSortedOrderPrefix; + } + + @Override + public PlanNode clone() { + return new WindowNode( + id, + child, + specification, + windowFunctions, + hashSymbol, + prePartitionedInputs, + preSortedOrderPrefix); + } + + @Override + public List getOutputColumnNames() { + return child.getOutputColumnNames(); + } + + @Override + public List getOutputSymbols() { + return ImmutableList.copyOf(concat(child.getOutputSymbols(), windowFunctions.keySet())); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + return new WindowNode( + id, + Iterables.getOnlyElement(newChildren), + specification, + windowFunctions, + hashSymbol, + prePartitionedInputs, + preSortedOrderPrefix); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitWindowFunction(this, context); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) {} + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException {} + + public Set getPrePartitionedInputs() { + return prePartitionedInputs; + } + + public DataOrganizationSpecification getSpecification() { + return specification; + } + + public int getPreSortedOrderPrefix() { + return preSortedOrderPrefix; + } + + public Map getWindowFunctions() { + return windowFunctions; + } + + public Optional getHashSymbol() { + return hashSymbol; + } + + @Immutable + public static class Frame { + public static final Frame DEFAULT_FRAME = + new WindowNode.Frame( + RANGE, + UNBOUNDED_PRECEDING, + Optional.empty(), + Optional.empty(), + CURRENT_ROW, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + private final WindowFrame.Type type; + private final FrameBound.Type startType; + private final Optional startValue; + private final Optional sortKeyCoercedForFrameStartComparison; + private final FrameBound.Type endType; + private final Optional endValue; + private final Optional sortKeyCoercedForFrameEndComparison; + + // This information is only used for printing the plan. + private final Optional originalStartValue; + private final Optional originalEndValue; + + public Frame( + WindowFrame.Type type, + FrameBound.Type startType, + Optional startValue, + Optional sortKeyCoercedForFrameStartComparison, + FrameBound.Type endType, + Optional endValue, + Optional sortKeyCoercedForFrameEndComparison, + Optional originalStartValue, + Optional originalEndValue) { + this.startType = requireNonNull(startType, "startType is null"); + this.startValue = requireNonNull(startValue, "startValue is null"); + this.sortKeyCoercedForFrameStartComparison = + requireNonNull( + sortKeyCoercedForFrameStartComparison, + "sortKeyCoercedForFrameStartComparison is null"); + this.endType = requireNonNull(endType, "endType is null"); + this.endValue = requireNonNull(endValue, "endValue is null"); + this.sortKeyCoercedForFrameEndComparison = + requireNonNull( + sortKeyCoercedForFrameEndComparison, "sortKeyCoercedForFrameEndComparison is null"); + this.type = requireNonNull(type, "type is null"); + this.originalStartValue = requireNonNull(originalStartValue, "originalStartValue is null"); + this.originalEndValue = requireNonNull(originalEndValue, "originalEndValue is null"); + + if (startValue.isPresent()) { + checkArgument( + originalStartValue.isPresent(), + "originalStartValue must be present if startValue is present"); + if (type == RANGE) { + checkArgument( + sortKeyCoercedForFrameStartComparison.isPresent(), + "for frame of type RANGE, sortKeyCoercedForFrameStartComparison must be present if startValue is present"); + } + } + + if (endValue.isPresent()) { + checkArgument( + originalEndValue.isPresent(), + "originalEndValue must be present if endValue is present"); + if (type == RANGE) { + checkArgument( + sortKeyCoercedForFrameEndComparison.isPresent(), + "for frame of type RANGE, sortKeyCoercedForFrameEndComparison must be present if endValue is present"); + } + } + } + + public WindowFrame.Type getType() { + return type; + } + + public FrameBound.Type getStartType() { + return startType; + } + + public Optional getStartValue() { + return startValue; + } + + public Optional getSortKeyCoercedForFrameStartComparison() { + return sortKeyCoercedForFrameStartComparison; + } + + public FrameBound.Type getEndType() { + return endType; + } + + public Optional getEndValue() { + return endValue; + } + + public Optional getSortKeyCoercedForFrameEndComparison() { + return sortKeyCoercedForFrameEndComparison; + } + + public Optional getOriginalStartValue() { + return originalStartValue; + } + + public Optional getOriginalEndValue() { + return originalEndValue; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Frame frame = (Frame) o; + return type == frame.type + && startType == frame.startType + && Objects.equals(startValue, frame.startValue) + && Objects.equals( + sortKeyCoercedForFrameStartComparison, frame.sortKeyCoercedForFrameStartComparison) + && endType == frame.endType + && Objects.equals(endValue, frame.endValue) + && Objects.equals( + sortKeyCoercedForFrameEndComparison, frame.sortKeyCoercedForFrameEndComparison); + } + + @Override + public int hashCode() { + return Objects.hash( + type, + startType, + startValue, + sortKeyCoercedForFrameStartComparison, + endType, + endValue, + sortKeyCoercedForFrameEndComparison); + } + } + + @Immutable + public static final class Function { + private final ResolvedFunction resolvedFunction; + private final List arguments; + private final Frame frame; + private final boolean ignoreNulls; + + public Function( + ResolvedFunction resolvedFunction, + List arguments, + Frame frame, + boolean ignoreNulls) { + this.resolvedFunction = requireNonNull(resolvedFunction, "resolvedFunction is null"); + this.arguments = requireNonNull(arguments, "arguments is null"); + this.frame = requireNonNull(frame, "frame is null"); + this.ignoreNulls = ignoreNulls; + } + + public ResolvedFunction getResolvedFunction() { + return resolvedFunction; + } + + public Frame getFrame() { + return frame; + } + + public List getArguments() { + return arguments; + } + + @Override + public int hashCode() { + return Objects.hash(resolvedFunction, arguments, frame, ignoreNulls); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Function other = (Function) obj; + return Objects.equals(this.resolvedFunction, other.resolvedFunction) + && Objects.equals(this.arguments, other.arguments) + && Objects.equals(this.frame, other.frame) + && this.ignoreNulls == other.ignoreNulls; + } + + public boolean isIgnoreNulls() { + return ignoreNulls; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java index 66e34f2693852..b360562d980ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -46,6 +47,7 @@ import java.util.stream.Collectors; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.Objects.requireNonNull; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.groupingSets; @@ -228,6 +230,47 @@ public OrderingScheme map(OrderingScheme orderingScheme) { return new OrderingScheme(newSymbols.build(), newOrderings.buildOrThrow()); } + public WindowNode map(WindowNode node, PlanNode source) { + ImmutableMap.Builder newFunctions = ImmutableMap.builder(); + node.getWindowFunctions() + .forEach( + (symbol, function) -> { + List newArguments = + function.getArguments().stream().map(this::map).collect(toImmutableList()); + WindowNode.Frame newFrame = map(function.getFrame()); + + newFunctions.put( + map(symbol), + new WindowNode.Function( + function.getResolvedFunction(), + newArguments, + newFrame, + function.isIgnoreNulls())); + }); + + return new WindowNode( + node.getPlanNodeId(), + source, + node.getSpecification(), + newFunctions.buildOrThrow(), + node.getHashSymbol().map(this::map), + node.getPrePartitionedInputs().stream().map(this::map).collect(toImmutableSet()), + node.getPreSortedOrderPrefix()); + } + + private WindowNode.Frame map(WindowNode.Frame frame) { + return new WindowNode.Frame( + frame.getType(), + frame.getStartType(), + frame.getStartValue().map(this::map), + frame.getSortKeyCoercedForFrameStartComparison().map(this::map), + frame.getEndType(), + frame.getEndValue().map(this::map), + frame.getSortKeyCoercedForFrameEndComparison().map(this::map), + frame.getOriginalStartValue(), + frame.getOriginalEndValue()); + } + public TopKNode map(TopKNode node, List source) { return map(node, source, node.getPlanNodeId()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 092a53b9d1a19..29c1b966a3f2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -51,6 +51,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -547,6 +548,17 @@ else if (DeterminismEvaluator.isDeterministic(expression) return newMapping; } + @Override + public PlanAndMappings visitWindowFunction(WindowNode node, UnaliasContext context) { + PlanAndMappings rewrittenSource = node.getChild().accept(this, context); + Map mapping = new HashMap<>(rewrittenSource.getMappings()); + SymbolMapper mapper = symbolMapper(mapping); + + WindowNode rewrittenWindow = mapper.map(node, rewrittenSource.getRoot()); + + return new PlanAndMappings(rewrittenWindow, mapping); + } + @Override public PlanAndMappings visitOutput(OutputNode node, UnaliasContext context) { PlanAndMappings rewrittenSource = node.getChild().accept(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index dbc979a4be1f2..163e2d20829d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -225,6 +225,18 @@ protected R visitNotExpression(NotExpression node, C context) { return visitExpression(node, context); } + protected R visitWindowSpecification(WindowSpecification node, C context) { + return visitNode(node, context); + } + + protected R visitWindowFrame(WindowFrame node, C context) { + return visitNode(node, context); + } + + protected R visitFrameBound(FrameBound node, C context) { + return visitNode(node, context); + } + protected R visitSelectItem(SelectItem node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java new file mode 100644 index 0000000000000..e78dc100447a7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -0,0 +1,85 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class FrameBound extends Node { + public enum Type { + UNBOUNDED_PRECEDING, + PRECEDING, + CURRENT_ROW, + FOLLOWING, + UNBOUNDED_FOLLOWING + } + + private final Type type; + private final Optional value; + + public FrameBound(NodeLocation location, Type type) { + this(location, type, null); + } + + public FrameBound(NodeLocation location, Type type, Expression value) { + super(location); + this.type = requireNonNull(type, "type is null"); + this.value = Optional.ofNullable(value); + } + + public Type getType() { + return type; + } + + public Optional getValue() { + return value; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitFrameBound(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder nodes = ImmutableList.builder(); + value.ifPresent(nodes::add); + return nodes.build(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + FrameBound o = (FrameBound) obj; + return type == o.type && Objects.equals(value, o.value); + } + + @Override + public int hashCode() { + return Objects.hash(type, value); + } + + @Override + public String toString() { + return toStringHelper(this).add("type", type).add("value", value).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + FrameBound otherNode = (FrameBound) other; + return type == otherNode.type; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 004e426ba2013..c29eca11929b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -28,17 +28,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import static java.util.Objects.requireNonNull; public class FunctionCall extends Expression { private final QualifiedName name; + private final Optional window; private final boolean distinct; private final List arguments; public FunctionCall(QualifiedName name, List arguments) { super(null); this.name = requireNonNull(name, "name is null"); + this.window = Optional.empty(); this.distinct = false; this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -46,6 +49,7 @@ public FunctionCall(QualifiedName name, List arguments) { public FunctionCall(QualifiedName name, boolean distinct, List arguments) { super(null); this.name = requireNonNull(name, "name is null"); + this.window = Optional.empty(); this.distinct = distinct; this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -58,10 +62,25 @@ public FunctionCall( NodeLocation location, QualifiedName name, boolean distinct, List arguments) { super(requireNonNull(location, "location is null")); this.name = requireNonNull(name, "name is null"); + this.window = Optional.empty(); this.distinct = distinct; this.arguments = requireNonNull(arguments, "arguments is null"); } + public FunctionCall( + NodeLocation location, + QualifiedName name, + Optional window, + boolean distinct, + List arguments) { + super(requireNonNull(location, "location is null")); + + this.name = name; + this.window = window; + this.distinct = distinct; + this.arguments = arguments; + } + public QualifiedName getName() { return name; } @@ -74,6 +93,10 @@ public List getArguments() { return arguments; } + public Optional getWindow() { + return window; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitFunctionCall(this, context); @@ -141,5 +164,8 @@ public FunctionCall(ByteBuffer byteBuffer) { while (size-- > 0) { arguments.add(Expression.deserialize(byteBuffer)); } + + // TODO: serialize window + this.window = Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java new file mode 100644 index 0000000000000..7769849fb6473 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java @@ -0,0 +1,86 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class WindowFrame extends Node { + public enum Type { + RANGE, + ROWS, + GROUPS + } + + private final Type type; + private final FrameBound start; + private final Optional end; + + public WindowFrame(NodeLocation location, Type type, FrameBound start, Optional end) { + super(requireNonNull(location)); + this.type = requireNonNull(type, "type is null"); + this.start = requireNonNull(start, "start is null"); + this.end = requireNonNull(end, "end is null"); + } + + public Type getType() { + return type; + } + + public FrameBound getStart() { + return start; + } + + public Optional getEnd() { + return end; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitWindowFrame(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder nodes = ImmutableList.builder(); + nodes.add(start); + end.ifPresent(nodes::add); + return nodes.build(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + WindowFrame o = (WindowFrame) obj; + return type == o.type && Objects.equals(start, o.start) && Objects.equals(end, o.end); + } + + @Override + public int hashCode() { + return Objects.hash(type, start, end); + } + + @Override + public String toString() { + return toStringHelper(this).add("type", type).add("start", start).add("end", end).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + WindowFrame otherNode = (WindowFrame) other; + return type == otherNode.type; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java new file mode 100644 index 0000000000000..9cf78095d7a05 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java @@ -0,0 +1,96 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class WindowSpecification extends Node { + private final Optional existingWindowName; + private final List partitionBy; + private final Optional orderBy; + private final Optional frame; + + public WindowSpecification( + NodeLocation location, + Optional existingWindowName, + List partitionBy, + Optional orderBy, + Optional frame) { + super(requireNonNull(location)); + this.existingWindowName = requireNonNull(existingWindowName, "existingWindowName is null"); + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null"); + this.orderBy = requireNonNull(orderBy, "orderBy is null"); + this.frame = requireNonNull(frame, "frame is null"); + } + + public Optional getExistingWindowName() { + return existingWindowName; + } + + public List getPartitionBy() { + return partitionBy; + } + + public Optional getOrderBy() { + return orderBy; + } + + public Optional getFrame() { + return frame; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitWindowSpecification(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder nodes = ImmutableList.builder(); + existingWindowName.ifPresent(nodes::add); + nodes.addAll(partitionBy); + orderBy.ifPresent(nodes::add); + frame.ifPresent(nodes::add); + return nodes.build(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + WindowSpecification o = (WindowSpecification) obj; + return Objects.equals(existingWindowName, o.existingWindowName) + && Objects.equals(partitionBy, o.partitionBy) + && Objects.equals(orderBy, o.orderBy) + && Objects.equals(frame, o.frame); + } + + @Override + public int hashCode() { + return Objects.hash(existingWindowName, partitionBy, orderBy, frame); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("existingWindowName", existingWindowName) + .add("partitionBy", partitionBy) + .add("orderBy", orderBy) + .add("frame", frame) + .toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index bdf32b0b9d784..bda8d0e92d4b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -35,161 +35,7 @@ import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BinaryLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountStatement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTopic; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentTime; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentUser; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataTypeParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTopic; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingElement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertRows; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNotNullPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNullPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NodeLocation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NumericParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveConfigNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentDatabase; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentSqlDialect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentTimestamp; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentUser; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowStatement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowSubscriptions; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVariables; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartRepairData; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopRepairData; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableExpressionType; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Union; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Update; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UpdateAssignment; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.*; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; @@ -2363,6 +2209,90 @@ public Node visitConcatenation(RelationalSqlParser.ConcatenationContext ctx) { } // ********************* primary expressions ********************** + // @Override + // public Node visitSimpleOver(RelationalSqlParser.SimpleOverContext ctx) { + // return visitWindowSpecification(ctx.over().windowSpecification()); + // } + + @Override + public Node visitOver(RelationalSqlParser.OverContext ctx) { + // TODO: Window Reference + return visit(ctx.windowSpecification()); + } + + @Override + public Node visitWindowSpecification(RelationalSqlParser.WindowSpecificationContext ctx) { + Optional existingWindowName = getIdentifierIfPresent(ctx.existingWindowName); + + List partitionBy = visit(ctx.partition, Expression.class); + + Optional orderBy = Optional.empty(); + if (ctx.ORDER() != null) { + orderBy = + Optional.of(new OrderBy(getLocation(ctx.ORDER()), visit(ctx.sortItem(), SortItem.class))); + } + + Optional frame = Optional.empty(); + if (ctx.windowFrame() != null) { + frame = Optional.of((WindowFrame) visitFrameExtent(ctx.windowFrame().frameExtent())); + } + + return new WindowSpecification( + getLocation(ctx), existingWindowName, partitionBy, orderBy, frame); + } + + @Override + public Node visitFrameExtent(RelationalSqlParser.FrameExtentContext ctx) { + WindowFrame.Type frameType = toWindowFrameType(ctx.frameType); + FrameBound start = (FrameBound) visit(ctx.start); + Optional end = visitIfPresent(ctx.end, FrameBound.class); + return new WindowFrame(getLocation(ctx), frameType, start, end); + } + + private static WindowFrame.Type toWindowFrameType(Token token) { + switch (token.getType()) { + case RelationalSqlLexer.ROWS: + return WindowFrame.Type.ROWS; + case RelationalSqlLexer.RANGE: + return WindowFrame.Type.RANGE; + case RelationalSqlLexer.GROUPS: + return WindowFrame.Type.GROUPS; + default: + throw new IllegalArgumentException("Unsupported window frame type: " + token.getText()); + } + } + + @Override + public Node visitUnboundedFrame(RelationalSqlParser.UnboundedFrameContext ctx) { + switch (ctx.boundType.getType()) { + case RelationalSqlLexer.PRECEDING: + return new FrameBound(getLocation(ctx), FrameBound.Type.UNBOUNDED_PRECEDING); + case RelationalSqlLexer.FOLLOWING: + return new FrameBound(getLocation(ctx), FrameBound.Type.UNBOUNDED_FOLLOWING); + default: + throw new IllegalArgumentException( + "Unsupported unbounded type: " + ctx.boundType.getText()); + } + } + + @Override + public Node visitCurrentRowBound(RelationalSqlParser.CurrentRowBoundContext ctx) { + return new FrameBound(getLocation(ctx), FrameBound.Type.CURRENT_ROW); + } + + @Override + public Node visitBoundedFrame(RelationalSqlParser.BoundedFrameContext ctx) { + Expression value = (Expression) visit(ctx.expression()); + switch (ctx.boundType.getType()) { + case RelationalSqlLexer.PRECEDING: + return new FrameBound(getLocation(ctx), FrameBound.Type.PRECEDING, value); + case RelationalSqlLexer.FOLLOWING: + return new FrameBound(getLocation(ctx), FrameBound.Type.FOLLOWING, value); + default: + throw new IllegalArgumentException("Unsupported bounded type: " + ctx.boundType.getText()); + } + } + @Override public Node visitParenthesizedExpression(RelationalSqlParser.ParenthesizedExpressionContext ctx) { return visit(ctx.expression()); @@ -2518,6 +2448,7 @@ public Node visitWhenClause(RelationalSqlParser.WhenClauseContext ctx) { @Override public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { + Optional window = visitIfPresent(ctx.over(), WindowSpecification.class); QualifiedName name = getQualifiedName(ctx.qualifiedName()); @@ -2603,7 +2534,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { } } - return new FunctionCall(getLocation(ctx), name, distinct, arguments); + return new FunctionCall(getLocation(ctx), name, window, distinct, arguments); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index d6222564ec7ef..b4efdb36a88b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -19,67 +19,14 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.util; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BinaryLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Cast; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentTime; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentUser; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DecimalLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericDataType; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingElement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNotNullPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IsNullPredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NotExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NumericParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TypeParameter; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.*; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -311,6 +258,10 @@ protected String visitFunctionCall(FunctionCall node, Void context) { builder.append(')'); + if (node.getWindow().isPresent()) { + builder.append(" OVER ").append(formatWindow(node.getWindow().get())); + } + return builder.toString(); } @@ -606,6 +557,72 @@ public static String formatSortItems(List sortItems) { return sortItems.stream().map(sortItemFormatterFunction()).collect(joining(", ")); } + private static String formatWindow(WindowSpecification window) { + // if (window instanceof WindowReference) { + // return formatExpression(((WindowReference) window).getName()); + // } + + return formatWindowSpecification((WindowSpecification) window); + } + + static String formatWindowSpecification(WindowSpecification windowSpecification) { + List parts = new ArrayList<>(); + + if (windowSpecification.getExistingWindowName().isPresent()) { + parts.add(formatExpression(windowSpecification.getExistingWindowName().get())); + } + if (!windowSpecification.getPartitionBy().isEmpty()) { + parts.add( + "PARTITION BY " + + windowSpecification.getPartitionBy().stream() + .map(ExpressionFormatter::formatExpression) + .collect(joining(", "))); + } + if (windowSpecification.getOrderBy().isPresent()) { + parts.add(formatOrderBy(windowSpecification.getOrderBy().get())); + } + if (windowSpecification.getFrame().isPresent()) { + parts.add(formatFrame(windowSpecification.getFrame().get())); + } + + return '(' + Joiner.on(' ').join(parts) + ')'; + } + + private static String formatFrame(WindowFrame windowFrame) { + StringBuilder builder = new StringBuilder(); + + builder.append(windowFrame.getType().toString()).append(' '); + + if (windowFrame.getEnd().isPresent()) { + builder + .append("BETWEEN ") + .append(formatFrameBound(windowFrame.getStart())) + .append(" AND ") + .append(formatFrameBound(windowFrame.getEnd().get())); + } else { + builder.append(formatFrameBound(windowFrame.getStart())); + } + + return builder.toString(); + } + + private static String formatFrameBound(FrameBound frameBound) { + switch (frameBound.getType()) { + case UNBOUNDED_PRECEDING: + return "UNBOUNDED PRECEDING"; + case PRECEDING: + return formatExpression(frameBound.getValue().get()) + " PRECEDING"; + case CURRENT_ROW: + return "CURRENT ROW"; + case FOLLOWING: + return formatExpression(frameBound.getValue().get()) + " FOLLOWING"; + case UNBOUNDED_FOLLOWING: + return "UNBOUNDED FOLLOWING"; + default: + throw new IllegalArgumentException("Unsupported frame type: " + frameBound.getType()); + } + } + static String formatGroupBy(List groupingElements) { return groupingElements.stream() .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java new file mode 100644 index 0000000000000..648be3e520cdc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java @@ -0,0 +1,55 @@ +package org.apache.iotdb.db.queryengine.plan.relational.utils; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class DataOrganizationSpecification { + private final List partitionBy; + private final Optional orderingScheme; + + public DataOrganizationSpecification( + List partitionBy, Optional orderingScheme) { + requireNonNull(partitionBy, "partitionBy is null"); + requireNonNull(orderingScheme, "orderingScheme is null"); + + this.partitionBy = ImmutableList.copyOf(partitionBy); + this.orderingScheme = requireNonNull(orderingScheme, "orderingScheme is null"); + } + + public List getPartitionBy() { + return partitionBy; + } + + public Optional getOrderingScheme() { + return orderingScheme; + } + + @Override + public int hashCode() { + return Objects.hash(partitionBy, orderingScheme); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + DataOrganizationSpecification other = (DataOrganizationSpecification) obj; + + return Objects.equals(this.partitionBy, other.partitionBy) + && Objects.equals(this.orderingScheme, other.orderingScheme); + } +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index ef5d303476fa3..23a37a7c6c9c8 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -916,8 +916,8 @@ primaryExpression | dateExpression #dateTimeExpression | '(' expression (',' expression)+ ')' #rowConstructor | ROW '(' expression (',' expression)* ')' #rowConstructor - | qualifiedName '(' (label=identifier '.')? ASTERISK ')' #functionCall - | qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' #functionCall + | qualifiedName '(' (label=identifier '.')? ASTERISK ')' over? #functionCall + | qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' over? #functionCall | '(' query ')' #subqueryExpression // This is an extension to ANSI SQL, which considers EXISTS to be a | EXISTS '(' query ')' #exists @@ -937,6 +937,39 @@ primaryExpression | DATE_BIN '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBin | DATE_BIN_GAPFILL '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBinGapFill | '(' expression ')' #parenthesizedExpression +// | identifier over #simpleOver + ; + +over + : OVER (windowName=identifier | '(' windowSpecification ')') + ; + + +windowSpecification + : (existingWindowName=identifier)? + (PARTITION BY partition+=expression (',' partition+=expression)*)? + (ORDER BY sortItem (',' sortItem)*)? + windowFrame? + ; + +windowFrame + : frameExtent + ; + +frameExtent + : frameType=RANGE start=frameBound + | frameType=ROWS start=frameBound + | frameType=GROUPS start=frameBound + | frameType=RANGE BETWEEN start=frameBound AND end=frameBound + | frameType=ROWS BETWEEN start=frameBound AND end=frameBound + | frameType=GROUPS BETWEEN start=frameBound AND end=frameBound + ; + +frameBound + : UNBOUNDED boundType=PRECEDING #unboundedFrame + | UNBOUNDED boundType=FOLLOWING #unboundedFrame + | CURRENT ROW #currentRowBound + | expression boundType=(PRECEDING | FOLLOWING) #boundedFrame ; literalExpression From e78b5ab47eb584cfe849172729ad41a0fc915168 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 20 Mar 2025 22:46:39 +0800 Subject: [PATCH 02/54] Finish window sub-clause in SQL. --- .../plan/relational/analyzer/Analysis.java | 32 ++ .../analyzer/ExpressionAnalysis.java | 11 +- .../analyzer/ExpressionAnalyzer.java | 69 +++- .../analyzer/ExpressionTreeUtils.java | 4 + .../analyzer/StatementAnalyzer.java | 296 ++++++++++-------- .../plan/relational/planner/QueryPlanner.java | 5 + .../plan/relational/sql/ast/AstVisitor.java | 8 + .../plan/relational/sql/ast/FunctionCall.java | 6 +- .../sql/ast/QuerySpecification.java | 10 +- .../plan/relational/sql/ast/Window.java | 3 + .../relational/sql/ast/WindowDefinition.java | 69 ++++ .../relational/sql/ast/WindowReference.java | 59 ++++ .../sql/ast/WindowSpecification.java | 2 +- .../relational/sql/parser/AstBuilder.java | 22 +- .../sql/util/ExpressionFormatter.java | 8 +- .../plan/relational/sql/util/QueryUtil.java | 38 ++- .../relational/grammar/sql/RelationalSql.g4 | 5 +- 17 files changed, 501 insertions(+), 146 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index ab1b62dfd887e..feb354084d2bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -193,9 +194,14 @@ public class Analysis implements IAnalysis { private final Map> tableColumnSchemas = new HashMap<>(); + private final Map< + NodeRef, Map, ResolvedWindow>> + windowDefinitions = new LinkedHashMap<>(); private final Map, ResolvedWindow> windows = new LinkedHashMap<>(); private final Map, List> windowFunctions = new LinkedHashMap<>(); + private final Map, List> orderByWindowFunctions = + new LinkedHashMap<>(); private DataPartition dataPartition; @@ -1190,6 +1196,24 @@ public Optional getSubqueryCoercion() { } } + public void addWindowDefinition( + QuerySpecification query, CanonicalizationAware name, ResolvedWindow window) { + windowDefinitions + .computeIfAbsent(NodeRef.of(query), key -> new LinkedHashMap<>()) + .put(name, window); + } + + public ResolvedWindow getWindowDefinition( + QuerySpecification query, CanonicalizationAware name) { + Map, ResolvedWindow> windows = + windowDefinitions.get(NodeRef.of(query)); + if (windows != null) { + return windows.get(name); + } + + return null; + } + public void setWindow(Node node, ResolvedWindow window) { windows.put(NodeRef.of(node), window); } @@ -1206,6 +1230,14 @@ public List getWindowFunctions(QuerySpecification query) { return windowFunctions.get(NodeRef.of(query)); } + public void setOrderByWindowFunctions(OrderBy node, List functions) { + orderByWindowFunctions.put(NodeRef.of(node), ImmutableList.copyOf(functions)); + } + + public List getOrderByWindowFunctions(OrderBy query) { + return orderByWindowFunctions.get(NodeRef.of(query)); + } + public static class ResolvedWindow { private final List partitionBy; private final Optional orderBy; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalysis.java index 858ff2e9660c1..e86090e2c06b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalysis.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -43,6 +44,7 @@ public class ExpressionAnalysis { private final Set> subqueries; private final Set> existsSubqueries; private final Set> quantifiedComparisons; + private final Set> windowFunctions; public ExpressionAnalysis( Map, Type> expressionTypes, @@ -50,7 +52,8 @@ public ExpressionAnalysis( Set> subqueries, Set> existsSubqueries, Map, ResolvedField> columnReferences, - Set> quantifiedComparisons) { + Set> quantifiedComparisons, + Set> windowFunctions) { this.expressionTypes = ImmutableMap.copyOf(requireNonNull(expressionTypes, "expressionTypes is null")); // this.expressionCoercions = @@ -67,6 +70,8 @@ public ExpressionAnalysis( ImmutableSet.copyOf(requireNonNull(existsSubqueries, "existsSubqueries is null")); this.quantifiedComparisons = ImmutableSet.copyOf(requireNonNull(quantifiedComparisons, "quantifiedComparisons is null")); + this.windowFunctions = + ImmutableSet.copyOf(requireNonNull(windowFunctions, "windowFunctions is null")); } public Type getType(Expression expression) { @@ -96,4 +101,8 @@ public Set> getExistsSubqueries() { public Set> getQuantifiedComparisons() { return quantifiedComparisons; } + + public Set> getWindowFunctions() { + return windowFunctions; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 9e06bb781157f..81c2e56b72df8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -151,6 +151,8 @@ public class ExpressionAnalyzer { private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); private final Set> subqueries = new LinkedHashSet<>(); private final Set> existsSubqueries = new LinkedHashSet<>(); + private final Map, Type> expressionCoercions = new LinkedHashMap<>(); + private final Set> typeOnlyCoercions = new LinkedHashSet<>(); private final Set> subqueryInPredicates = new LinkedHashSet<>(); private final Map, Analysis.PredicateCoercions> predicateCoercions = @@ -234,6 +236,55 @@ private ExpressionAnalyzer( this.getPreanalyzedType = requireNonNull(getPreanalyzedType, "getPreanalyzedType is null"); } + public static ExpressionAnalysis analyzeWindow( + Metadata metadata, + SessionInfo session, + MPPQueryContext queryContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector noop, + CorrelationSupport correlationSupport, + Analysis.ResolvedWindow window, + Node originalNode) { + ExpressionAnalyzer analyzer = + new ExpressionAnalyzer( + metadata, + queryContext, + accessControl, + statementAnalyzerFactory, + analysis, + session, + TypeProvider.empty(), + noop); + analyzer.analyzeWindow(window, scope, originalNode, correlationSupport); + + updateAnalysis(analysis, analyzer, session, accessControl); + + return new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions()); + } + + private void analyzeWindow( + Analysis.ResolvedWindow window, + Scope scope, + Node originalNode, + CorrelationSupport correlationSupport) { + Visitor visitor = new Visitor(scope, warningCollector); + visitor.analyzeWindow( + window, + new StackableAstVisitor.StackableAstVisitorContext<>( + Context.notInLambda(scope, correlationSupport)), + originalNode); + } + public Map, ResolvedFunction> getResolvedFunctions() { return unmodifiableMap(resolvedFunctions); } @@ -251,6 +302,18 @@ public Type setExpressionType(Expression expression, Type type) { return type; } + public Set> getWindowFunctions() { + return unmodifiableSet(windowFunctions); + } + + public Map, Type> getExpressionCoercions() { + return unmodifiableMap(expressionCoercions); + } + + public Set> getTypeOnlyCoercions() { + return unmodifiableSet(typeOnlyCoercions); + } + private Type getExpressionType(Expression expression) { requireNonNull(expression, "expression cannot be null"); @@ -1675,7 +1738,8 @@ public static ExpressionAnalysis analyzeExpressions( analyzer.getSubqueries(), analyzer.getExistsSubqueries(), analyzer.getColumnReferences(), - analyzer.getQuantifiedComparisons()); + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions()); } public static ExpressionAnalysis analyzeExpression( @@ -1710,7 +1774,8 @@ public static ExpressionAnalysis analyzeExpression( analyzer.getSubqueries(), analyzer.getExistsSubqueries(), analyzer.getColumnReferences(), - analyzer.getQuantifiedComparisons()); + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions()); } public static void analyzeExpressionWithoutSubqueries( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java index e84a8a87113aa..e1105c4ef1504 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java @@ -69,6 +69,10 @@ public static List extractWindowFunctions(Iterable return extractExpressions(nodes, FunctionCall.class, ExpressionTreeUtils::isWindowFunction); } + static List extractWindowExpressions(Iterable nodes) { + return ImmutableList.builder().addAll(extractWindowFunctions(nodes)).build(); + } + private static boolean isWindowFunction(FunctionCall functionCall) { return functionCall.getWindow().isPresent(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 7f6f2450cd1f5..4f3bcde73ad01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -95,6 +95,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.CanonicalizationAware.canonicalizationAwareKey; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.asQualifiedName; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractAggregateFunctions; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractWindowExpressions; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractWindowFunctions; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.Scope.BasisType.TABLE; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.MetadataUtil.createQualifiedObjectName; @@ -817,6 +818,7 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope sourceScope = analyzeFrom(node, scope); + analyzeWindowDefinitions(node, sourceScope); resolveFunctionCallAndMeasureWindows(node); node.getWhere().ifPresent(where -> analyzeWhere(node, sourceScope, where)); @@ -872,6 +874,19 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional .forEach(sourceExpressions::add); node.getHaving().ifPresent(sourceExpressions::add); + for (WindowDefinition windowDefinition : node.getWindows()) { + WindowSpecification window = windowDefinition.getWindow(); + sourceExpressions.addAll(window.getPartitionBy()); + getSortItemsFromOrderBy(window.getOrderBy()).stream() + .map(SortItem::getSortKey) + .forEach(sourceExpressions::add); + if (window.getFrame().isPresent()) { + WindowFrame frame = window.getFrame().get(); + frame.getStart().getValue().ifPresent(sourceExpressions::add); + frame.getEnd().flatMap(FrameBound::getValue).ifPresent(sourceExpressions::add); + } + } + analyzeAggregations( node, sourceScope, orderByScope, groupByAnalysis, sourceExpressions, orderByExpressions); analyzeWindowFunctionsAndMeasures(node, outputExpressions, orderByExpressions); @@ -902,64 +917,47 @@ private void analyzeWindowFunctionsAndMeasures( List outputExpressions, List orderByExpressions) { analysis.setWindowFunctions(node, analyzeWindowFunctions(node, outputExpressions)); - // if (node.getOrderBy().isPresent()) { - // OrderBy orderBy = node.getOrderBy().get(); - // analysis.setOrderByWindowFunctions(orderBy, analyzeWindowFunctions(node, - // orderByExpressions)); - // analysis.setOrderByWindowMeasures(orderBy, - // extractWindowMeasures(orderByExpressions)); - // } + if (node.getOrderBy().isPresent()) { + OrderBy orderBy = node.getOrderBy().get(); + analysis.setOrderByWindowFunctions( + orderBy, analyzeWindowFunctions(node, orderByExpressions)); + } } private List analyzeWindowFunctions( QuerySpecification node, List expressions) { List windowFunctions = extractWindowFunctions(expressions); - // for (FunctionCall windowFunction : windowFunctions) { - // List nestedWindowExpressions = - // extractWindowExpressions(windowFunction.getArguments()); - // if (!nestedWindowExpressions.isEmpty()) { - // throw semanticException(NESTED_WINDOW, nestedWindowExpressions.get(0), "Cannot - // nest window functions or row pattern measures inside window function arguments"); - // } - // - // if (windowFunction.isDistinct()) { - // throw semanticException(NOT_SUPPORTED, node, "DISTINCT in window function - // parameters not yet supported: %s", windowFunction); - // } - // - // Analysis.ResolvedWindow window = analysis.getWindow(windowFunction); - // // TODO get function requirements from window function metadata when we have it - // String name = windowFunction.getName().toString().toLowerCase(ENGLISH); - // if (name.equals("lag") || name.equals("lead")) { - // if (!window.getOrderBy().isPresent()) { - // throw semanticException(MISSING_ORDER_BY, (Node) - // windowFunction.getWindow().orElseThrow(), "%s function requires an ORDER BY window clause", - // windowFunction.getName()); - // } - // if (window.getFrame().isPresent()) { - // throw semanticException(INVALID_WINDOW_FRAME, window.getFrame().get(), "Cannot - // specify window frame for %s function", windowFunction.getName()); - // } - // } - // - // if (!WINDOW_VALUE_FUNCTIONS.contains(name) && - // windowFunction.getNullTreatment().isPresent()) { - // throw semanticException(NULL_TREATMENT_NOT_ALLOWED, windowFunction, "Cannot - // specify null treatment clause for %s function", windowFunction.getName()); - // } - // - // List argumentTypes = mappedCopy(windowFunction.getArguments(), - // analysis::getType); - // - // ResolvedFunction resolvedFunction = functionResolver.resolveFunction(session, - // windowFunction.getName(), fromTypes(argumentTypes), accessControl); - // FunctionKind kind = resolvedFunction.getFunctionKind(); - // if (kind != AGGREGATE && kind != WINDOW) { - // throw semanticException(FUNCTION_NOT_WINDOW, node, "Not a window function: %s", - // windowFunction.getName()); - // } - // } + for (FunctionCall windowFunction : windowFunctions) { + List nestedWindowExpressions = + extractWindowExpressions(windowFunction.getArguments()); + if (!nestedWindowExpressions.isEmpty()) { + throw new SemanticException( + "Cannot nest window functions or row pattern measures inside window function arguments"); + } + + if (windowFunction.isDistinct()) { + throw new SemanticException( + String.format( + "DISTINCT in window function parameters not yet supported: %s", windowFunction)); + } + + Analysis.ResolvedWindow window = analysis.getWindow(windowFunction); + // TODO get function requirements from window function metadata when we have it + String name = windowFunction.getName().toString().toLowerCase(ENGLISH); + if (name.equals("lag") || name.equals("lead")) { + if (!window.getOrderBy().isPresent()) { + throw new SemanticException( + String.format( + "%s function requires an ORDER BY window clause", windowFunction.getName())); + } + if (window.getFrame().isPresent()) { + throw new SemanticException( + String.format( + "Cannot specify window frame for %s function", windowFunction.getName())); + } + } + } return windowFunctions; } @@ -986,83 +984,131 @@ private void resolveFunctionCallAndMeasureWindows(QuerySpecification querySpecif } } + private void analyzeWindowDefinitions(QuerySpecification node, Scope scope) { + for (WindowDefinition windowDefinition : node.getWindows()) { + CanonicalizationAware canonicalName = + canonicalizationAwareKey(windowDefinition.getName()); + + if (analysis.getWindowDefinition(node, canonicalName) != null) { + throw new SemanticException( + String.format( + "WINDOW name '%s' specified more than once", windowDefinition.getName())); + } + + Analysis.ResolvedWindow resolvedWindow = + resolveWindowSpecification(node, windowDefinition.getWindow()); + + // Analyze window after it is resolved, because resolving might provide necessary + // information, e.g. ORDER BY necessary for frame analysis. + // Analyze only newly introduced window properties. Properties of the referenced window have + // been already analyzed. + analyzeWindow(node, resolvedWindow, scope, windowDefinition.getWindow()); + + analysis.addWindowDefinition(node, canonicalName, resolvedWindow); + } + } + + private void analyzeWindow( + QuerySpecification querySpecification, + Analysis.ResolvedWindow window, + Scope scope, + Node originalNode) { + ExpressionAnalysis expressionAnalysis = + ExpressionAnalyzer.analyzeWindow( + metadata, + sessionContext, + queryContext, + statementAnalyzerFactory, + accessControl, + scope, + analysis, + WarningCollector.NOOP, + correlationSupport, + window, + originalNode); + analysis.recordSubqueries(querySpecification, expressionAnalysis); + } + private Analysis.ResolvedWindow resolveWindowSpecification( - QuerySpecification querySpecification, WindowSpecification window) { - // if (window instanceof WindowReference windowReference) { - // CanonicalizationAware canonicalName = - // canonicalizationAwareKey(windowReference.getName()); - // ResolvedWindow referencedWindow = analysis.getWindowDefinition(querySpecification, - // canonicalName); - // if (referencedWindow == null) { - // throw semanticException(INVALID_WINDOW_REFERENCE, windowReference.getName(), - // "Cannot resolve WINDOW name %s", windowReference.getName()); - // } - // - // return new ResolvedWindow( - // referencedWindow.getPartitionBy(), - // referencedWindow.getOrderBy(), - // referencedWindow.getFrame(), - // !referencedWindow.getPartitionBy().isEmpty(), - // referencedWindow.getOrderBy().isPresent(), - // referencedWindow.getFrame().isPresent()); - // } + QuerySpecification querySpecification, Window window) { + if (window instanceof WindowReference) { + WindowReference windowReference = (WindowReference) window; + CanonicalizationAware canonicalName = + canonicalizationAwareKey(windowReference.getName()); + Analysis.ResolvedWindow referencedWindow = + analysis.getWindowDefinition(querySpecification, canonicalName); + if (referencedWindow == null) { + throw new SemanticException( + String.format("Cannot resolve WINDOW name %s", windowReference.getName())); + } + + return new Analysis.ResolvedWindow( + referencedWindow.getPartitionBy(), + referencedWindow.getOrderBy(), + referencedWindow.getFrame(), + !referencedWindow.getPartitionBy().isEmpty(), + referencedWindow.getOrderBy().isPresent(), + referencedWindow.getFrame().isPresent()); + } WindowSpecification windowSpecification = (WindowSpecification) window; - // if (windowSpecification.getExistingWindowName().isPresent()) { - // Identifier referencedName = windowSpecification.getExistingWindowName().get(); - // CanonicalizationAware canonicalName = - // canonicalizationAwareKey(referencedName); - // Analysis.ResolvedWindow referencedWindow = - // analysis.getWindowDefinition(querySpecification, canonicalName); - // if (referencedWindow == null) { - // throw semanticException(INVALID_WINDOW_REFERENCE, referencedName, "Cannot resolve - // WINDOW name %s", referencedName); - // } - // - // // analyze dependencies between this window specification and referenced window - // specification - // if (!windowSpecification.getPartitionBy().isEmpty()) { - // throw semanticException(INVALID_PARTITION_BY, - // windowSpecification.getPartitionBy().get(0), "WINDOW specification with named WINDOW - // reference cannot specify PARTITION BY"); - // } - // if (windowSpecification.getOrderBy().isPresent() && - // referencedWindow.getOrderBy().isPresent()) { - // throw semanticException(INVALID_ORDER_BY, windowSpecification.getOrderBy().get(), - // "Cannot specify ORDER BY if referenced named WINDOW specifies ORDER BY"); - // } - // if (referencedWindow.getFrame().isPresent()) { - // throw semanticException(INVALID_WINDOW_REFERENCE, - // windowSpecification.getExistingWindowName().get(), "Cannot reference named WINDOW - // containing frame specification"); - // } - // - // // resolve window - // Optional orderBy = windowSpecification.getOrderBy(); - // boolean orderByInherited = false; - // if (!orderBy.isPresent() && referencedWindow.getOrderBy().isPresent()) { - // orderBy = referencedWindow.getOrderBy(); - // orderByInherited = true; - // } - // - // List partitionBy = windowSpecification.getPartitionBy(); - // boolean partitionByInherited = false; - // if (!referencedWindow.getPartitionBy().isEmpty()) { - // partitionBy = referencedWindow.getPartitionBy(); - // partitionByInherited = true; - // } - // - // Optional windowFrame = windowSpecification.getFrame(); - // boolean frameInherited = false; - // if (!windowFrame.isPresent() && referencedWindow.getFrame().isPresent()) { - // windowFrame = referencedWindow.getFrame(); - // frameInherited = true; - // } - // - // return new Analysis.ResolvedWindow(partitionBy, orderBy, windowFrame, - // partitionByInherited, orderByInherited, frameInherited); - // } + if (windowSpecification.getExistingWindowName().isPresent()) { + Identifier referencedName = windowSpecification.getExistingWindowName().get(); + CanonicalizationAware canonicalName = canonicalizationAwareKey(referencedName); + Analysis.ResolvedWindow referencedWindow = + analysis.getWindowDefinition(querySpecification, canonicalName); + if (referencedWindow == null) { + throw new SemanticException( + String.format("Cannot resolve WINDOW name %s", referencedName)); + } + + // analyze dependencies between this window specification and referenced window + // specification + if (!windowSpecification.getPartitionBy().isEmpty()) { + throw new SemanticException( + "WINDOW specification with named WINDOW reference cannot specify PARTITION BY"); + } + if (windowSpecification.getOrderBy().isPresent() + && referencedWindow.getOrderBy().isPresent()) { + throw new SemanticException( + "Cannot specify ORDER BY if referenced named WINDOW specifies ORDER BY"); + } + if (referencedWindow.getFrame().isPresent()) { + throw new SemanticException( + "Cannot reference named WINDOW containing frame specification"); + } + + // resolve window + Optional orderBy = windowSpecification.getOrderBy(); + boolean orderByInherited = false; + if (!orderBy.isPresent() && referencedWindow.getOrderBy().isPresent()) { + orderBy = referencedWindow.getOrderBy(); + orderByInherited = true; + } + + List partitionBy = windowSpecification.getPartitionBy(); + boolean partitionByInherited = false; + if (!referencedWindow.getPartitionBy().isEmpty()) { + partitionBy = referencedWindow.getPartitionBy(); + partitionByInherited = true; + } + + Optional windowFrame = windowSpecification.getFrame(); + boolean frameInherited = false; + if (!windowFrame.isPresent() && referencedWindow.getFrame().isPresent()) { + windowFrame = referencedWindow.getFrame(); + frameInherited = true; + } + + return new Analysis.ResolvedWindow( + partitionBy, + orderBy, + windowFrame, + partitionByInherited, + orderByInherited, + frameInherited); + } return new Analysis.ResolvedWindow( windowSpecification.getPartitionBy(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 6dabd2b8e1e5a..ff73b876da40a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -274,6 +274,11 @@ public RelationPlan plan(QuerySpecification node) { outputs.stream().map(builder::translate).forEach(newFields::add); builder = builder.withScope(analysis.getScope(node.getOrderBy().orElse(null)), newFields); + builder = + planWindowFunctions( + node, + builder, + ImmutableList.copyOf(analysis.getOrderByWindowFunctions(node.getOrderBy().get()))); analysis.setSortNode(true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 163e2d20829d2..e3f91674c53c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -225,6 +225,14 @@ protected R visitNotExpression(NotExpression node, C context) { return visitExpression(node, context); } + protected R visitWindowDefinition(WindowDefinition node, C context) { + return visitNode(node, context); + } + + protected R visitWindowReference(WindowReference node, C context) { + return visitNode(node, context); + } + protected R visitWindowSpecification(WindowSpecification node, C context) { return visitNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index c29eca11929b8..e522e45709bcd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -34,7 +34,7 @@ public class FunctionCall extends Expression { private final QualifiedName name; - private final Optional window; + private final Optional window; private final boolean distinct; private final List arguments; @@ -70,7 +70,7 @@ public FunctionCall( public FunctionCall( NodeLocation location, QualifiedName name, - Optional window, + Optional window, boolean distinct, List arguments) { super(requireNonNull(location, "location is null")); @@ -93,7 +93,7 @@ public List getArguments() { return arguments; } - public Optional getWindow() { + public Optional getWindow() { return window; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java index 0be5ed4ccdaa0..7c59844c5d6c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuerySpecification.java @@ -36,6 +36,7 @@ public class QuerySpecification extends QueryBody { private final Optional groupBy; private final Optional having; private final Optional fill; + private final List windows; private final Optional orderBy; private final Optional offset; private final Optional limit; @@ -47,10 +48,11 @@ public QuerySpecification( Optional groupBy, Optional having, Optional fill, + List windows, Optional orderBy, Optional offset, Optional limit) { - this(null, select, from, where, groupBy, having, fill, orderBy, offset, limit); + this(null, select, from, where, groupBy, having, fill, windows, orderBy, offset, limit); } public QuerySpecification( @@ -61,6 +63,7 @@ public QuerySpecification( Optional groupBy, Optional having, Optional fill, + List windows, Optional orderBy, Optional offset, Optional limit) { @@ -72,6 +75,7 @@ public QuerySpecification( this.groupBy = requireNonNull(groupBy, "groupBy is null"); this.having = requireNonNull(having, "having is null"); this.fill = requireNonNull(fill, "fill is null"); + this.windows = requireNonNull(windows, "windows is null"); this.orderBy = requireNonNull(orderBy, "orderBy is null"); this.offset = requireNonNull(offset, "offset is null"); this.limit = requireNonNull(limit, "limit is null"); @@ -101,6 +105,10 @@ public Optional getFill() { return fill; } + public List getWindows() { + return windows; + } + public Optional getOrderBy() { return orderBy; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java new file mode 100644 index 0000000000000..2f750d77d94cb --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java @@ -0,0 +1,3 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public interface Window {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java new file mode 100644 index 0000000000000..67c553f79f5c8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java @@ -0,0 +1,69 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class WindowDefinition extends Node { + private final Identifier name; + private final WindowSpecification window; + + public WindowDefinition(NodeLocation location, Identifier name, WindowSpecification window) { + super(location); + this.name = requireNonNull(name, "name is null"); + this.window = requireNonNull(window, "window is null"); + } + + public Identifier getName() { + return name; + } + + public WindowSpecification getWindow() { + return window; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitWindowDefinition(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(window); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + WindowDefinition o = (WindowDefinition) obj; + return Objects.equals(name, o.name) && Objects.equals(window, o.window); + } + + @Override + public int hashCode() { + return Objects.hash(name, window); + } + + @Override + public String toString() { + return toStringHelper(this).add("name", name).add("window", window).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return name.equals(((WindowDefinition) other).name); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java new file mode 100644 index 0000000000000..203032797820a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java @@ -0,0 +1,59 @@ +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class WindowReference extends Node implements Window { + private final Identifier name; + + public WindowReference(NodeLocation location, Identifier name) { + super(location); + this.name = requireNonNull(name, "name is null"); + } + + public Identifier getName() { + return name; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitWindowReference(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(name); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + WindowReference o = (WindowReference) obj; + return Objects.equals(name, o.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return toStringHelper(this).add("name", name).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java index 9cf78095d7a05..2b3a442e576ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java @@ -9,7 +9,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; -public class WindowSpecification extends Node { +public class WindowSpecification extends Node implements Window { private final Optional existingWindowName; private final List partitionBy; private final Optional orderBy; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index bda8d0e92d4b0..5cc2a784b2726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -1631,6 +1631,7 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { query.getGroupBy(), query.getHaving(), fill, + query.getWindows(), orderBy, offset, limit), @@ -1757,6 +1758,7 @@ public Node visitQuerySpecification(RelationalSqlParser.QuerySpecificationContex visitIfPresent(ctx.groupBy(), GroupBy.class), visitIfPresent(ctx.having, Expression.class), Optional.empty(), + visit(ctx.windowDefinition(), WindowDefinition.class), Optional.empty(), Optional.empty(), Optional.empty()); @@ -2209,17 +2211,23 @@ public Node visitConcatenation(RelationalSqlParser.ConcatenationContext ctx) { } // ********************* primary expressions ********************** - // @Override - // public Node visitSimpleOver(RelationalSqlParser.SimpleOverContext ctx) { - // return visitWindowSpecification(ctx.over().windowSpecification()); - // } - @Override public Node visitOver(RelationalSqlParser.OverContext ctx) { - // TODO: Window Reference + if (ctx.windowName != null) { + return new WindowReference(getLocation(ctx), (Identifier) visit(ctx.windowName)); + } + return visit(ctx.windowSpecification()); } + @Override + public Node visitWindowDefinition(RelationalSqlParser.WindowDefinitionContext ctx) { + return new WindowDefinition( + getLocation(ctx), + (Identifier) visit(ctx.name), + (WindowSpecification) visit(ctx.windowSpecification())); + } + @Override public Node visitWindowSpecification(RelationalSqlParser.WindowSpecificationContext ctx) { Optional existingWindowName = getIdentifierIfPresent(ctx.existingWindowName); @@ -2448,7 +2456,7 @@ public Node visitWhenClause(RelationalSqlParser.WhenClauseContext ctx) { @Override public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { - Optional window = visitIfPresent(ctx.over(), WindowSpecification.class); + Optional window = visitIfPresent(ctx.over(), Window.class); QualifiedName name = getQualifiedName(ctx.qualifiedName()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index b4efdb36a88b5..3f3df0f0354dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -557,10 +557,10 @@ public static String formatSortItems(List sortItems) { return sortItems.stream().map(sortItemFormatterFunction()).collect(joining(", ")); } - private static String formatWindow(WindowSpecification window) { - // if (window instanceof WindowReference) { - // return formatExpression(((WindowReference) window).getName()); - // } + private static String formatWindow(Window window) { + if (window instanceof WindowReference) { + return formatExpression(((WindowReference) window).getName()); + } return formatWindowSpecification((WindowSpecification) window); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java index a5a86a892c98c..f0bd840453e2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/QueryUtil.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowDefinition; import com.google.common.collect.ImmutableList; @@ -156,6 +157,7 @@ public static Query simpleQuery(Select select) { Optional.empty(), Optional.empty(), Optional.empty(), + ImmutableList.of(), Optional.empty(), Optional.empty(), Optional.empty())); @@ -203,7 +205,41 @@ public static Query simpleQuery( Optional limit) { return query( new QuerySpecification( - select, Optional.of(from), where, groupBy, having, fill, orderBy, offset, limit)); + select, + Optional.of(from), + where, + groupBy, + having, + fill, + ImmutableList.of(), + orderBy, + offset, + limit)); + } + + public static Query simpleQuery( + Select select, + Relation from, + Optional where, + Optional groupBy, + Optional having, + Optional fill, + List windows, + Optional orderBy, + Optional offset, + Optional limit) { + return query( + new QuerySpecification( + select, + Optional.of(from), + where, + groupBy, + having, + fill, + windows, + orderBy, + offset, + limit)); } public static Query query(QueryBody body) { diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 23a37a7c6c9c8..0674057d832d1 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -793,6 +793,7 @@ querySpecification (WHERE where=booleanExpression)? (GROUP BY groupBy)? (HAVING having=booleanExpression)? + (WINDOW windowDefinition (',' windowDefinition)*)? ; groupBy @@ -937,13 +938,15 @@ primaryExpression | DATE_BIN '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBin | DATE_BIN_GAPFILL '(' timeDuration ',' valueExpression (',' timeValue)? ')' #dateBinGapFill | '(' expression ')' #parenthesizedExpression -// | identifier over #simpleOver ; over : OVER (windowName=identifier | '(' windowSpecification ')') ; +windowDefinition + : name=identifier AS '(' windowSpecification ')' + ; windowSpecification : (existingWindowName=identifier)? From 56e9dcb0fd7f9ba1431c6f12d154e94e8e629726 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 21 Mar 2025 00:49:18 +0800 Subject: [PATCH 03/54] Replace parameters with channels in window function. --- .../window/function/rank/NTileFunction.java | 13 +++-- .../window/function/value/LagFunction.java | 56 ++++++++++++++++--- .../window/function/value/LeadFunction.java | 55 +++++++++++++++--- .../function/value/NthValueFunction.java | 23 ++++---- .../function/rank/NTileFunctionTest.java | 27 ++++++--- .../function/value/LagFunctionTest.java | 54 +++++++++++++++--- .../function/value/LeadFunctionTest.java | 54 +++++++++++++++--- .../function/value/NthValueFunctionTest.java | 35 +++++++++--- 8 files changed, 257 insertions(+), 60 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunction.java index 48b70cc8ba69b..c218215f19832 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunction.java @@ -24,10 +24,10 @@ import org.apache.tsfile.block.column.ColumnBuilder; public class NTileFunction extends RankWindowFunction { - private final int n; + private final int nChannel; - public NTileFunction(int n) { - this.n = n; + public NTileFunction(int nChannel) { + this.nChannel = nChannel; } @Override @@ -37,7 +37,12 @@ public void transform( int index, boolean isNewPeerGroup, int peerGroupCount) { - builder.writeLong(bucket(n, index, partition.getPositionCount()) + 1); + if (partition.isNull(nChannel, index)) { + builder.appendNull(); + } else { + long n = partition.getLong(nChannel, index); + builder.writeLong(bucket(n, index, partition.getPositionCount()) + 1); + } } private long bucket(long buckets, int index, int count) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index cfa48742f9240..63b99016873b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -22,23 +22,34 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.Partition; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.util.List; public class LagFunction extends ValueWindowFunction { private final int channel; - private final Integer offset; - private final Object defaultVal; + private final int offsetChannel; + private final int defaultValChannel; private final boolean ignoreNull; - public LagFunction(int channel, Integer offset, Object defaultVal, boolean ignoreNull) { - this.channel = channel; - this.offset = offset == null ? 1 : offset; - this.defaultVal = defaultVal; + public LagFunction(List argumentChannels, boolean ignoreNull) { + this.channel = argumentChannels.get(0); + this.offsetChannel = argumentChannels.size() > 1 ? argumentChannels.get(1) : -1; + this.defaultValChannel = argumentChannels.size() > 2 ? argumentChannels.get(2) : -1; this.ignoreNull = ignoreNull; } @Override public void transform( Partition partition, ColumnBuilder builder, int index, int frameStart, int frameEnd) { + if (offsetChannel >= 0 && partition.isNull(offsetChannel, index)) { + builder.appendNull(); + return; + } + + int offset = offsetChannel >= 0? partition.getInt(offsetChannel, index) : 1; + int pos; if (ignoreNull) { int nonNullCount = 0; @@ -63,13 +74,42 @@ public void transform( } else { builder.appendNull(); } - } else if (defaultVal != null) { - builder.writeObject(defaultVal); + } else if (defaultValChannel >= 0) { + writeDefaultValue(partition, defaultValChannel, index, builder); } else { builder.appendNull(); } } + private void writeDefaultValue(Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { + TSDataType dataType = builder.getDataType(); + switch (dataType) { + case INT32: + case DATE: + builder.writeInt(partition.getInt(defaultValChannel, index)); + return; + case INT64: + case TIMESTAMP: + builder.writeLong(partition.getLong(defaultValChannel, index)); + return; + case FLOAT: + builder.writeFloat(partition.getFloat(defaultValChannel, index)); + return; + case DOUBLE: + builder.writeDouble(partition.getDouble(defaultValChannel, index)); + return; + case BOOLEAN: + builder.writeBoolean(partition.getBoolean(defaultValChannel, index)); + return; + case TEXT: + case STRING: + builder.writeBinary(partition.getBinary(defaultValChannel, index)); + return; + default: + throw new UnSupportedDataTypeException("Unsupported default value's data type in Lag: " + dataType); + } + } + @Override public boolean needFrame() { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java index 514357df57a98..2ae5c756aa737 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java @@ -22,24 +22,34 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.Partition; import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.util.List; public class LeadFunction extends ValueWindowFunction { private final int channel; - private final Integer offset; - private final Integer defaultVal; + private final int offsetChannel; + private final int defaultValChannel; private final boolean ignoreNull; - public LeadFunction(int channel, Integer offset, Integer defaultVal, boolean ignoreNull) { - this.channel = channel; - this.offset = offset == null ? 1 : offset; - this.defaultVal = defaultVal; + public LeadFunction(List argumentChannels, boolean ignoreNull) { + this.channel = argumentChannels.get(0); + this.offsetChannel = argumentChannels.size() > 1 ? argumentChannels.get(1) : -1; + this.defaultValChannel = argumentChannels.size() > 2 ? argumentChannels.get(2) : -1; this.ignoreNull = ignoreNull; } @Override public void transform( Partition partition, ColumnBuilder builder, int index, int frameStart, int frameEnd) { + if (offsetChannel >= 0 && partition.isNull(offsetChannel, index)) { + builder.appendNull(); + return; + } + int length = partition.getPositionCount(); + int offset = offsetChannel >= 0? partition.getInt(offsetChannel, index) : 1; int pos; if (ignoreNull) { @@ -65,13 +75,42 @@ public void transform( } else { builder.appendNull(); } - } else if (defaultVal != null) { - builder.writeObject(defaultVal); + } else if (defaultValChannel >= 0) { + writeDefaultValue(partition, defaultValChannel, index, builder); } else { builder.appendNull(); } } + private void writeDefaultValue(Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { + TSDataType dataType = builder.getDataType(); + switch (dataType) { + case INT32: + case DATE: + builder.writeInt(partition.getInt(defaultValChannel, index)); + return; + case INT64: + case TIMESTAMP: + builder.writeLong(partition.getLong(defaultValChannel, index)); + return; + case FLOAT: + builder.writeFloat(partition.getFloat(defaultValChannel, index)); + return; + case DOUBLE: + builder.writeDouble(partition.getDouble(defaultValChannel, index)); + return; + case BOOLEAN: + builder.writeBoolean(partition.getBoolean(defaultValChannel, index)); + return; + case TEXT: + case STRING: + builder.writeBinary(partition.getBinary(defaultValChannel, index)); + return; + default: + throw new UnSupportedDataTypeException("Unsupported default value's data type in Lag: " + dataType); + } + } + @Override public boolean needFrame() { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunction.java index 8978b5f13b612..83c245119ff01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunction.java @@ -23,14 +23,16 @@ import org.apache.tsfile.block.column.ColumnBuilder; +import java.util.List; + public class NthValueFunction extends ValueWindowFunction { - private final int n; - private final int channel; + private final int valueChannel; + private final int nChannel; private final boolean ignoreNull; - public NthValueFunction(int n, int channel, boolean ignoreNull) { - this.n = n; - this.channel = channel; + public NthValueFunction(List argumentChannels, boolean ignoreNull) { + this.valueChannel = argumentChannels.get(0); + this.nChannel = argumentChannels.get(1); this.ignoreNull = ignoreNull; } @@ -38,18 +40,19 @@ public NthValueFunction(int n, int channel, boolean ignoreNull) { public void transform( Partition partition, ColumnBuilder builder, int index, int frameStart, int frameEnd) { // Empty frame - if (frameStart < 0) { + if (frameStart < 0 || partition.isNull(nChannel, index)) { builder.appendNull(); return; } int pos; + int n = partition.getInt(nChannel, index); if (ignoreNull) { // Handle nulls pos = frameStart; int nonNullCount = 0; while (pos <= frameEnd) { - if (!partition.isNull(channel, pos)) { + if (!partition.isNull(valueChannel, pos)) { nonNullCount++; if (nonNullCount == n) { break; @@ -59,7 +62,7 @@ public void transform( } if (pos <= frameEnd) { - partition.writeTo(builder, channel, pos); + partition.writeTo(builder, valueChannel, pos); } else { builder.appendNull(); } @@ -69,8 +72,8 @@ public void transform( // n starts with 1 pos = frameStart + n - 1; if (pos <= frameEnd) { - if (!partition.isNull(channel, pos)) { - partition.writeTo(builder, channel, pos); + if (!partition.isNull(valueChannel, pos)) { + partition.writeTo(builder, valueChannel, pos); } else { builder.appendNull(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java index 92916a258a3b2..3b79cbae245df 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java @@ -19,11 +19,11 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperatorTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.FunctionTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -48,8 +48,8 @@ public void testNTileFunctionWhenNIsLarge() { int[] inputs = {1, 2, 3}; int[] expected = {1, 2, 3}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithoutNulls(inputs); - NTileFunction function = new NTileFunction(n); + TsBlock tsBlock = createTsBlockWithN(inputs, n); + NTileFunction function = new NTileFunction(1); List sortedColumns = Collections.singletonList(0); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function, sortedColumns); @@ -76,8 +76,8 @@ public void testNTileFunctionUniform() { int[] inputs = {1, 2, 3, 4, 5, 6}; int[] expected = {1, 1, 2, 2, 3, 3}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithoutNulls(inputs); - NTileFunction function = new NTileFunction(n); + TsBlock tsBlock = createTsBlockWithN(inputs, n); + NTileFunction function = new NTileFunction(1); List sortedColumns = Collections.singletonList(0); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function, sortedColumns); @@ -104,8 +104,8 @@ public void testNTileFunctionNonUniform() { int[] inputs = {1, 2, 3, 4, 5, 6, 7}; int[] expected = {1, 1, 1, 2, 2, 3, 3}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithoutNulls(inputs); - NTileFunction function = new NTileFunction(n); + TsBlock tsBlock = createTsBlockWithN(inputs, n); + NTileFunction function = new NTileFunction(1); List sortedColumns = Collections.singletonList(0); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function, sortedColumns); @@ -125,4 +125,17 @@ public void testNTileFunctionNonUniform() { Assert.assertEquals(column.getLong(i), expected[i]); } } + + private static TsBlock createTsBlockWithN(int[] inputs, int n) { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + columnBuilders[0].writeInt(input); + columnBuilders[1].writeInt(n); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java index 367eb15bb6c59..831227d4116b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -49,8 +50,8 @@ public class LagFunctionTest { public void testLagFunctionIgnoreNullWithoutDefault() { int[] expected = {-1, -1, -1, -1, 0, 1, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LagFunction function = new LagFunction(0, 2, null, true); + TsBlock tsBlock = createTsBlockWithoutDefault(inputs, 2); + LagFunction function = new LagFunction(Arrays.asList(0, 1), true); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -78,8 +79,8 @@ public void testLagFunctionIgnoreNullWithoutDefault() { public void testLagFunctionIgnoreNullWithDefault() { int[] expected = {10, 10, 10, 10, 0, 1, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LagFunction function = new LagFunction(0, 2, 10, true); + TsBlock tsBlock = createTsBlockWithDefault(inputs, 2, 10); + LagFunction function = new LagFunction(Arrays.asList(0 ,1, 2), true); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -103,8 +104,8 @@ public void testLagFunctionIgnoreNullWithDefault() { public void testLagFunctionNotIgnoreNullWithoutDefault() { int[] expected = {-1, -1, 0, -1, -1, 1, 2, -1, 3, 4, -1, 5, 6, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LagFunction function = new LagFunction(0, 2, null, false); + TsBlock tsBlock = createTsBlockWithoutDefault(inputs, 2); + LagFunction function = new LagFunction(Arrays.asList(0, 1), false); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -132,8 +133,8 @@ public void testLagFunctionNotIgnoreNullWithoutDefault() { public void testLagFunctionNotIgnoreNullWithDefault() { int[] expected = {10, 10, 0, -1, -1, 1, 2, -1, 3, 4, -1, 5, 6, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LagFunction function = new LagFunction(0, 2, 10, false); + TsBlock tsBlock = createTsBlockWithDefault(inputs, 2, 10); + LagFunction function = new LagFunction(Arrays.asList(0, 1, 2), false); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -156,4 +157,41 @@ public void testLagFunctionNotIgnoreNullWithDefault() { } } } + + private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int defaultValue) { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + if (input >= 0) { + columnBuilders[0].writeInt(input); + } else { + // Mimic null value + columnBuilders[0].appendNull(); + } + columnBuilders[1].writeInt(offset); + columnBuilders[2].writeInt(defaultValue); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } + + private static TsBlock createTsBlockWithoutDefault(int[] inputs, int offset) { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + if (input >= 0) { + columnBuilders[0].writeInt(input); + } else { + // Mimic null value + columnBuilders[0].appendNull(); + } + columnBuilders[1].writeInt(offset); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java index 90deb02b41513..c82cb837b4fb2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -49,8 +50,8 @@ public class LeadFunctionTest { public void testLeadFunctionIgnoreNullWithoutDefault() { int[] expected = {2, 2, 2, 3, 4, 4, 5, 6, 6, -1, -1, -1, -1, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LeadFunction function = new LeadFunction(0, 2, null, true); + TsBlock tsBlock = createTsBlockWithoutDefault(inputs, 2); + LeadFunction function = new LeadFunction(Arrays.asList(0, 1), true); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -78,8 +79,8 @@ public void testLeadFunctionIgnoreNullWithoutDefault() { public void testLeadFunctionIgnoreNullWithDefault() { int[] expected = {2, 2, 2, 3, 4, 4, 5, 6, 6, 10, 10, 10, 10, 10, 10, 10}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LeadFunction function = new LeadFunction(0, 2, 10, true); + TsBlock tsBlock = createTsBlockWithDefault(inputs, 2, 10); + LeadFunction function = new LeadFunction(Arrays.asList(0, 1, 2), true); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -103,8 +104,8 @@ public void testLeadFunctionIgnoreNullWithDefault() { public void testLeadFunctionNotIgnoreNullWithoutDefault() { int[] expected = {-1, 1, 2, -1, 3, 4, -1, 5, 6, -1, -1, -1, -1, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LeadFunction function = new LeadFunction(0, 2, null, false); + TsBlock tsBlock = createTsBlockWithoutDefault(inputs, 2); + LeadFunction function = new LeadFunction(Arrays.asList(0, 1), false); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -132,8 +133,8 @@ public void testLeadFunctionNotIgnoreNullWithoutDefault() { public void testLeadFunctionNotIgnoreNullWithDefault() { int[] expected = {-1, 1, 2, -1, 3, 4, -1, 5, 6, -1, -1, -1, -1, -1, 10, 10}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs); - LeadFunction function = new LeadFunction(0, 2, 10, false); + TsBlock tsBlock = createTsBlockWithDefault(inputs, 2, 10); + LeadFunction function = new LeadFunction(Arrays.asList(0, 1, 2), false); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -156,4 +157,41 @@ public void testLeadFunctionNotIgnoreNullWithDefault() { } } } + + private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int defaultValue) { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + if (input >= 0) { + columnBuilders[0].writeInt(input); + } else { + // Mimic null value + columnBuilders[0].appendNull(); + } + columnBuilders[1].writeInt(offset); + columnBuilders[2].writeInt(defaultValue); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } + + private static TsBlock createTsBlockWithoutDefault(int[] inputs, int offset) { + TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + if (input >= 0) { + columnBuilders[0].writeInt(input); + } else { + // Mimic null value + columnBuilders[0].appendNull(); + } + columnBuilders[1].writeInt(offset); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java index b350a2a7aa394..b3d3c4c5aa887 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java @@ -19,12 +19,12 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperatorTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.FunctionTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.frame.FrameInfo; import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -50,8 +50,8 @@ public class NthValueFunctionTest { public void testNthValueFunctionIgnoreNull() { int[] expected = {-1, -1, 2, -1, 3, 3, 4, 5, 5, 6, -1, -1, -1, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs, 2, 2); - NthValueFunction function = new NthValueFunction(3, 0, true); + TsBlock tsBlock = createTsBlock(inputs, 2, 2, 3); + NthValueFunction function = new NthValueFunction(Arrays.asList(0, 3), true); FrameInfo frameInfo = new FrameInfo( FrameInfo.FrameType.ROWS, @@ -86,8 +86,8 @@ public void testNthValueFunctionIgnoreNull() { public void testNthValueFunctionNotIgnoreNull() { int[] expected = {-1, -1, -1, 1, 2, -1, 3, 4, -1, 5, 6, -1, -1, -1, -1, -1}; - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs, 2, 2); - NthValueFunction function = new NthValueFunction(3, 0, false); + TsBlock tsBlock = createTsBlock(inputs, 2, 2, 3); + NthValueFunction function = new NthValueFunction(Arrays.asList(0, 3), false); FrameInfo frameInfo = new FrameInfo( FrameInfo.FrameType.ROWS, @@ -120,8 +120,8 @@ public void testNthValueFunctionNotIgnoreNull() { @Test public void testNthValueFunctionNotIgnoreNullOutOfBounds() { - TsBlock tsBlock = TableWindowOperatorTestUtils.createIntsTsBlockWithNulls(inputs, 2, 2); - NthValueFunction function = new NthValueFunction(10, 0, false); + TsBlock tsBlock = createTsBlock(inputs, 2, 2, 10); + NthValueFunction function = new NthValueFunction(Arrays.asList(0, 3), false); FrameInfo frameInfo = new FrameInfo( FrameInfo.FrameType.ROWS, @@ -147,4 +147,25 @@ public void testNthValueFunctionNotIgnoreNullOutOfBounds() { Assert.assertTrue(column.isNull(i)); } } + + private static TsBlock createTsBlock(int[] inputs, int startOffset, int endOffset, int value) { + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); + for (int input : inputs) { + if (input >= 0) { + columnBuilders[0].writeInt(input); + } else { + // Mimic null value + columnBuilders[0].appendNull(); + } + columnBuilders[1].writeInt(startOffset); + columnBuilders[2].writeInt(endOffset); + columnBuilders[3].writeInt(value); + tsBlockBuilder.declarePosition(); + } + + return tsBlockBuilder.build( + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, tsBlockBuilder.getPositionCount())); + } } From ceaaf44e1bb18b86dfd25f50d77bb3718cae37a1 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Sun, 13 Apr 2025 18:11:21 +0800 Subject: [PATCH 04/54] Temporally stash cause Im lazy. --- .../confignode1conf/iotdb-system.properties | 4 ++-- .../window/function/WindowFunctionFactory.java | 8 ++++++-- .../plan/planner/TableOperatorGenerator.java | 2 +- .../plan/relational/planner/QueryPlanner.java | 5 ++++- .../plan/relational/sql/ast/FunctionCall.java | 15 +++++++++++++++ .../plan/relational/sql/parser/AstBuilder.java | 17 ++++++++++++++++- .../db/relational/grammar/sql/RelationalSql.g4 | 8 +++++++- 7 files changed, 51 insertions(+), 8 deletions(-) diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties index b396e373f8699..508ab394aa346 100644 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties +++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties @@ -33,8 +33,8 @@ cn_metric_prometheus_reporter_port=9091 timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus -schema_replication_factor=3 -data_replication_factor=3 +schema_replication_factor=1 +data_replication_factor=1 udf_lib_dir=target/confignode1/ext/udf trigger_lib_dir=target/confignode1/ext/trigger pipe_lib_dir=target/confignode1/ext/pipe diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java index 29ff5376493fe..aabd3c2f41dba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java @@ -1,10 +1,14 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.NthValueFunction; + import java.util.List; public class WindowFunctionFactory { public static WindowFunction createBuiltinWindowFunction( - String functionName, List argumentChannels) { - return null; + String functionName, List argumentChannels, boolean ignoreNulls) { + if (functionName.equals("nth_value")) { + return new NthValueFunction(argumentChannels, ); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index b05774c58cbd2..a7d4a35f14c6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -2578,7 +2578,7 @@ public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext c } else if (functionKind == FunctionKind.WINDOW) { String functionName = function.getResolvedFunction().getSignature().getName(); windowFunction = - WindowFunctionFactory.createBuiltinWindowFunction(functionName, argumentChannels); + WindowFunctionFactory.createBuiltinWindowFunction(functionName, argumentChannels, function.isIgnoreNulls()); } else { throw new UnsupportedOperationException("Unsupported function kind: " + functionKind); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index ff73b876da40a..7aecb02ee215f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -475,6 +475,9 @@ private PlanBuilder planWindow( Symbol newSymbol = symbolAllocator.newSymbol(windowFunction, analysis.getType(windowFunction)); + FunctionCall.NullTreatment nullTreatment = windowFunction.getNullTreatment() + .orElse(FunctionCall.NullTreatment.RESPECT); + WindowNode.Function function = new WindowNode.Function( analysis.getResolvedFunction(windowFunction), @@ -483,7 +486,7 @@ private PlanBuilder planWindow( .collect(toImmutableList()), frame, // TODO: remove ignore null - false); + nullTreatment == FunctionCall.NullTreatment.IGNORE); functions.put(newSymbol, function); mappings.put(scopeAwareKey(windowFunction, analysis, subPlan.getScope()), newSymbol); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index e522e45709bcd..c5626d77e6690 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -37,11 +37,13 @@ public class FunctionCall extends Expression { private final Optional window; private final boolean distinct; private final List arguments; + private final Optional nullTreatment; public FunctionCall(QualifiedName name, List arguments) { super(null); this.name = requireNonNull(name, "name is null"); this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); this.distinct = false; this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -50,6 +52,7 @@ public FunctionCall(QualifiedName name, boolean distinct, List argum super(null); this.name = requireNonNull(name, "name is null"); this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); this.distinct = distinct; this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -63,6 +66,7 @@ public FunctionCall( super(requireNonNull(location, "location is null")); this.name = requireNonNull(name, "name is null"); this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); this.distinct = distinct; this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -71,12 +75,14 @@ public FunctionCall( NodeLocation location, QualifiedName name, Optional window, + Optional nullTreatment, boolean distinct, List arguments) { super(requireNonNull(location, "location is null")); this.name = name; this.window = window; + this.nullTreatment = nullTreatment; this.distinct = distinct; this.arguments = arguments; } @@ -97,6 +103,10 @@ public Optional getWindow() { return window; } + public Optional getNullTreatment() { + return nullTreatment; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitFunctionCall(this, context); @@ -128,6 +138,10 @@ public int hashCode() { return Objects.hash(name, distinct, arguments); } + public enum NullTreatment { + IGNORE, RESPECT + } + @Override public boolean shallowEquals(Node other) { if (!sameClass(this, other)) { @@ -167,5 +181,6 @@ public FunctionCall(ByteBuffer byteBuffer) { // TODO: serialize window this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 5cc2a784b2726..f6f4cadfe3dd0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2462,12 +2462,15 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { boolean distinct = isDistinct(ctx.setQuantifier()); + RelationalSqlParser.NullTreatmentContext nullTreatment = ctx.nullTreatment(); + if (name.toString().equalsIgnoreCase("if")) { check( ctx.expression().size() == 2 || ctx.expression().size() == 3, "Invalid number of arguments for 'if' function", ctx); check(!distinct, "DISTINCT not valid for 'if' function", ctx); + check(nullTreatment == null, "Null treatment clause not valid for 'if' function", ctx); Expression elseExpression = null; if (ctx.expression().size() == 3) { @@ -2484,6 +2487,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { if (name.toString().equalsIgnoreCase("nullif")) { check(ctx.expression().size() == 2, "Invalid number of arguments for 'nullif' function", ctx); check(!distinct, "DISTINCT not valid for 'nullif' function", ctx); + check(nullTreatment == null, "Null treatment clause not valid for 'nullif' function", ctx); return new NullIfExpression( getLocation(ctx), @@ -2497,10 +2501,21 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { "The 'coalesce' function must have at least two arguments", ctx); check(!distinct, "DISTINCT not valid for 'coalesce' function", ctx); + check(nullTreatment == null, "Null treatment clause not valid for 'coalesce' function", ctx); return new CoalesceExpression(getLocation(ctx), visit(ctx.expression(), Expression.class)); } + Optional nulls = Optional.empty(); + if (nullTreatment != null) { + if (nullTreatment.IGNORE() != null) { + nulls = Optional.of(FunctionCall.NullTreatment.IGNORE); + } + else if (nullTreatment.RESPECT() != null) { + nulls = Optional.of(FunctionCall.NullTreatment.RESPECT); + } + } + List arguments = visit(ctx.expression(), Expression.class); if (ctx.label != null) { arguments = @@ -2542,7 +2557,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { } } - return new FunctionCall(getLocation(ctx), name, window, distinct, arguments); + return new FunctionCall(getLocation(ctx), name, window, nulls, distinct, arguments); } @Override diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 0674057d832d1..587882907f45a 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -918,7 +918,8 @@ primaryExpression | '(' expression (',' expression)+ ')' #rowConstructor | ROW '(' expression (',' expression)* ')' #rowConstructor | qualifiedName '(' (label=identifier '.')? ASTERISK ')' over? #functionCall - | qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' over? #functionCall + | qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' + (nullTreatment? over)? #functionCall | '(' query ')' #subqueryExpression // This is an extension to ANSI SQL, which considers EXISTS to be a | EXISTS '(' query ')' #exists @@ -991,6 +992,11 @@ trimsSpecification | BOTH ; +nullTreatment + : IGNORE NULLS + | RESPECT NULLS + ; + string : STRING #basicStringLiteral | UNICODE_STRING (UESCAPE STRING)? #unicodeStringLiteral From 93990ab12f30a1ec6057e9c5d8fb77b4bd4cefd0 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 30 Apr 2025 00:43:34 +0800 Subject: [PATCH 05/54] Prepare for draft PR. --- .../function/WindowFunctionFactory.java | 38 +++++- .../window/function/value/LagFunction.java | 8 +- .../window/function/value/LeadFunction.java | 8 +- .../process/window/utils/RowComparator.java | 3 + .../plan/planner/TableOperatorGenerator.java | 36 +++--- .../analyzer/ExpressionAnalyzer.java | 114 ++++++++++-------- .../metadata/FunctionNullability.java | 5 + .../plan/relational/metadata/Metadata.java | 6 + .../metadata/TableMetadataImpl.java | 52 ++++++++ .../plan/relational/planner/QueryPlanner.java | 4 +- .../plan/relational/sql/ast/FunctionCall.java | 3 +- .../relational/sql/parser/AstBuilder.java | 3 +- .../iotdb/db/utils/constant/SqlConstant.java | 14 +++ .../function/rank/NTileFunctionTest.java | 3 +- .../function/value/LagFunctionTest.java | 9 +- .../function/value/LeadFunctionTest.java | 7 +- .../function/value/NthValueFunctionTest.java | 3 +- .../TableBuiltinWindowFunction.java | 41 +++++++ 18 files changed, 270 insertions(+), 87 deletions(-) create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java index aabd3c2f41dba..652a674e7152f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java @@ -1,5 +1,15 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.CumeDistFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.DenseRankFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.NTileFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.PercentRankFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.RankFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.RowNumberFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.FirstValueFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.LagFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.LastValueFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.LeadFunction; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value.NthValueFunction; import java.util.List; @@ -7,8 +17,32 @@ public class WindowFunctionFactory { public static WindowFunction createBuiltinWindowFunction( String functionName, List argumentChannels, boolean ignoreNulls) { - if (functionName.equals("nth_value")) { - return new NthValueFunction(argumentChannels, ); + switch (functionName) { + case "nth_value": + return new NthValueFunction(argumentChannels, ignoreNulls); + case "first_value": + return new FirstValueFunction(argumentChannels.get(0), ignoreNulls); + case "last_value": + return new LastValueFunction(argumentChannels.get(0), ignoreNulls); + case "lead": + return new LeadFunction(argumentChannels, ignoreNulls); + case "lag": + return new LagFunction(argumentChannels, ignoreNulls); + case "rank": + return new RankFunction(); + case "dense_rank": + return new DenseRankFunction(); + case "row_number": + return new RowNumberFunction(); + case "percent_rank": + return new PercentRankFunction(); + case "cume_dist": + return new CumeDistFunction(); + case "ntile": + return new NTileFunction(argumentChannels.get(0)); + default: + throw new UnsupportedOperationException( + "Unsupported built-in window function name: " + functionName); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index 63b99016873b6..05178d41ec32d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -48,7 +48,7 @@ public void transform( return; } - int offset = offsetChannel >= 0? partition.getInt(offsetChannel, index) : 1; + int offset = offsetChannel >= 0 ? partition.getInt(offsetChannel, index) : 1; int pos; if (ignoreNull) { @@ -81,7 +81,8 @@ public void transform( } } - private void writeDefaultValue(Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { + private void writeDefaultValue( + Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { TSDataType dataType = builder.getDataType(); switch (dataType) { case INT32: @@ -106,7 +107,8 @@ private void writeDefaultValue(Partition partition, int defaultValChannel, int i builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: - throw new UnSupportedDataTypeException("Unsupported default value's data type in Lag: " + dataType); + throw new UnSupportedDataTypeException( + "Unsupported default value's data type in Lag: " + dataType); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java index 2ae5c756aa737..ecbc3b62256dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java @@ -49,7 +49,7 @@ public void transform( } int length = partition.getPositionCount(); - int offset = offsetChannel >= 0? partition.getInt(offsetChannel, index) : 1; + int offset = offsetChannel >= 0 ? partition.getInt(offsetChannel, index) : 1; int pos; if (ignoreNull) { @@ -82,7 +82,8 @@ public void transform( } } - private void writeDefaultValue(Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { + private void writeDefaultValue( + Partition partition, int defaultValChannel, int index, ColumnBuilder builder) { TSDataType dataType = builder.getDataType(); switch (dataType) { case INT32: @@ -107,7 +108,8 @@ private void writeDefaultValue(Partition partition, int defaultValChannel, int i builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: - throw new UnSupportedDataTypeException("Unsupported default value's data type in Lag: " + dataType); + throw new UnSupportedDataTypeException( + "Unsupported default value's data type in Lag: " + dataType); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index 5823a83875be8..0c7401d5c9ca6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -66,6 +66,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse } break; case INT64: + case TIMESTAMP: long long1 = column.getLong(offset1); long long2 = column.getLong(offset2); if (long1 != long2) { @@ -134,6 +135,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o } break; case INT64: + case TIMESTAMP: long long1 = column.getLong(offset1); long long2 = column.getLong(offset2); if (long1 != long2) { @@ -190,6 +192,7 @@ public boolean equal(List columns1, int offset1, List columns2, } break; case INT64: + case TIMESTAMP: long long1 = column1.getLong(offset1); long long2 = column2.getLong(offset2); if (long1 != long2) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index a7d4a35f14c6b..d6f0339a1d6f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -248,7 +248,10 @@ import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.getLinearFill; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.getPreviousFill; import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions.updateFilterUsingTTL; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_LAST; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.DESC_NULLS_FIRST; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.DESC_NULLS_LAST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.db.utils.constant.SqlConstant.AVG; @@ -2481,11 +2484,11 @@ public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext c // Sort channel List sortChannels = ImmutableList.of(); - // List sortOrder = ImmutableList.of(); + List sortOrder = ImmutableList.of(); if (node.getSpecification().getOrderingScheme().isPresent()) { OrderingScheme orderingScheme = node.getSpecification().getOrderingScheme().get(); sortChannels = getChannelsForSymbols(orderingScheme.getOrderBy(), childLayout); - // sortOrder = orderingScheme.getOrderingList(); + sortOrder = orderingScheme.getOrderingList(); } // Output channel @@ -2529,19 +2532,19 @@ public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext c Optional ordering = Optional.empty(); if (node.getSpecification().getOrderingScheme().isPresent()) { sortKeyChannel = Optional.of(sortChannels.get(0)); - // if (sortOrder.get(0).isNullsFirst()) { - // if (sortOrder.get(0).isAscending()) { - // ordering = Optional.of(ASC_NULLS_FIRST); - // } else { - // ordering = Optional.of(DESC_NULLS_FIRST); - // } - // } else { - // if (sortOrder.get(0).isAscending()) { - // ordering = Optional.of(ASC_NULLS_LAST); - // } else { - // ordering = Optional.of(DESC_NULLS_LAST); - // } - // } + if (sortOrder.get(0).isNullsFirst()) { + if (sortOrder.get(0).isAscending()) { + ordering = Optional.of(ASC_NULLS_FIRST); + } else { + ordering = Optional.of(DESC_NULLS_FIRST); + } + } else { + if (sortOrder.get(0).isAscending()) { + ordering = Optional.of(ASC_NULLS_LAST); + } else { + ordering = Optional.of(DESC_NULLS_LAST); + } + } } FrameInfo frameInfo = new FrameInfo( @@ -2578,7 +2581,8 @@ public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext c } else if (functionKind == FunctionKind.WINDOW) { String functionName = function.getResolvedFunction().getSignature().getName(); windowFunction = - WindowFunctionFactory.createBuiltinWindowFunction(functionName, argumentChannels, function.isIgnoreNulls()); + WindowFunctionFactory.createBuiltinWindowFunction( + functionName, argumentChannels, function.isIgnoreNulls()); } else { throw new UnsupportedOperationException("Unsupported function kind: " + functionKind); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 81c2e56b72df8..db2a3d062e897 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -93,7 +93,6 @@ import com.google.common.collect.Multimap; import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.UnknownType; import javax.annotation.Nullable; @@ -452,8 +451,7 @@ private Type handleResolvedField( if (!resolvedField.isLocal() && context.getContext().getCorrelationSupport() != CorrelationSupport.ALLOWED) { throw new SemanticException( - String.format( - "Reference to column '%s' from outer scope not allowed in this context", node)); + format("Reference to column '%s' from outer scope not allowed in this context", node)); } FieldId fieldId = FieldId.from(resolvedField); @@ -509,8 +507,7 @@ protected Type visitDereferenceExpression( Type baseType = process(node.getBase(), context); if (!(baseType instanceof RowType)) { - throw new SemanticException( - String.format("Expression %s is not of type ROW", node.getBase())); + throw new SemanticException(format("Expression %s is not of type ROW", node.getBase())); } RowType rowType = (RowType) baseType; @@ -524,8 +521,7 @@ protected Type visitDereferenceExpression( for (RowType.Field rowField : rowType.getFields()) { if (fieldName.equalsIgnoreCase(rowField.getName().orElse(null))) { if (foundFieldName) { - throw new SemanticException( - String.format("Ambiguous row field reference: %s", fieldName)); + throw new SemanticException(format("Ambiguous row field reference: %s", fieldName)); } foundFieldName = true; rowFieldType = rowField.getType(); @@ -606,7 +602,7 @@ protected Type visitNullIfExpression( if (!firstType.equals(secondType)) { throw new SemanticException( - String.format("Types are not comparable with NULLIF: %s vs %s", firstType, secondType)); + format("Types are not comparable with NULLIF: %s vs %s", firstType, secondType)); } return setExpressionType(node, firstType); @@ -689,7 +685,7 @@ private void coerceCaseOperandToToSingleType( if (!operandType.equals(whenOperandType)) { throw new SemanticException( - String.format( + format( "CASE operand type does not match WHEN clause operand type: %s vs %s", operandType, whenOperandType)); } @@ -700,7 +696,7 @@ private void coerceCaseOperandToToSingleType( if (!whenOperandType.equals(operandType)) { // Expression whenOperand = whenClauses.get(i).getOperand(); throw new SemanticException( - String.format( + format( "CASE operand type does not match WHEN clause operand type: %s vs %s", operandType, whenOperandType)); // addOrReplaceExpressionCoercion(whenOperand, whenOperandType, operandType); @@ -739,7 +735,7 @@ protected Type visitArithmeticUnary( // that types can chose to implement, or piggyback on the existence of the negation // operator throw new SemanticException( - String.format("Unary '+' operator cannot by applied to %s type", type)); + format("Unary '+' operator cannot by applied to %s type", type)); } return setExpressionType(node, type); case MINUS: @@ -766,7 +762,7 @@ protected Type visitLikePredicate( Type valueType = process(node.getValue(), context); if (!isCharType(valueType)) { throw new SemanticException( - String.format( + format( "Left side of LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", valueType)); } @@ -774,7 +770,7 @@ protected Type visitLikePredicate( Type patternType = process(node.getPattern(), context); if (!isCharType(patternType)) { throw new SemanticException( - String.format( + format( "Pattern for LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", patternType)); } @@ -783,7 +779,7 @@ protected Type visitLikePredicate( Type escapeType = process(escape, context); if (!isCharType(escapeType)) { throw new SemanticException( - String.format( + format( "Escape for LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", escapeType)); } @@ -876,28 +872,53 @@ protected Type visitFunctionCall( if (node.getArguments().size() > 127) { throw new SemanticException( - String.format("Too many arguments for function call %s()", functionName)); + format("Too many arguments for function call %s()", functionName)); } for (Type argumentType : argumentTypes) { if (node.isDistinct() && !argumentType.isComparable()) { throw new SemanticException( - String.format( + format( "DISTINCT can only be applied to comparable types (actual: %s)", argumentType)); } } Type type = metadata.getFunctionReturnType(functionName, argumentTypes); + FunctionKind functionKind = FunctionKind.SCALAR; + if (isAggregation) { + functionKind = FunctionKind.AGGREGATE; + } else { + boolean isWindow = metadata.isWindowFunction(session, functionName, accessControl); + if (isWindow) { + functionKind = FunctionKind.WINDOW; + } + } + FunctionNullability functionNullability = null; + switch (functionKind) { + case AGGREGATE: + functionNullability = + FunctionNullability.getAggregationFunctionNullability(argumentTypes.size()); + break; + case SCALAR: + functionNullability = + FunctionNullability.getScalarFunctionNullability(argumentTypes.size()); + break; + case WINDOW: + functionNullability = + FunctionNullability.getWindowFunctionNullability(argumentTypes.size()); + break; + default: + // ignore + } + // now we only support scalar or agg functions ResolvedFunction resolvedFunction = new ResolvedFunction( new BoundSignature(functionName.toLowerCase(Locale.ENGLISH), type, argumentTypes), new FunctionId("noop"), - isAggregation ? FunctionKind.AGGREGATE : FunctionKind.SCALAR, + functionKind, true, - isAggregation - ? FunctionNullability.getAggregationFunctionNullability(argumentTypes.size()) - : FunctionNullability.getScalarFunctionNullability(argumentTypes.size())); + functionNullability); resolvedFunctions.put(NodeRef.of(node), resolvedFunction); return setExpressionType(node, type); } @@ -913,7 +934,7 @@ public List getCallArgumentTypes( String label = label((Identifier) allRowsDereference.getBase()); if (!context.getContext().getLabels().contains(label)) { throw new SemanticException( - String.format("%s is not a primary pattern variable or subset name", label)); + format("%s is not a primary pattern variable or subset name", label)); } labelDereferences.put(NodeRef.of(allRowsDereference), new LabelPrefixedReference(label)); } else { @@ -962,7 +983,7 @@ protected Type visitParameter(Parameter node, StackableAstVisitorContext= parameters.size()) { throw new SemanticException( - String.format( + format( "Invalid parameter index %s, max value is %s", node.getId(), parameters.size() - 1)); } @@ -985,12 +1006,12 @@ protected Type visitBetweenPredicate( if (!isTwoTypeComparable(Arrays.asList(valueType, minType)) || !isTwoTypeComparable(Arrays.asList(valueType, maxType))) { throw new SemanticException( - String.format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); + format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); } if (!valueType.isOrderable()) { throw new SemanticException( - String.format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); + format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); } return setExpressionType(node, BOOLEAN); @@ -1003,18 +1024,16 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { try { type = metadata.getType(toTypeSignature(node.getType())); } catch (TypeNotFoundException e) { - throw new SemanticException(String.format("Unknown type: %s", node.getType())); + throw new SemanticException(format("Unknown type: %s", node.getType())); } - if (type.equals(UnknownType.UNKNOWN)) { + if (type.equals(UNKNOWN)) { throw new SemanticException("UNKNOWN is not a valid type"); } Type value = process(node.getExpression(), context); - if (!value.equals(UnknownType.UNKNOWN) - && !node.isTypeOnly() - && (!metadata.canCoerce(value, type))) { - throw new SemanticException(String.format("Cannot cast %s to %s", value, type)); + if (!value.equals(UNKNOWN) && !node.isTypeOnly() && (!metadata.canCoerce(value, type))) { + throw new SemanticException(format("Cannot cast %s to %s", value, type)); } return setExpressionType(node, type); @@ -1155,7 +1174,7 @@ private Type analyzeSubquery( private void analyzeWindow( Analysis.ResolvedWindow window, - StackableAstVisitor.StackableAstVisitorContext context, + StackableAstVisitorContext context, Node originalNode) { // check no nested window functions ImmutableList.Builder childNodes = ImmutableList.builder(); @@ -1175,7 +1194,7 @@ private void analyzeWindow( Type type = getExpressionType(expression); if (!type.isComparable()) { throw new SemanticException( - String.format( + format( "%s is not comparable, and therefore cannot be used in window function PARTITION BY", type)); } @@ -1188,7 +1207,7 @@ private void analyzeWindow( Type type = getExpressionType(sortItem.getSortKey()); if (!type.isOrderable()) { throw new SemanticException( - String.format( + format( "%s is not orderable, and therefore cannot be used in window function ORDER BY", type)); } @@ -1228,7 +1247,7 @@ private void analyzeWindow( Type type = process(startValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - String.format( + format( "Window frame ROWS start value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1238,7 +1257,7 @@ private void analyzeWindow( Type type = process(endValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - String.format( + format( "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1264,7 +1283,7 @@ private void analyzeWindow( Type type = process(startValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - String.format( + format( "Window frame GROUPS start value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1278,7 +1297,7 @@ private void analyzeWindow( Type type = process(endValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - String.format( + format( "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1316,7 +1335,7 @@ private void analyzeFrameRangeOffset( } if (!isNumericType(sortKeyType)) { throw new SemanticException( - String.format( + format( "Window frame of type RANGE PRECEDING or FOLLOWING requires that sort item type be numeric, datetime or interval (actual: %s)", sortKeyType)); } @@ -1326,7 +1345,7 @@ private void analyzeFrameRangeOffset( if (isNumericType(sortKeyType)) { if (!isNumericType(offsetValueType)) { throw new SemanticException( - String.format( + format( "Window frame RANGE value type (%s) not compatible with sort item type (%s)", offsetValueType, sortKeyType)); } @@ -1432,7 +1451,7 @@ protected Type visitQuantifiedComparisonExpression( case GREATER_THAN_OR_EQUAL: if (!comparisonType.isOrderable()) { throw new SemanticException( - String.format( + format( "Type [%s] must be orderable in order to be used in quantified comparison", comparisonType)); } @@ -1441,7 +1460,7 @@ protected Type visitQuantifiedComparisonExpression( case NOT_EQUAL: if (!comparisonType.isComparable()) { throw new SemanticException( - String.format( + format( "Type [%s] must be comparable in order to be used in quantified comparison", comparisonType)); } @@ -1463,14 +1482,12 @@ public Type visitFieldReference( @Override protected Type visitExpression(Expression node, StackableAstVisitorContext context) { - throw new SemanticException( - String.format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException(format("not yet implemented: %s", node.getClass().getName())); } @Override protected Type visitNode(Node node, StackableAstVisitorContext context) { - throw new SemanticException( - String.format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException(format("not yet implemented: %s", node.getClass().getName())); } private Type getOperator( @@ -1498,8 +1515,7 @@ private void coerceType( if (!actualType.equals(expectedType)) { // if (!typeCoercion.canCoerce(actualType, expectedType)) { throw new SemanticException( - String.format( - "%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); + format("%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); // } // addOrReplaceExpressionCoercion(expression, actualType, expectedType); } @@ -1530,7 +1546,7 @@ private Type coerceToSingleType( } if (!firstType.equals(secondType)) { - throw new SemanticException(String.format("%s: %s vs %s", message, firstType, secondType)); + throw new SemanticException(format("%s: %s vs %s", message, firstType, secondType)); } return firstType; @@ -1559,7 +1575,7 @@ private Type coerceToSingleType( } else { if (!isTwoTypeComparable(Arrays.asList(superType, type))) { throw new SemanticException( - String.format( + format( "%s must be the same type or coercible to a common type. Cannot find common type between %s and %s, all types (without duplicates): %s", description, superType, type, typeExpressions.keySet())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java index d838d335b8d53..7dabe159f4931 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java @@ -51,6 +51,11 @@ public static FunctionNullability getScalarFunctionNullability(int argumentsNumb return new FunctionNullability(true, Collections.nCopies(argumentsNumber, true)); } + // TODO modify for each window function + public static FunctionNullability getWindowFunctionNullability(int argumentsNumber) { + return new FunctionNullability(true, Collections.nCopies(argumentsNumber, true)); + } + public boolean isReturnNullable() { return returnNullable; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index 1234c1cf14584..6e6807ca6d72b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.DataPartitionQueryParam; import org.apache.iotdb.commons.partition.SchemaPartition; +import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinWindowFunction; import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -61,6 +62,11 @@ Type getOperatorReturnType( boolean isAggregationFunction( final SessionInfo session, final String functionName, final AccessControl accessControl); + default boolean isWindowFunction( + final SessionInfo session, final String functionName, final AccessControl accessControl) { + return TableBuiltinWindowFunction.getBuiltInWindowFunctionName().contains(functionName); + } + Type getType(final TypeSignature signature) throws TypeNotFoundException; boolean canCoerce(final Type from, final Type to); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 8364ea489b368..bfa77b7550fdd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -664,6 +664,58 @@ && isIntegerNumber(argumentTypes.get(2)))) { // ignore } + // builtin window function + // check argument type + switch (functionName.toLowerCase(Locale.ENGLISH)) { + case SqlConstant.NTILE: + if (argumentTypes.size() != 1) { + throw new SemanticException( + String.format("Window function [%s] should only have one argument", functionName)); + } + break; + case SqlConstant.NTH_VALUE: + if (argumentTypes.size() != 2 || isIntegerNumber(argumentTypes.get(1))) { + throw new SemanticException( + "Window function [nth] should only have two argument, and second argument must be integer type"); + } + break; + // case SqlConstant.FIRST_VALUE: + // case SqlConstant.LAST_VALUE: + case SqlConstant.LEAD: + case SqlConstant.LAG: + if (!(argumentTypes.size() >= 1 && argumentTypes.size() <= 3)) { + throw new SemanticException( + String.format( + "Window function [%s] should only have one to three argument", functionName)); + } + if (argumentTypes.size() >= 2 && !isIntegerNumber(argumentTypes.get(1))) { + throw new SemanticException( + String.format( + "Window function [%s]'s second argument must be integer type", functionName)); + } + break; + default: + // ignore + } + + // get return type + switch (functionName.toLowerCase(Locale.ENGLISH)) { + case SqlConstant.RANK: + case SqlConstant.DENSE_RANK: + case SqlConstant.ROW_NUMBER: + case SqlConstant.NTILE: + return INT64; + case SqlConstant.PERCENT_RANK: + case SqlConstant.CUME_DIST: + return DOUBLE; + case SqlConstant.NTH_VALUE: + case SqlConstant.LEAD: + case SqlConstant.LAG: + return argumentTypes.get(0); + default: + // ignore + } + // User-defined scalar function if (TableUDFUtils.isScalarFunction(functionName)) { ScalarFunction scalarFunction = TableUDFUtils.getScalarFunction(functionName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 7aecb02ee215f..ce19fd275876c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -475,8 +475,8 @@ private PlanBuilder planWindow( Symbol newSymbol = symbolAllocator.newSymbol(windowFunction, analysis.getType(windowFunction)); - FunctionCall.NullTreatment nullTreatment = windowFunction.getNullTreatment() - .orElse(FunctionCall.NullTreatment.RESPECT); + FunctionCall.NullTreatment nullTreatment = + windowFunction.getNullTreatment().orElse(FunctionCall.NullTreatment.RESPECT); WindowNode.Function function = new WindowNode.Function( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index c5626d77e6690..69cfff551c001 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -139,7 +139,8 @@ public int hashCode() { } public enum NullTreatment { - IGNORE, RESPECT + IGNORE, + RESPECT } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index f6f4cadfe3dd0..4633bfeb1a3dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -2510,8 +2510,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { if (nullTreatment != null) { if (nullTreatment.IGNORE() != null) { nulls = Optional.of(FunctionCall.NullTreatment.IGNORE); - } - else if (nullTreatment.RESPECT() != null) { + } else if (nullTreatment.RESPECT() != null) { nulls = Optional.of(FunctionCall.NullTreatment.RESPECT); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index 625af89356144..b3ac4f95b6c44 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -97,6 +97,20 @@ protected SqlConstant() { public static final String SUBSTRING_IS_STANDARD = "isStandard"; public static final String SUBSTRING_FOR = "FOR"; + // names of window functions + public static final String RANK = "rank"; + public static final String DENSE_RANK = "dense_rank"; + public static final String ROW_NUMBER = "row_number"; + public static final String PERCENT_RANK = "percent_rank"; + public static final String CUME_DIST = "cume_dist"; + public static final String NTILE = "ntile"; + // Duplicate with aggregation function + // public static final String FIRST_VALUE = "first_value"; + // public static final String LAST_VALUE = "last_value"; + public static final String NTH_VALUE = "nth_value"; + public static final String LEAD = "lead"; + public static final String LAG = "lag"; + public static String[] getSingleRootArray() { return SINGLE_ROOT_ARRAY; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java index 3b79cbae245df..3d08daeef551c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/rank/NTileFunctionTest.java @@ -127,7 +127,8 @@ public void testNTileFunctionNonUniform() { } private static TsBlock createTsBlockWithN(int[] inputs, int n) { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { columnBuilders[0].writeInt(input); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java index 831227d4116b1..2728df2acc69e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunctionTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperatorTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.FunctionTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; @@ -80,7 +79,7 @@ public void testLagFunctionIgnoreNullWithDefault() { int[] expected = {10, 10, 10, 10, 0, 1, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5}; TsBlock tsBlock = createTsBlockWithDefault(inputs, 2, 10); - LagFunction function = new LagFunction(Arrays.asList(0 ,1, 2), true); + LagFunction function = new LagFunction(Arrays.asList(0, 1, 2), true); PartitionExecutor partitionExecutor = FunctionTestUtils.createPartitionExecutor(tsBlock, inputDataTypes, function); @@ -159,7 +158,8 @@ public void testLagFunctionNotIgnoreNullWithDefault() { } private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int defaultValue) { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { if (input >= 0) { @@ -178,7 +178,8 @@ private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int de } private static TsBlock createTsBlockWithoutDefault(int[] inputs, int offset) { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { if (input >= 0) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java index c82cb837b4fb2..662be90dc226f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunctionTest.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.window.function.value; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperatorTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.FunctionTestUtils; import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.PartitionExecutor; @@ -159,7 +158,8 @@ public void testLeadFunctionNotIgnoreNullWithDefault() { } private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int defaultValue) { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { if (input >= 0) { @@ -178,7 +178,8 @@ private static TsBlock createTsBlockWithDefault(int[] inputs, int offset, int de } private static TsBlock createTsBlockWithoutDefault(int[] inputs, int offset) { - TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); + TsBlockBuilder tsBlockBuilder = + new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { if (input >= 0) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java index b3d3c4c5aa887..bc8f4f5094f44 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/NthValueFunctionTest.java @@ -150,7 +150,8 @@ public void testNthValueFunctionNotIgnoreNullOutOfBounds() { private static TsBlock createTsBlock(int[] inputs, int startOffset, int endOffset, int value) { TsBlockBuilder tsBlockBuilder = - new TsBlockBuilder(Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); + new TsBlockBuilder( + Arrays.asList(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32, TSDataType.INT32)); ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders(); for (int input : inputs) { if (input >= 0) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java new file mode 100644 index 0000000000000..1ec26a8bf1cec --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java @@ -0,0 +1,41 @@ +package org.apache.iotdb.commons.udf.builtin.relational; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +public enum TableBuiltinWindowFunction { + RANK("rank"), + DENSE_RANK("dense_rank"), + ROW_NUMBER("row_number"), + PERCENT_RANK("percent_rank"), + CUME_DIST("cume_dist"), + NTILE("ntile"), + FIRST_VALUE("first_value"), + LAST_VALUE("last_value"), + NTH_VALUE("nth_value"), + LEAD("lead"), + LAG("lag"), + ; + + private final String functionName; + + TableBuiltinWindowFunction(String functionName) { + this.functionName = functionName; + } + + public String getFunctionName() { + return functionName; + } + + private static final Set BUILT_IN_WINDOW_FUNCTION_NAME = + new HashSet<>( + Arrays.stream(TableBuiltinWindowFunction.values()) + .map(TableBuiltinWindowFunction::getFunctionName) + .collect(Collectors.toList())); + + public static Set getBuiltInWindowFunctionName() { + return BUILT_IN_WINDOW_FUNCTION_NAME; + } +} From b64d98cec45e22a1c8920a097d6b6ff3c1699c5f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 6 May 2025 16:18:17 +0800 Subject: [PATCH 06/54] Fix nth_value check cond error. --- .../plan/relational/metadata/TableMetadataImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index bfa77b7550fdd..7a2ac9183d0fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -674,9 +674,9 @@ && isIntegerNumber(argumentTypes.get(2)))) { } break; case SqlConstant.NTH_VALUE: - if (argumentTypes.size() != 2 || isIntegerNumber(argumentTypes.get(1))) { + if (argumentTypes.size() != 2 || !isIntegerNumber(argumentTypes.get(1))) { throw new SemanticException( - "Window function [nth] should only have two argument, and second argument must be integer type"); + "Window function [nth_value] should only have two argument, and second argument must be integer type"); } break; // case SqlConstant.FIRST_VALUE: From 869a293fe6c9de88c7eec65131a3b19cde81a499 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 6 May 2025 16:23:43 +0800 Subject: [PATCH 07/54] Fix rows frame indexing error. --- .../window/partition/PartitionExecutor.java | 2 +- .../process/window/partition/frame/RowsFrame.java | 14 ++++++-------- .../window/partition/frame/FrameTestUtils.java | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/PartitionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/PartitionExecutor.java index b7a4239a314b4..bf136c12391be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/PartitionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/PartitionExecutor.java @@ -111,7 +111,7 @@ public PartitionExecutor( frame = new RangeFrame(partition, frameInfo, sortedColumns, peerGroupComparator); break; case ROWS: - frame = new RowsFrame(partition, frameInfo, partitionStart, partitionEnd); + frame = new RowsFrame(partition, frameInfo); break; case GROUPS: frame = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java index ef9fa4b7baf9d..b41acc803532a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java @@ -28,16 +28,14 @@ public class RowsFrame implements Frame { private final Partition partition; private final FrameInfo frameInfo; - private final int partitionStart; private final int partitionSize; - public RowsFrame(Partition partition, FrameInfo frameInfo, int partitionStart, int partitionEnd) { + public RowsFrame(Partition partition, FrameInfo frameInfo) { checkArgument(frameInfo.getFrameType() == FrameInfo.FrameType.ROWS); this.partition = partition; this.frameInfo = frameInfo; - this.partitionStart = partitionStart; - this.partitionSize = partitionEnd - partitionStart; + this.partitionSize = partition.getPositionCount(); } @Override @@ -51,7 +49,7 @@ public Range getRange( break; case PRECEDING: offset = - (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition + partitionStart); + (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); frameStart = currentPosition - offset; break; case CURRENT_ROW: @@ -59,7 +57,7 @@ public Range getRange( break; case FOLLOWING: offset = - (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition + partitionStart); + (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); frameStart = currentPosition + offset; break; default: @@ -70,14 +68,14 @@ public Range getRange( int frameEnd; switch (frameInfo.getEndType()) { case PRECEDING: - offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition + partitionStart); + offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition); frameEnd = currentPosition - offset; break; case CURRENT_ROW: frameEnd = currentPosition; break; case FOLLOWING: - offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition + partitionStart); + offset = (int) getOffset(frameInfo.getEndOffsetChannel(), currentPosition); frameEnd = currentPosition + offset; break; case UNBOUNDED_FOLLOWING: diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameTestUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameTestUtils.java index d8a55d45aa29f..b8a808ae13588 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameTestUtils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/FrameTestUtils.java @@ -118,7 +118,7 @@ private Frame createFrame(FrameInfo frameInfo) { frame = new RangeFrame(partition, frameInfo, sortedColumns, peerGroupComparator); break; case ROWS: - frame = new RowsFrame(partition, frameInfo, partitionStart, partitionEnd); + frame = new RowsFrame(partition, frameInfo); break; case GROUPS: frame = From ea542c8157c68b83cb12e6c93d0f3e1185642e6f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 6 May 2025 22:57:20 +0800 Subject: [PATCH 08/54] Support expression in frame clause. --- .../execution/operator/process/FilterAndProjectOperator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java index c759c3a0fdfb7..5429cf23fc0e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java @@ -155,6 +155,7 @@ private TsBlock getFilterTsBlock(TsBlock input) { if (!hasNonMappableUDF) { // get result of calculated common sub expressions for (ColumnTransformer columnTransformer : commonTransformerList) { + columnTransformer.tryEvaluate(); resultColumns.add(columnTransformer.getColumn()); } } From 6b974abff332197f6ef2b0a53934c49448c9dfb9 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:20:07 +0800 Subject: [PATCH 09/54] Remove useless sort key comparator. --- .../plan/planner/TableOperatorGenerator.java | 11 +- .../analyzer/ExpressionAnalyzer.java | 123 +++++------------- 2 files changed, 36 insertions(+), 98 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index d6f0339a1d6f3..62c7d4bd66373 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -2514,20 +2514,11 @@ public Operator visitWindowFunction(WindowNode node, LocalExecutionPlanContext c if (frame.getStartValue().isPresent()) { frameStartChannel = Optional.ofNullable(childLayout.get(frame.getStartValue().get())); } - // Optional sortKeyChannelForStartComparison = Optional.empty(); - // if (frame.getSortKeyCoercedForFrameStartComparison().isPresent()) { - // sortKeyChannelForStartComparison = - // Optional.ofNullable(childLayout.get(frame.getSortKeyCoercedForFrameStartComparison().get())); - // } Optional frameEndChannel = Optional.empty(); if (frame.getEndValue().isPresent()) { frameEndChannel = Optional.ofNullable(childLayout.get(frame.getEndValue().get())); } - // Optional sortKeyChannelForEndComparison = Optional.empty(); - // if (frame.getSortKeyCoercedForFrameEndComparison().isPresent()) { - // sortKeyChannelForEndComparison = - // Optional.ofNullable(childLayout.get(frame.getSortKeyCoercedForFrameEndComparison().get())); - // } + Optional sortKeyChannel = Optional.empty(); Optional ordering = Optional.empty(); if (node.getSpecification().getOrderingScheme().isPresent()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index db2a3d062e897..e9b97e50ae64d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -1,7 +1,7 @@ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information + * distributed with this work for additional inString.formation * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance @@ -112,7 +112,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.getOnlyElement; -import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; import static java.util.Objects.requireNonNull; @@ -451,7 +450,7 @@ private Type handleResolvedField( if (!resolvedField.isLocal() && context.getContext().getCorrelationSupport() != CorrelationSupport.ALLOWED) { throw new SemanticException( - format("Reference to column '%s' from outer scope not allowed in this context", node)); + String.format("Reference to column '%s' from outer scope not allowed in this context", node)); } FieldId fieldId = FieldId.from(resolvedField); @@ -507,7 +506,7 @@ protected Type visitDereferenceExpression( Type baseType = process(node.getBase(), context); if (!(baseType instanceof RowType)) { - throw new SemanticException(format("Expression %s is not of type ROW", node.getBase())); + throw new SemanticException(String.format("Expression %s is not of type ROW", node.getBase())); } RowType rowType = (RowType) baseType; @@ -521,7 +520,7 @@ protected Type visitDereferenceExpression( for (RowType.Field rowField : rowType.getFields()) { if (fieldName.equalsIgnoreCase(rowField.getName().orElse(null))) { if (foundFieldName) { - throw new SemanticException(format("Ambiguous row field reference: %s", fieldName)); + throw new SemanticException(String.format("Ambiguous row field reference: %s", fieldName)); } foundFieldName = true; rowFieldType = rowField.getType(); @@ -602,7 +601,7 @@ protected Type visitNullIfExpression( if (!firstType.equals(secondType)) { throw new SemanticException( - format("Types are not comparable with NULLIF: %s vs %s", firstType, secondType)); + String.format("Types are not comparable with NULLIF: %s vs %s", firstType, secondType)); } return setExpressionType(node, firstType); @@ -685,7 +684,7 @@ private void coerceCaseOperandToToSingleType( if (!operandType.equals(whenOperandType)) { throw new SemanticException( - format( + String.format( "CASE operand type does not match WHEN clause operand type: %s vs %s", operandType, whenOperandType)); } @@ -696,7 +695,7 @@ private void coerceCaseOperandToToSingleType( if (!whenOperandType.equals(operandType)) { // Expression whenOperand = whenClauses.get(i).getOperand(); throw new SemanticException( - format( + String.format( "CASE operand type does not match WHEN clause operand type: %s vs %s", operandType, whenOperandType)); // addOrReplaceExpressionCoercion(whenOperand, whenOperandType, operandType); @@ -735,7 +734,7 @@ protected Type visitArithmeticUnary( // that types can chose to implement, or piggyback on the existence of the negation // operator throw new SemanticException( - format("Unary '+' operator cannot by applied to %s type", type)); + String.format("Unary '+' operator cannot by applied to %s type", type)); } return setExpressionType(node, type); case MINUS: @@ -762,7 +761,7 @@ protected Type visitLikePredicate( Type valueType = process(node.getValue(), context); if (!isCharType(valueType)) { throw new SemanticException( - format( + String.format( "Left side of LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", valueType)); } @@ -770,7 +769,7 @@ protected Type visitLikePredicate( Type patternType = process(node.getPattern(), context); if (!isCharType(patternType)) { throw new SemanticException( - format( + String.format( "Pattern for LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", patternType)); } @@ -779,7 +778,7 @@ protected Type visitLikePredicate( Type escapeType = process(escape, context); if (!isCharType(escapeType)) { throw new SemanticException( - format( + String.format( "Escape for LIKE expression must evaluate to TEXT or STRING Type (actual: %s)", escapeType)); } @@ -872,13 +871,13 @@ protected Type visitFunctionCall( if (node.getArguments().size() > 127) { throw new SemanticException( - format("Too many arguments for function call %s()", functionName)); + String.format("Too many arguments for function call %s()", functionName)); } for (Type argumentType : argumentTypes) { if (node.isDistinct() && !argumentType.isComparable()) { throw new SemanticException( - format( + String.format( "DISTINCT can only be applied to comparable types (actual: %s)", argumentType)); } } @@ -934,7 +933,7 @@ public List getCallArgumentTypes( String label = label((Identifier) allRowsDereference.getBase()); if (!context.getContext().getLabels().contains(label)) { throw new SemanticException( - format("%s is not a primary pattern variable or subset name", label)); + String.format("%s is not a primary pattern variable or subset name", label)); } labelDereferences.put(NodeRef.of(allRowsDereference), new LabelPrefixedReference(label)); } else { @@ -983,7 +982,7 @@ protected Type visitParameter(Parameter node, StackableAstVisitorContext= parameters.size()) { throw new SemanticException( - format( + String.format( "Invalid parameter index %s, max value is %s", node.getId(), parameters.size() - 1)); } @@ -1006,12 +1005,12 @@ protected Type visitBetweenPredicate( if (!isTwoTypeComparable(Arrays.asList(valueType, minType)) || !isTwoTypeComparable(Arrays.asList(valueType, maxType))) { throw new SemanticException( - format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); + String.format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); } if (!valueType.isOrderable()) { throw new SemanticException( - format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); + String.format("Cannot check if %s is BETWEEN %s and %s", valueType, minType, maxType)); } return setExpressionType(node, BOOLEAN); @@ -1024,7 +1023,7 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { try { type = metadata.getType(toTypeSignature(node.getType())); } catch (TypeNotFoundException e) { - throw new SemanticException(format("Unknown type: %s", node.getType())); + throw new SemanticException(String.format("Unknown type: %s", node.getType())); } if (type.equals(UNKNOWN)) { @@ -1033,7 +1032,7 @@ public Type visitCast(Cast node, StackableAstVisitorContext context) { Type value = process(node.getExpression(), context); if (!value.equals(UNKNOWN) && !node.isTypeOnly() && (!metadata.canCoerce(value, type))) { - throw new SemanticException(format("Cannot cast %s to %s", value, type)); + throw new SemanticException(String.format("Cannot cast %s to %s", value, type)); } return setExpressionType(node, type); @@ -1194,7 +1193,7 @@ private void analyzeWindow( Type type = getExpressionType(expression); if (!type.isComparable()) { throw new SemanticException( - format( + String.format( "%s is not comparable, and therefore cannot be used in window function PARTITION BY", type)); } @@ -1207,7 +1206,7 @@ private void analyzeWindow( Type type = getExpressionType(sortItem.getSortKey()); if (!type.isOrderable()) { throw new SemanticException( - format( + String.format( "%s is not orderable, and therefore cannot be used in window function ORDER BY", type)); } @@ -1247,7 +1246,7 @@ private void analyzeWindow( Type type = process(startValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - format( + String.format( "Window frame ROWS start value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1257,7 +1256,7 @@ private void analyzeWindow( Type type = process(endValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - format( + String.format( "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1283,7 +1282,7 @@ private void analyzeWindow( Type type = process(startValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - format( + String.format( "Window frame GROUPS start value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1297,7 +1296,7 @@ private void analyzeWindow( Type type = process(endValue, context); if (!isExactNumericWithScaleZero(type)) { throw new SemanticException( - format( + String.format( "Window frame ROWS end value type must be exact numeric type with scale 0 (actual %s)", type)); } @@ -1335,7 +1334,7 @@ private void analyzeFrameRangeOffset( } if (!isNumericType(sortKeyType)) { throw new SemanticException( - format( + String.format( "Window frame of type RANGE PRECEDING or FOLLOWING requires that sort item type be numeric, datetime or interval (actual: %s)", sortKeyType)); } @@ -1345,63 +1344,11 @@ private void analyzeFrameRangeOffset( if (isNumericType(sortKeyType)) { if (!isNumericType(offsetValueType)) { throw new SemanticException( - format( + String.format( "Window frame RANGE value type (%s) not compatible with sort item type (%s)", offsetValueType, sortKeyType)); } } - - // // resolve function to calculate frame boundary value (add / subtract offset from - // sortKey) - // SortItem.Ordering ordering = - // Iterables.getOnlyElement(orderBy.getSortItems()).getOrdering(); - // OperatorType operatorType; - // ResolvedFunction function; - // if ((boundType == PRECEDING && ordering == ASCENDING) || (boundType == FOLLOWING && - // ordering == DESCENDING)) { - // operatorType = SUBTRACT; - // } - // else { - // operatorType = ADD; - // } - // try { - // function = metadata.resolveOperator(operatorType, ImmutableList.of(sortKeyType, - // offsetValueType)); - // } - // catch (TrinoException e) { - // ErrorCode errorCode = e.getErrorCode(); - // if (errorCode.equals(OPERATOR_NOT_FOUND.toErrorCode())) { - // throw semanticException(TYPE_MISMATCH, offsetValue, "Window frame RANGE value type - // (%s) not compatible with sort item type (%s)", offsetValueType, sortKeyType); - // } - // throw e; - // } - // BoundSignature signature = function.getSignature(); - // Type expectedSortKeyType = signature.getArgumentTypes().get(0); - // if (!expectedSortKeyType.equals(sortKeyType)) { - // if (!typeCoercion.canCoerce(sortKeyType, expectedSortKeyType)) { - // throw semanticException(TYPE_MISMATCH, sortKey, "Sort key must evaluate to a %s - // (actual: %s)", expectedSortKeyType, sortKeyType); - // } - // sortKeyCoercionsForFrameBoundCalculation.put(NodeRef.of(offsetValue), - // expectedSortKeyType); - // } - // Type expectedOffsetValueType = signature.getArgumentTypes().get(1); - // if (!expectedOffsetValueType.equals(offsetValueType)) { - // coerceType(offsetValue, offsetValueType, expectedOffsetValueType, format("Function - // %s argument 1", function)); - // } - // Type expectedFunctionResultType = signature.getReturnType(); - // if (!expectedFunctionResultType.equals(sortKeyType)) { - // if (!typeCoercion.canCoerce(sortKeyType, expectedFunctionResultType)) { - // throw semanticException(TYPE_MISMATCH, sortKey, "Sort key must evaluate to a %s - // (actual: %s)", expectedFunctionResultType, sortKeyType); - // } - // sortKeyCoercionsForFrameBoundComparison.put(NodeRef.of(offsetValue), - // expectedFunctionResultType); - // } - // - // frameBoundCalculations.put(NodeRef.of(offsetValue), function); } @Override @@ -1451,7 +1398,7 @@ protected Type visitQuantifiedComparisonExpression( case GREATER_THAN_OR_EQUAL: if (!comparisonType.isOrderable()) { throw new SemanticException( - format( + String.format( "Type [%s] must be orderable in order to be used in quantified comparison", comparisonType)); } @@ -1460,14 +1407,14 @@ protected Type visitQuantifiedComparisonExpression( case NOT_EQUAL: if (!comparisonType.isComparable()) { throw new SemanticException( - format( + String.format( "Type [%s] must be comparable in order to be used in quantified comparison", comparisonType)); } break; default: throw new IllegalStateException( - format("Unexpected comparison type: %s", node.getOperator())); + String.format("Unexpected comparison type: %s", node.getOperator())); } return setExpressionType(node, BOOLEAN); @@ -1482,12 +1429,12 @@ public Type visitFieldReference( @Override protected Type visitExpression(Expression node, StackableAstVisitorContext context) { - throw new SemanticException(format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException(String.format("not yet implemented: %s", node.getClass().getName())); } @Override protected Type visitNode(Node node, StackableAstVisitorContext context) { - throw new SemanticException(format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException(String.format("not yet implemented: %s", node.getClass().getName())); } private Type getOperator( @@ -1515,7 +1462,7 @@ private void coerceType( if (!actualType.equals(expectedType)) { // if (!typeCoercion.canCoerce(actualType, expectedType)) { throw new SemanticException( - format("%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); + String.format("%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); // } // addOrReplaceExpressionCoercion(expression, actualType, expectedType); } @@ -1546,7 +1493,7 @@ private Type coerceToSingleType( } if (!firstType.equals(secondType)) { - throw new SemanticException(format("%s: %s vs %s", message, firstType, secondType)); + throw new SemanticException(String.format("%s: %s vs %s", message, firstType, secondType)); } return firstType; @@ -1575,7 +1522,7 @@ private Type coerceToSingleType( } else { if (!isTwoTypeComparable(Arrays.asList(superType, type))) { throw new SemanticException( - format( + String.format( "%s must be the same type or coercible to a common type. Cannot find common type between %s and %s, all types (without duplicates): %s", description, superType, type, typeExpressions.keySet())); } From f4ed436f083d007a369bd772975d4a0dda5a4b43 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:20:20 +0800 Subject: [PATCH 10/54] Support window function serialize/deserialize. --- .../plan/relational/sql/ast/FrameBound.java | 26 ++++++++ .../plan/relational/sql/ast/FunctionCall.java | 38 +++++++++++- .../plan/relational/sql/ast/OrderBy.java | 22 +++++++ .../plan/relational/sql/ast/SortItem.java | 18 ++++++ .../plan/relational/sql/ast/Window.java | 9 ++- .../plan/relational/sql/ast/WindowFrame.java | 27 ++++++++ .../relational/sql/ast/WindowReference.java | 14 +++++ .../sql/ast/WindowSpecification.java | 62 +++++++++++++++++++ 8 files changed, 212 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java index e78dc100447a7..68198bb0b8803 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -1,7 +1,12 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.iotdb.consensus.config.RatisConfig; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -82,4 +87,25 @@ public boolean shallowEquals(Node other) { FrameBound otherNode = (FrameBound) other; return type == otherNode.type; } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((byte) type.ordinal(), stream); + if (value.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Expression.serialize(value.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + } + + public FrameBound(ByteBuffer byteBuffer) { + super(null); + + type = Type.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + this.value = Optional.of(Expression.deserialize(byteBuffer)); + } else { + this.value = Optional.empty(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 69cfff551c001..fe7a4209af09b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.iotdb.consensus.config.RatisConfig; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -168,6 +169,25 @@ public void serialize(DataOutputStream stream) throws IOException { for (Expression argument : arguments) { Expression.serialize(argument, stream); } + if (nullTreatment.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + ReadWriteIOUtils.write((byte) nullTreatment.get().ordinal(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + + if (window.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + // Window type + if (window.get() instanceof WindowReference) { + ReadWriteIOUtils.write((byte) 0, stream); + } else { + ReadWriteIOUtils.write((byte) 1, stream); + } + window.get().serialize(stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } } public FunctionCall(ByteBuffer byteBuffer) { @@ -179,9 +199,21 @@ public FunctionCall(ByteBuffer byteBuffer) { while (size-- > 0) { arguments.add(Expression.deserialize(byteBuffer)); } + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + this.nullTreatment = Optional.of(NullTreatment.values()[ReadWriteIOUtils.readByte(byteBuffer)]); + } else { + this.nullTreatment = Optional.empty(); + } - // TODO: serialize window - this.window = Optional.empty(); - this.nullTreatment = Optional.empty(); + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + // Window type + if (ReadWriteIOUtils.readByte(byteBuffer) == 0) { + this.window = Optional.of(new WindowReference(byteBuffer)); + } else { + this.window = Optional.of(new WindowSpecification(byteBuffer)); + } + } else { + this.window = Optional.empty(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java index 25eff2b10303a..31a0d1acadfa3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java @@ -20,7 +20,12 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -86,4 +91,21 @@ public int hashCode() { public boolean shallowEquals(Node other) { return sameClass(this, other); } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(sortItems.size(), stream); + for (SortItem sortItem : sortItems) { + sortItem.serialize(stream); + } + } + + public OrderBy(ByteBuffer byteBuffer) { + super(null); + int size = ReadWriteIOUtils.readInt(byteBuffer); + sortItems = new ArrayList<>(size); + + for (int i = 0; i < size; i++) { + sortItems.add(new SortItem(byteBuffer)); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java index 0ec1f4c8dd42f..df9fe6d4559f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java @@ -20,7 +20,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; @@ -119,4 +123,18 @@ public boolean shallowEquals(Node other) { SortItem otherItem = (SortItem) other; return ordering == otherItem.ordering && nullOrdering == otherItem.nullOrdering; } + + void serialize(DataOutputStream stream) throws IOException { + Expression.serialize(sortKey, stream); + ReadWriteIOUtils.write((byte) ordering.ordinal(), stream); + ReadWriteIOUtils.write((byte) nullOrdering.ordinal(), stream); + } + + public SortItem(ByteBuffer byteBuffer) { + super(null); + this.sortKey = Expression.deserialize(byteBuffer); + + ordering = Ordering.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + nullOrdering = NullOrdering.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java index 2f750d77d94cb..3f27125385d1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java @@ -1,3 +1,10 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -public interface Window {} +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; + +public interface Window { + void serialize(DataOutputStream stream) throws IOException; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java index 7769849fb6473..7dfcb41615b11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java @@ -1,7 +1,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -83,4 +87,27 @@ public boolean shallowEquals(Node other) { WindowFrame otherNode = (WindowFrame) other; return type == otherNode.type; } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((byte) type.ordinal(), stream); + start.serialize(stream); + if (end.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + end.get().serialize(stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + } + + public WindowFrame(ByteBuffer byteBuffer) { + super(null); + type = Type.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + start = new FrameBound(byteBuffer); + + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + end = Optional.of(new FrameBound(byteBuffer)); + } else { + end = Optional.empty(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java index 203032797820a..bf6f935e1aaed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java @@ -1,7 +1,11 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; @@ -56,4 +60,14 @@ public String toString() { public boolean shallowEquals(Node other) { return sameClass(this, other); } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + name.serialize(stream); + } + + public WindowReference(ByteBuffer byteBuffer) { + super(null); + this.name = new Identifier(byteBuffer); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java index 2b3a442e576ce..b5e30cb053965 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java @@ -1,7 +1,12 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -93,4 +98,61 @@ public String toString() { public boolean shallowEquals(Node other) { return sameClass(this, other); } + + @Override + public void serialize(DataOutputStream stream) throws IOException { + if (existingWindowName.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + existingWindowName.get().serialize(stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + + ReadWriteIOUtils.write(partitionBy.size(), stream); + for (Expression expression : partitionBy) { + Expression.serialize(expression, stream); + } + + if (orderBy.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + orderBy.get().serialize(stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + + if (frame.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + frame.get().serialize(stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + } + + public WindowSpecification(ByteBuffer byteBuffer) { + super(null); + + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + existingWindowName = Optional.of(new Identifier(byteBuffer)); + } else { + existingWindowName = Optional.empty(); + } + + int size = ReadWriteIOUtils.readInt(byteBuffer); + partitionBy = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + partitionBy.add(Expression.deserialize(byteBuffer)); + } + + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + orderBy = Optional.of(new OrderBy(byteBuffer)); + } else { + orderBy = Optional.empty(); + } + + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + frame = Optional.of(new WindowFrame(byteBuffer)); + } else { + frame = Optional.empty(); + } + } } From 02e3f469a02c524ddaa571d4648a60078494823b Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:24:54 +0800 Subject: [PATCH 11/54] Mvn spotless apply. --- .../window/partition/frame/RowsFrame.java | 6 ++---- .../analyzer/ExpressionAnalyzer.java | 18 ++++++++++++------ .../plan/relational/sql/ast/FrameBound.java | 1 - .../plan/relational/sql/ast/FunctionCall.java | 4 ++-- .../plan/relational/sql/ast/Window.java | 2 -- .../relational/sql/ast/WindowReference.java | 1 - 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java index b41acc803532a..51f386e4f6577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RowsFrame.java @@ -48,16 +48,14 @@ public Range getRange( frameStart = 0; break; case PRECEDING: - offset = - (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); + offset = (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); frameStart = currentPosition - offset; break; case CURRENT_ROW: frameStart = currentPosition; break; case FOLLOWING: - offset = - (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); + offset = (int) getOffset(frameInfo.getStartOffsetChannel(), currentPosition); frameStart = currentPosition + offset; break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index e9b97e50ae64d..a84037f314e17 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -450,7 +450,8 @@ private Type handleResolvedField( if (!resolvedField.isLocal() && context.getContext().getCorrelationSupport() != CorrelationSupport.ALLOWED) { throw new SemanticException( - String.format("Reference to column '%s' from outer scope not allowed in this context", node)); + String.format( + "Reference to column '%s' from outer scope not allowed in this context", node)); } FieldId fieldId = FieldId.from(resolvedField); @@ -506,7 +507,8 @@ protected Type visitDereferenceExpression( Type baseType = process(node.getBase(), context); if (!(baseType instanceof RowType)) { - throw new SemanticException(String.format("Expression %s is not of type ROW", node.getBase())); + throw new SemanticException( + String.format("Expression %s is not of type ROW", node.getBase())); } RowType rowType = (RowType) baseType; @@ -520,7 +522,8 @@ protected Type visitDereferenceExpression( for (RowType.Field rowField : rowType.getFields()) { if (fieldName.equalsIgnoreCase(rowField.getName().orElse(null))) { if (foundFieldName) { - throw new SemanticException(String.format("Ambiguous row field reference: %s", fieldName)); + throw new SemanticException( + String.format("Ambiguous row field reference: %s", fieldName)); } foundFieldName = true; rowFieldType = rowField.getType(); @@ -1429,12 +1432,14 @@ public Type visitFieldReference( @Override protected Type visitExpression(Expression node, StackableAstVisitorContext context) { - throw new SemanticException(String.format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException( + String.format("not yet implemented: %s", node.getClass().getName())); } @Override protected Type visitNode(Node node, StackableAstVisitorContext context) { - throw new SemanticException(String.format("not yet implemented: %s", node.getClass().getName())); + throw new SemanticException( + String.format("not yet implemented: %s", node.getClass().getName())); } private Type getOperator( @@ -1462,7 +1467,8 @@ private void coerceType( if (!actualType.equals(expectedType)) { // if (!typeCoercion.canCoerce(actualType, expectedType)) { throw new SemanticException( - String.format("%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); + String.format( + "%s must evaluate to a %s (actual: %s)", message, expectedType, actualType)); // } // addOrReplaceExpressionCoercion(expression, actualType, expectedType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java index 68198bb0b8803..98886d2e382fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -1,7 +1,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; -import org.apache.iotdb.consensus.config.RatisConfig; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index fe7a4209af09b..4d425e69f43d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; -import org.apache.iotdb.consensus.config.RatisConfig; import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; @@ -200,7 +199,8 @@ public FunctionCall(ByteBuffer byteBuffer) { arguments.add(Expression.deserialize(byteBuffer)); } if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { - this.nullTreatment = Optional.of(NullTreatment.values()[ReadWriteIOUtils.readByte(byteBuffer)]); + this.nullTreatment = + Optional.of(NullTreatment.values()[ReadWriteIOUtils.readByte(byteBuffer)]); } else { this.nullTreatment = Optional.empty(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java index 3f27125385d1c..3a6719e3c11b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java @@ -1,7 +1,5 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -import org.apache.tsfile.utils.ReadWriteIOUtils; - import java.io.DataOutputStream; import java.io.IOException; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java index bf6f935e1aaed..0bed98eb21cbe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java @@ -1,7 +1,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; -import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; From 80468768ff34a83d1eb46c47d497c0cc3f32df8b Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:34:00 +0800 Subject: [PATCH 12/54] Insert sort by node in distributed env. --- .../TableDistributedPlanGenerator.java | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 939e7c47f650c..e48770c53ad52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; @@ -1251,9 +1252,51 @@ public List visitWindowFunction(WindowNode node, PlanContext context) nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); } - // TODO: multi-child if (childrenNodes.size() == 1) { node.setChild(childrenNodes.get(0)); + } else { + final PlanNode firstChild = childrenNodes.get(0); + if (ordering.isPresent()) { // preSort order columns + for (int i = 0; i < childrenNodes.size(); i++) { + PlanNode child = childrenNodes.get(i); + + if (child instanceof ExchangeNode) { // Insert SortNode under ExchangeNode + ExchangeNode exchangeNode = (ExchangeNode) child; + PlanNode exchangeChild = exchangeNode.getChild(); + + SortNode sortNode = + new SortNode(queryId.genPlanNodeId(), exchangeChild, ordering.get(), false, false); + exchangeNode.setChild(sortNode); + + childrenNodes.set(i, exchangeNode); + } else { // Insert SortNode above other childNode + SortNode sortNode = + new SortNode(queryId.genPlanNodeId(), child, ordering.get(), false, false); + + childrenNodes.set(i, sortNode); + } + } + + final MergeSortNode mergeSortNode = + new MergeSortNode( + queryId.genPlanNodeId(), ordering.get(), firstChild.getOutputSymbols()); + childrenNodes.forEach(mergeSortNode::addChild); + nodeOrderingMap.put(mergeSortNode.getPlanNodeId(), childOrdering); + node.setChild(mergeSortNode); + } else if (childOrdering != null) { + final MergeSortNode mergeSortNode = + new MergeSortNode( + queryId.genPlanNodeId(), childOrdering, firstChild.getOutputSymbols()); + childrenNodes.forEach(mergeSortNode::addChild); + nodeOrderingMap.put(mergeSortNode.getPlanNodeId(), childOrdering); + node.setChild(mergeSortNode); + } else { + // children has no sort property, use CollectNode to merge children + final CollectNode collectNode = + new CollectNode(queryId.genPlanNodeId(), firstChild.getOutputSymbols()); + childrenNodes.forEach(collectNode::addChild); + node.setChild(collectNode); + } } return Collections.singletonList(node); From 430d756f47e78c0fb160c4ba95a89fc90fd3eae0 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:34:17 +0800 Subject: [PATCH 13/54] Revert system config file. --- .../test/resources/confignode1conf/iotdb-system.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties index 508ab394aa346..b396e373f8699 100644 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties +++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties @@ -33,8 +33,8 @@ cn_metric_prometheus_reporter_port=9091 timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus -schema_replication_factor=1 -data_replication_factor=1 +schema_replication_factor=3 +data_replication_factor=3 udf_lib_dir=target/confignode1/ext/udf trigger_lib_dir=target/confignode1/ext/trigger pipe_lib_dir=target/confignode1/ext/pipe From 85374da6f2bc2a8c6c45f98df9faeb3c99c39729 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 02:49:32 +0800 Subject: [PATCH 14/54] Add missing import after conflict resolving. --- .../db/queryengine/plan/relational/sql/parser/AstBuilder.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 87d90febc85c7..0236df62bd456 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -198,6 +198,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Window; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowReference; From 85872695de3bf575fb6ebf5c7a23f9bfe8f93f5d Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 10:03:35 +0800 Subject: [PATCH 15/54] Remove setup.py file. --- iotdb-client/client-py/setup.py | 65 --------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 iotdb-client/client-py/setup.py diff --git a/iotdb-client/client-py/setup.py b/iotdb-client/client-py/setup.py deleted file mode 100644 index cd5f1ed198cd6..0000000000000 --- a/iotdb-client/client-py/setup.py +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -import setuptools -import io - - -try: - with io.open("README.md", encoding="utf-8") as f: - long_description = f.read() -except FileNotFoundError: - long_description = "" - - -print(long_description) - -setuptools.setup( - name="apache-iotdb", # Replace with your own username - version="1.3.1.dev0", - author=" Apache Software Foundation", - author_email="dev@iotdb.apache.org", - description="Apache IoTDB client API", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/apache/iotdb", - packages=setuptools.find_packages(), - install_requires=[ - "thrift>=0.13.0", - "pandas>=1.0.0,<1.99.99", - "numpy>=1.0.0", - "testcontainers>=2.0.0", - "sqlalchemy<1.5,>=1.4", - "sqlalchemy-utils>=0.37.8", - ], - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.7", - license="Apache License, Version 2.0", - website="https://iotdb.apache.org", - entry_points={ - "sqlalchemy.dialects": [ - "iotdb = iotdb.sqlalchemy.IoTDBDialect:IoTDBDialect", - ], - }, -) From 65c06dd1d6e0925f68e7dc947194e72b218dbfc5 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 7 May 2025 15:25:41 +0800 Subject: [PATCH 16/54] Support explain SQL clause. --- .../planner/plan/node/PlanGraphPrinter.java | 19 +++++++++++++++++++ .../plan/planner/plan/node/PlanNodeType.java | 1 + 2 files changed, 20 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index c26327b59b664..25d564c4c0980 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -85,6 +85,9 @@ import com.google.common.base.Joiner; import org.apache.commons.lang3.Validate; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; +import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import org.apache.tsfile.utils.Pair; import org.eclipse.jetty.util.StringUtil; @@ -1050,6 +1053,22 @@ public List visitTableFunctionProcessor( return render(node, boxValue, context); } + @Override + public List visitWindowFunction(WindowNode node, GraphContext context) { + List boxValue = new ArrayList<>(); + boxValue.add(String.format("WindowFunction-%s", node.getPlanNodeId().getId())); + boxValue.add(String.format("OutputSymbols: %s", node.getOutputSymbols())); + + DataOrganizationSpecification specification = node.getSpecification(); + if (!specification.getPartitionBy().isEmpty()) { + boxValue.add( + "Partition by: [" + Joiner.on(", ").join(specification.getPartitionBy()) + "]"); + } + specification.getOrderingScheme().ifPresent(orderingScheme -> boxValue.add("Order by: " + orderingScheme)); + + return render(node, boxValue, context); + } + private String printRegion(TRegionReplicaSet regionReplicaSet) { return String.format( "Partition: %s", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 4e5b3082dddc1..690b67c8358a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -300,6 +300,7 @@ public enum PlanNodeType { TABLE_FUNCTION_NODE((short) 1028), TABLE_FUNCTION_PROCESSOR_NODE((short) 1029), TABLE_GROUP_NODE((short) 1030), + TABLE_WINDOW_FUNCTION((short) 1032), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), From 45f425c0a35e73d31cf5c1ba0ab36d9e83a29e3f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 8 May 2025 10:31:50 +0800 Subject: [PATCH 17/54] Add Apache license header for each new added files. --- .../function/WindowFunctionFactory.java | 19 +++++++++++++++++++ .../relational/planner/node/WindowNode.java | 19 +++++++++++++++++++ .../plan/relational/sql/ast/FrameBound.java | 19 +++++++++++++++++++ .../plan/relational/sql/ast/Window.java | 19 +++++++++++++++++++ .../relational/sql/ast/WindowDefinition.java | 19 +++++++++++++++++++ .../plan/relational/sql/ast/WindowFrame.java | 19 +++++++++++++++++++ .../relational/sql/ast/WindowReference.java | 19 +++++++++++++++++++ .../sql/ast/WindowSpecification.java | 19 +++++++++++++++++++ .../utils/DataOrganizationSpecification.java | 19 +++++++++++++++++++ .../TableBuiltinWindowFunction.java | 19 +++++++++++++++++++ 10 files changed, 190 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java index 652a674e7152f..04c8a774deb27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/WindowFunctionFactory.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.execution.operator.process.window.function; import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.rank.CumeDistFunction; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java index b2a1f7a82d3d8..f3674c204487f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.planner.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java index 98886d2e382fe..2296a19e8db0b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java index 3a6719e3c11b2..b9c34a9496cf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import java.io.DataOutputStream; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java index 67c553f79f5c8..fde2ef6c6268b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowDefinition.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java index 7dfcb41615b11..37165fc7e4e7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java index 0bed98eb21cbe..dcf91c59aefc1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java index b5e30cb053965..39e42a3fbab65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java index 648be3e520cdc..1a2fb4fcaef2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.utils; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java index 1ec26a8bf1cec..25d6cfa8df886 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinWindowFunction.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.commons.udf.builtin.relational; import java.util.Arrays; From 7f51f403ebda8f8eb05c7a434995fc788258d26c Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 8 May 2025 10:33:04 +0800 Subject: [PATCH 18/54] Mvn spotless apply again. --- .../plan/planner/TableOperatorGenerator.java | 2 +- .../plan/planner/plan/node/PlanGraphPrinter.java | 12 ++++++------ .../plan/relational/analyzer/Analysis.java | 2 +- .../relational/sql/util/ExpressionFormatter.java | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index e20582a1600f0..e7864d93b17a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -275,8 +275,8 @@ import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.getLinearFill; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.getPreviousFill; import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions.updateFilterUsingTTL; -import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceLastCache.EMPTY_PRIMITIVE_TYPE; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.ASC_NULLS_LAST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.DESC_NULLS_FIRST; import static org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder.DESC_NULLS_LAST; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 25d564c4c0980..224458091f271 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -82,12 +82,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; +import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import com.google.common.base.Joiner; import org.apache.commons.lang3.Validate; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; -import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import org.apache.tsfile.utils.Pair; import org.eclipse.jetty.util.StringUtil; @@ -1061,10 +1060,11 @@ public List visitWindowFunction(WindowNode node, GraphContext context) { DataOrganizationSpecification specification = node.getSpecification(); if (!specification.getPartitionBy().isEmpty()) { - boxValue.add( - "Partition by: [" + Joiner.on(", ").join(specification.getPartitionBy()) + "]"); + boxValue.add("Partition by: [" + Joiner.on(", ").join(specification.getPartitionBy()) + "]"); } - specification.getOrderingScheme().ifPresent(orderingScheme -> boxValue.add("Order by: " + orderingScheme)); + specification + .getOrderingScheme() + .ifPresent(orderingScheme -> boxValue.add("Order by: " + orderingScheme)); return render(node, boxValue, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 4eff276774e47..b013400981a87 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -63,8 +63,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableFunctionInvocation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; import com.google.common.collect.ArrayListMultimap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index cc1fee82ed54a..df14e2e10b61c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -78,11 +78,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Window; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowReference; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowSpecification; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowReference; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowSpecification; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; From 5ac09aba4449089fcc7f0ef58fdbd216dac6b3d6 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 8 May 2025 10:54:39 +0800 Subject: [PATCH 19/54] Support table window function: first_value and last_value. --- .../plan/relational/metadata/TableMetadataImpl.java | 12 +++++++++--- .../apache/iotdb/db/utils/constant/SqlConstant.java | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 836077ace705d..1585b1b377878 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -696,11 +696,15 @@ && isIntegerNumber(argumentTypes.get(2)))) { "Window function [nth_value] should only have two argument, and second argument must be integer type"); } break; - // case SqlConstant.FIRST_VALUE: - // case SqlConstant.LAST_VALUE: + case SqlConstant.TABLE_FIRST_VALUE: + case SqlConstant.TABLE_LAST_VALUE: + if (argumentTypes.size() != 1) { + throw new SemanticException( + String.format("Window function [%s] should only have one argument", functionName)); + } case SqlConstant.LEAD: case SqlConstant.LAG: - if (!(argumentTypes.size() >= 1 && argumentTypes.size() <= 3)) { + if (argumentTypes.isEmpty() || argumentTypes.size() > 3) { throw new SemanticException( String.format( "Window function [%s] should only have one to three argument", functionName)); @@ -725,6 +729,8 @@ && isIntegerNumber(argumentTypes.get(2)))) { case SqlConstant.PERCENT_RANK: case SqlConstant.CUME_DIST: return DOUBLE; + case SqlConstant.TABLE_FIRST_VALUE: + case SqlConstant.TABLE_LAST_VALUE: case SqlConstant.NTH_VALUE: case SqlConstant.LEAD: case SqlConstant.LAG: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index 3ffba61925e2d..9517f7c0e58e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -107,8 +107,8 @@ protected SqlConstant() { public static final String CUME_DIST = "cume_dist"; public static final String NTILE = "ntile"; // Duplicate with aggregation function - // public static final String FIRST_VALUE = "first_value"; - // public static final String LAST_VALUE = "last_value"; + public static final String TABLE_FIRST_VALUE = "first_value"; + public static final String TABLE_LAST_VALUE = "last_value"; public static final String NTH_VALUE = "nth_value"; public static final String LEAD = "lead"; public static final String LAG = "lag"; From fc68738b34a42234bd5324568cc5b7ca6981d80e Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 8 May 2025 11:37:42 +0800 Subject: [PATCH 20/54] Return whole partition when use range frame without ORDER BY. --- .../window/partition/frame/RangeFrame.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java index 59e9de731483e..b02e184b57bc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java @@ -38,11 +38,13 @@ public class RangeFrame implements Frame { private final Partition partition; private final FrameInfo frameInfo; - private final ColumnList column; - private final TSDataType dataType; + private boolean noOrderBy = false; - private final int partitionSize; - private final RowComparator peerGroupComparator; + private ColumnList column; + private TSDataType dataType; + + private int partitionSize; + private RowComparator peerGroupComparator; private Range recentRange; public RangeFrame( @@ -52,11 +54,19 @@ public RangeFrame( RowComparator comparator) { this.partition = partition; this.frameInfo = frameInfo; + this.partitionSize = partition.getPositionCount(); + + if (frameInfo.getSortChannel() == -1) { + // No ORDER BY but uses RANGE frame + // Return whole partition + this.noOrderBy = true; + return; + } + // Only one sort key is allowed in range frame checkArgument(sortedColumns.size() == 1); this.column = sortedColumns.get(0); this.dataType = column.getDataType(); - this.partitionSize = partition.getPositionCount(); this.peerGroupComparator = comparator; this.recentRange = new Range(0, 0); } @@ -66,7 +76,7 @@ public Range getRange( int currentPosition, int currentGroup, int peerGroupStart, int peerGroupEnd) { // Full partition if (frameInfo.getStartType() == UNBOUNDED_PRECEDING - && frameInfo.getEndType() == UNBOUNDED_FOLLOWING) { + && frameInfo.getEndType() == UNBOUNDED_FOLLOWING || noOrderBy) { return new Range(0, partitionSize - 1); } From 00ea7065e73f0517491165222c0f72f072b0b4e2 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Mon, 12 May 2025 13:51:09 +0800 Subject: [PATCH 21/54] Add sort node at query plan stage. --- .../plan/relational/planner/QueryPlanner.java | 29 ++++++++++++++++++- .../relational/planner/node/Patterns.java | 4 +++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 389382503dce3..d4843572c683f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GroupNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; @@ -492,12 +493,38 @@ private PlanBuilder planWindow( mappings.put(scopeAwareKey(windowFunction, analysis, subPlan.getScope()), newSymbol); } + // Create GroupNode + List sortSymbols = new ArrayList<>(); + Map sortOrderings = new HashMap<>(); + for (Symbol symbol : specification.getPartitionBy()) { + sortSymbols.add(symbol); + sortOrderings.put(symbol, ASC_NULLS_LAST); + } + int sortKeyOffset = sortSymbols.size(); + specification + .getOrderingScheme() + .ifPresent( + orderingScheme -> { + for (Symbol symbol : orderingScheme.getOrderBy()) { + if (!sortOrderings.containsKey(symbol)) { + sortSymbols.add(symbol); + sortOrderings.put(symbol, orderingScheme.getOrdering(symbol)); + } + } + }); + GroupNode groupNode = new GroupNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + new OrderingScheme(sortSymbols, sortOrderings), + sortKeyOffset); + PlanBuilder planBuilder = new PlanBuilder(subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), groupNode); + // create window node return new PlanBuilder( subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), new WindowNode( idAllocator.genPlanNodeId(), - subPlan.getRoot(), + planBuilder.getRoot(), specification, functions.buildOrThrow(), Optional.empty(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index 1ebc30a72ed4b..56992a5241d63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -225,6 +225,10 @@ public static Pattern tableFunctionProcessor() { return typeOf(TableFunctionProcessorNode.class); } + public static Pattern windowFunction() { + return typeOf(WindowNode.class); + } + /* public static Pattern rowNumber() From 20b405aee4f340f7cf9dae99b26773fbe4a7d898 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Mon, 12 May 2025 15:01:04 +0800 Subject: [PATCH 22/54] Mvn spotless apply. --- .../window/partition/frame/RangeFrame.java | 3 ++- .../relational/metadata/TableMetadataImpl.java | 2 +- .../plan/relational/planner/QueryPlanner.java | 15 +++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java index b02e184b57bc5..e25be8ef9a901 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java @@ -76,7 +76,8 @@ public Range getRange( int currentPosition, int currentGroup, int peerGroupStart, int peerGroupEnd) { // Full partition if (frameInfo.getStartType() == UNBOUNDED_PRECEDING - && frameInfo.getEndType() == UNBOUNDED_FOLLOWING || noOrderBy) { + && frameInfo.getEndType() == UNBOUNDED_FOLLOWING + || noOrderBy) { return new Range(0, partitionSize - 1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 1585b1b377878..5f1136515a61a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -699,7 +699,7 @@ && isIntegerNumber(argumentTypes.get(2)))) { case SqlConstant.TABLE_FIRST_VALUE: case SqlConstant.TABLE_LAST_VALUE: if (argumentTypes.size() != 1) { - throw new SemanticException( + throw new SemanticException( String.format("Window function [%s] should only have one argument", functionName)); } case SqlConstant.LEAD: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index d4843572c683f..57fc8b58e9ee0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -512,12 +512,15 @@ private PlanBuilder planWindow( } } }); - GroupNode groupNode = new GroupNode( - idAllocator.genPlanNodeId(), - subPlan.getRoot(), - new OrderingScheme(sortSymbols, sortOrderings), - sortKeyOffset); - PlanBuilder planBuilder = new PlanBuilder(subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), groupNode); + GroupNode groupNode = + new GroupNode( + idAllocator.genPlanNodeId(), + subPlan.getRoot(), + new OrderingScheme(sortSymbols, sortOrderings), + sortKeyOffset); + PlanBuilder planBuilder = + new PlanBuilder( + subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), groupNode); // create window node return new PlanBuilder( From f11a379b1dda58cb53c84a65d00b4e5061eb55ca Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Mon, 12 May 2025 16:24:26 +0800 Subject: [PATCH 23/54] Support distributed plan. --- .../execution/operator/process/window/TableWindowOperator.java | 3 +++ .../execution/operator/process/window/utils/RowComparator.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java index ed1cbbcc481b8..6dfb4ab4e15ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java @@ -148,6 +148,9 @@ public TsBlock next() throws Exception { // In this case, all partition executors are done } + if (!inputOperator.isBlocked().isDone()) { + return null; + } if (inputOperator.hasNextWithTimer()) { // This TsBlock is pre-sorted with PARTITION BY and ORDER BY channels TsBlock preSortedBlock = inputOperator.nextWithTimer(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index 0c7401d5c9ca6..066b8867a418d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -157,6 +157,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o } break; case TEXT: + case STRING: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -214,6 +215,7 @@ public boolean equal(List columns1, int offset1, List columns2, } break; case TEXT: + case STRING: Binary bin1 = column1.getBinary(offset1); Binary bin2 = column2.getBinary(offset2); if (!bin1.equals(bin2)) { From 2e1ab810c68573b9959b7c910bb25015e43767e7 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 13 May 2025 17:25:37 +0800 Subject: [PATCH 24/54] Rewrite generating distributed plan stage. --- .../TableDistributedPlanGenerator.java | 61 ++++--------------- 1 file changed, 12 insertions(+), 49 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 6376d345bec14..902992e5bbb6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -1513,61 +1513,24 @@ public List visitTableDeviceFetch( @Override public List visitWindowFunction(WindowNode node, PlanContext context) { - List childrenNodes = node.getChild().accept(this, context); - OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); - Optional ordering = node.getSpecification().getOrderingScheme(); - if (childOrdering != null) { - nodeOrderingMap.put(node.getPlanNodeId(), childOrdering); + context.clearExpectedOrderingScheme(); + if (node.getChildren().isEmpty()) { + return Collections.singletonList(node); } + boolean canSplitPushDown = node.getChild() instanceof GroupNode; + List childrenNodes = node.getChild().accept(this, context); if (childrenNodes.size() == 1) { node.setChild(childrenNodes.get(0)); + return Collections.singletonList(node); + } else if (!canSplitPushDown) { + CollectNode collectNode = new CollectNode(queryId.genPlanNodeId(), node.getChildren().get(0).getOutputSymbols()); + childrenNodes.forEach(collectNode::addChild); + node.setChild(collectNode); + return Collections.singletonList(node); } else { - final PlanNode firstChild = childrenNodes.get(0); - if (ordering.isPresent()) { // preSort order columns - for (int i = 0; i < childrenNodes.size(); i++) { - PlanNode child = childrenNodes.get(i); - - if (child instanceof ExchangeNode) { // Insert SortNode under ExchangeNode - ExchangeNode exchangeNode = (ExchangeNode) child; - PlanNode exchangeChild = exchangeNode.getChild(); - - SortNode sortNode = - new SortNode(queryId.genPlanNodeId(), exchangeChild, ordering.get(), false, false); - exchangeNode.setChild(sortNode); - - childrenNodes.set(i, exchangeNode); - } else { // Insert SortNode above other childNode - SortNode sortNode = - new SortNode(queryId.genPlanNodeId(), child, ordering.get(), false, false); - - childrenNodes.set(i, sortNode); - } - } - - final MergeSortNode mergeSortNode = - new MergeSortNode( - queryId.genPlanNodeId(), ordering.get(), firstChild.getOutputSymbols()); - childrenNodes.forEach(mergeSortNode::addChild); - nodeOrderingMap.put(mergeSortNode.getPlanNodeId(), childOrdering); - node.setChild(mergeSortNode); - } else if (childOrdering != null) { - final MergeSortNode mergeSortNode = - new MergeSortNode( - queryId.genPlanNodeId(), childOrdering, firstChild.getOutputSymbols()); - childrenNodes.forEach(mergeSortNode::addChild); - nodeOrderingMap.put(mergeSortNode.getPlanNodeId(), childOrdering); - node.setChild(mergeSortNode); - } else { - // children has no sort property, use CollectNode to merge children - final CollectNode collectNode = - new CollectNode(queryId.genPlanNodeId(), firstChild.getOutputSymbols()); - childrenNodes.forEach(collectNode::addChild); - node.setChild(collectNode); - } + return splitForEachChild(node, childrenNodes); } - - return Collections.singletonList(node); } public static class PlanContext { From 243e17aced1be47f73d0170aa7ddb6c3ef0d55ae Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 13 May 2025 17:26:10 +0800 Subject: [PATCH 25/54] Mvn spotless apply. --- .../planner/distribute/TableDistributedPlanGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index 902992e5bbb6b..eb1c1b8bc6cec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -54,7 +54,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.EnforceSingleRowNode; -import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; @@ -1524,7 +1523,8 @@ public List visitWindowFunction(WindowNode node, PlanContext context) node.setChild(childrenNodes.get(0)); return Collections.singletonList(node); } else if (!canSplitPushDown) { - CollectNode collectNode = new CollectNode(queryId.genPlanNodeId(), node.getChildren().get(0).getOutputSymbols()); + CollectNode collectNode = + new CollectNode(queryId.genPlanNodeId(), node.getChildren().get(0).getOutputSymbols()); childrenNodes.forEach(collectNode::addChild); node.setChild(collectNode); return Collections.singletonList(node); From 044586c24647f9f5d052dc12a6ee30c6e8e10219 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 13 May 2025 18:22:05 +0800 Subject: [PATCH 26/54] Modify code due to code review. --- .../window/function/value/LagFunction.java | 2 + .../window/function/value/LeadFunction.java | 1 + .../process/window/utils/RowComparator.java | 6 ++ .../planner/plan/node/PlanGraphPrinter.java | 2 +- .../analyzer/ExpressionAnalyzer.java | 12 --- .../plan/relational/planner/QueryPlanner.java | 21 ++---- .../relational/planner/RelationPlanner.java | 2 - .../relational/planner/node/WindowNode.java | 2 +- .../utils/DataOrganizationSpecification.java | 74 ------------------- 9 files changed, 19 insertions(+), 103 deletions(-) delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index 05178d41ec32d..c2797782db047 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -25,6 +25,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; +import java.sql.Blob; import java.util.List; public class LagFunction extends ValueWindowFunction { @@ -104,6 +105,7 @@ private void writeDefaultValue( return; case TEXT: case STRING: + case BLOB: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java index ecbc3b62256dd..e21ff13a861bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java @@ -105,6 +105,7 @@ private void writeDefaultValue( return; case TEXT: case STRING: + case BLOB: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index 066b8867a418d..400756d94f204 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -59,6 +59,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse } break; case INT32: + case DATE: int int1 = column.getInt(offset1); int int2 = column.getInt(offset2); if (int1 != int2) { @@ -89,6 +90,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse break; case TEXT: case STRING: + case BLOB: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -128,6 +130,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o } break; case INT32: + case DATE: int int1 = column.getInt(offset1); int int2 = column.getInt(offset2); if (int1 != int2) { @@ -158,6 +161,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o break; case TEXT: case STRING: + case BLOB: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -186,6 +190,7 @@ public boolean equal(List columns1, int offset1, List columns2, } break; case INT32: + case DATE: int int1 = column1.getInt(offset1); int int2 = column2.getInt(offset2); if (int1 != int2) { @@ -216,6 +221,7 @@ public boolean equal(List columns1, int offset1, List columns2, break; case TEXT: case STRING: + case BLOB: Binary bin1 = column1.getBinary(offset1); Binary bin2 = column2.getBinary(offset2); if (!bin1.equals(bin2)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java index 224458091f271..bce0ad4c7fe04 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java @@ -65,6 +65,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.CrossSeriesAggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.DataOrganizationSpecification; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AssignUniqueId; @@ -83,7 +84,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; -import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import com.google.common.base.Joiner; import org.apache.commons.lang3.Validate; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index b150b65ffbde1..cc185abece91e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -150,8 +150,6 @@ public class ExpressionAnalyzer { private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); private final Set> subqueries = new LinkedHashSet<>(); private final Set> existsSubqueries = new LinkedHashSet<>(); - private final Map, Type> expressionCoercions = new LinkedHashMap<>(); - private final Set> typeOnlyCoercions = new LinkedHashSet<>(); private final Set> subqueryInPredicates = new LinkedHashSet<>(); private final Map, Analysis.PredicateCoercions> predicateCoercions = @@ -305,14 +303,6 @@ public Set> getWindowFunctions() { return unmodifiableSet(windowFunctions); } - public Map, Type> getExpressionCoercions() { - return unmodifiableMap(expressionCoercions); - } - - public Set> getTypeOnlyCoercions() { - return unmodifiableSet(typeOnlyCoercions); - } - private Type getExpressionType(Expression expression) { requireNonNull(expression, "expression cannot be null"); @@ -910,8 +900,6 @@ protected Type visitFunctionCall( functionNullability = FunctionNullability.getWindowFunctionNullability(argumentTypes.size()); break; - default: - // ignore } // now we only support scalar or agg functions diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 57fc8b58e9ee0..7a51c84203914 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -62,7 +62,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; -import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -116,7 +115,6 @@ public class QueryPlanner { private final Analysis analysis; private final SymbolAllocator symbolAllocator; - private final QueryId idAllocator; private final MPPQueryContext queryContext; private final QueryId queryIdAllocator; private final SessionInfo session; @@ -130,7 +128,6 @@ public class QueryPlanner { public QueryPlanner( Analysis analysis, SymbolAllocator symbolAllocator, - QueryId idAllocator, MPPQueryContext queryContext, Optional outerContext, SessionInfo session, @@ -144,7 +141,6 @@ public QueryPlanner( this.analysis = analysis; this.symbolAllocator = symbolAllocator; - this.idAllocator = idAllocator; this.queryContext = queryContext; this.queryIdAllocator = queryContext.getQueryId(); this.session = session; @@ -360,7 +356,7 @@ private PlanBuilder planWindowFunctions( // avg(v) OVER (ORDER BY v) // Needs to be rewritten as // avg(CAST(v AS double)) OVER (ORDER BY v) - PlanAndMappings coercions = coerce(subPlan, inputs, analysis, idAllocator, symbolAllocator); + PlanAndMappings coercions = coerce(subPlan, inputs, analysis, queryIdAllocator, symbolAllocator); subPlan = coercions.getSubPlan(); // For frame of type RANGE, append casts and functions necessary for frame bound calculations @@ -486,7 +482,6 @@ private PlanBuilder planWindow( .map(argument -> coercions.get(argument).toSymbolReference()) .collect(toImmutableList()), frame, - // TODO: remove ignore null nullTreatment == FunctionCall.NullTreatment.IGNORE); functions.put(newSymbol, function); @@ -514,7 +509,7 @@ private PlanBuilder planWindow( }); GroupNode groupNode = new GroupNode( - idAllocator.genPlanNodeId(), + queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), new OrderingScheme(sortSymbols, sortOrderings), sortKeyOffset); @@ -526,7 +521,7 @@ private PlanBuilder planWindow( return new PlanBuilder( subPlan.getTranslations().withAdditionalMappings(mappings.buildOrThrow()), new WindowNode( - idAllocator.genPlanNodeId(), + queryIdAllocator.genPlanNodeId(), planBuilder.getRoot(), specification, functions.buildOrThrow(), @@ -594,7 +589,7 @@ private FrameBoundPlanAndSymbols planFrameBound( new Cast(new NullLiteral(), toSqlType(BOOLEAN))); subPlan = subPlan.withNewRoot( - new FilterNode(idAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); + new FilterNode(queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); // Then, coerce the sortKey so that we can add / subtract the offset. // Note: for that we cannot rely on the usual mechanism of using the coerce() method. The @@ -622,7 +617,7 @@ private FrameBoundPlanAndSymbols planFrameBound( subPlan = subPlan.withNewRoot( new ProjectNode( - idAllocator.genPlanNodeId(), + queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), Assignments.builder() .putIdentities(subPlan.getRoot().getOutputSymbols()) @@ -645,7 +640,7 @@ private FrameBoundPlanAndSymbols planFrameBound( subPlan = subPlan.withNewRoot( new ProjectNode( - idAllocator.genPlanNodeId(), + queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), Assignments.builder() .putIdentities(subPlan.getRoot().getOutputSymbols()) @@ -674,7 +669,7 @@ private FrameBoundPlanAndSymbols planFrameBound( subPlan = subPlan.withNewRoot( new ProjectNode( - idAllocator.genPlanNodeId(), + queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), Assignments.builder() .putIdentities(subPlan.getRoot().getOutputSymbols()) @@ -707,7 +702,7 @@ private FrameOffsetPlanAndSymbol planFrameOffset( new Cast(new NullLiteral(), toSqlType(BOOLEAN))); subPlan = subPlan.withNewRoot( - new FilterNode(idAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); + new FilterNode(queryIdAllocator.genPlanNodeId(), subPlan.getRoot(), predicate)); return new FrameOffsetPlanAndSymbol(subPlan, Optional.of(offsetSymbol)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 191ab05022d3c..72419b5036709 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -168,7 +168,6 @@ protected RelationPlan visitQuery(final Query node, final Void context) { return new QueryPlanner( analysis, symbolAllocator, - idAllocator, queryContext, outerContext, sessionInfo, @@ -266,7 +265,6 @@ protected RelationPlan visitQuerySpecification( return new QueryPlanner( analysis, symbolAllocator, - idAllocator, queryContext, outerContext, sessionInfo, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java index f3674c204487f..b2ab99ee933be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java @@ -24,12 +24,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.planner.DataOrganizationSpecification; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FrameBound; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; -import org.apache.iotdb.db.queryengine.plan.relational.utils.DataOrganizationSpecification; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java deleted file mode 100644 index 1a2fb4fcaef2a..0000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/DataOrganizationSpecification.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.utils; - -import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; -import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public class DataOrganizationSpecification { - private final List partitionBy; - private final Optional orderingScheme; - - public DataOrganizationSpecification( - List partitionBy, Optional orderingScheme) { - requireNonNull(partitionBy, "partitionBy is null"); - requireNonNull(orderingScheme, "orderingScheme is null"); - - this.partitionBy = ImmutableList.copyOf(partitionBy); - this.orderingScheme = requireNonNull(orderingScheme, "orderingScheme is null"); - } - - public List getPartitionBy() { - return partitionBy; - } - - public Optional getOrderingScheme() { - return orderingScheme; - } - - @Override - public int hashCode() { - return Objects.hash(partitionBy, orderingScheme); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - DataOrganizationSpecification other = (DataOrganizationSpecification) obj; - - return Objects.equals(this.partitionBy, other.partitionBy) - && Objects.equals(this.orderingScheme, other.orderingScheme); - } -} From a941c06e7316ddb6e6695a03a28356ad258e47c1 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 13 May 2025 18:22:34 +0800 Subject: [PATCH 27/54] Mvn spotless apply again. --- .../process/window/function/value/LagFunction.java | 1 - .../plan/relational/planner/QueryPlanner.java | 3 ++- .../plan/relational/planner/RelationPlanner.java | 14 ++------------ 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index c2797782db047..90b519d52cd97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -25,7 +25,6 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.sql.Blob; import java.util.List; public class LagFunction extends ValueWindowFunction { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 7a51c84203914..647afa8ef2c76 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -356,7 +356,8 @@ private PlanBuilder planWindowFunctions( // avg(v) OVER (ORDER BY v) // Needs to be rewritten as // avg(CAST(v AS double)) OVER (ORDER BY v) - PlanAndMappings coercions = coerce(subPlan, inputs, analysis, queryIdAllocator, symbolAllocator); + PlanAndMappings coercions = + coerce(subPlan, inputs, analysis, queryIdAllocator, symbolAllocator); subPlan = coercions.getSubPlan(); // For frame of type RANGE, append casts and functions necessary for frame bound calculations diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 72419b5036709..1cffc4612d9e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -166,12 +166,7 @@ public RelationPlanner( @Override protected RelationPlan visitQuery(final Query node, final Void context) { return new QueryPlanner( - analysis, - symbolAllocator, - queryContext, - outerContext, - sessionInfo, - recursiveSubqueries) + analysis, symbolAllocator, queryContext, outerContext, sessionInfo, recursiveSubqueries) .plan(node); } @@ -263,12 +258,7 @@ protected RelationPlan visitTable(final Table table, final Void context) { protected RelationPlan visitQuerySpecification( final QuerySpecification node, final Void context) { return new QueryPlanner( - analysis, - symbolAllocator, - queryContext, - outerContext, - sessionInfo, - recursiveSubqueries) + analysis, symbolAllocator, queryContext, outerContext, sessionInfo, recursiveSubqueries) .plan(node); } From 84ba9cd9b47c06c2e3b6948e4363b17a7284a255 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 20 May 2025 11:44:26 +0800 Subject: [PATCH 28/54] Added comment for update in original operator. --- .../db/queryengine/plan/planner/TableOperatorGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 74b04718ccce5..59a4df033c5e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -138,9 +138,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.db.queryengine.plan.relational.metadata.NonAlignedDeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.planner.CastToBlobLiteralVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.CastToBooleanLiteralVisitor; From 4bcb07fc6de779aa0efad5798a892f38734509c8 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 20 May 2025 11:45:35 +0800 Subject: [PATCH 29/54] Mvn spotless apply. --- .../execution/operator/process/FilterAndProjectOperator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java index 80abeda3b502b..807ead00ec2ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java @@ -170,6 +170,8 @@ private TsBlock getFilterTsBlock(TsBlock input) { if (!hasNonMappableUDF) { // get result of calculated common sub expressions for (ColumnTransformer columnTransformer : commonTransformerList) { + // CASE WHEN clause would clear all its cache + // evaluate again to acquire cache columnTransformer.tryEvaluate(); resultColumns.add(columnTransformer.getColumn()); } From 12252899c7fecc9f69bb56c07148c3fe7b51f0e0 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Tue, 20 May 2025 12:39:18 +0800 Subject: [PATCH 30/54] Add parentheses in RangeFrame. --- .../operator/process/window/partition/frame/RangeFrame.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java index e25be8ef9a901..b4b4615be152b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrame.java @@ -75,8 +75,8 @@ public RangeFrame( public Range getRange( int currentPosition, int currentGroup, int peerGroupStart, int peerGroupEnd) { // Full partition - if (frameInfo.getStartType() == UNBOUNDED_PRECEDING - && frameInfo.getEndType() == UNBOUNDED_FOLLOWING + if ((frameInfo.getStartType() == UNBOUNDED_PRECEDING + && frameInfo.getEndType() == UNBOUNDED_FOLLOWING) || noOrderBy) { return new Range(0, partitionSize - 1); } From 9b5958c1201225748e2f12da36098479249a83d4 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 21 May 2025 00:28:27 +0800 Subject: [PATCH 31/54] Separate child operator block judgment to single method. --- .../operator/process/window/TableWindowOperator.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java index 6dfb4ab4e15ce..cb7fb15d470f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/TableWindowOperator.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; @@ -134,6 +135,11 @@ public OperatorContext getOperatorContext() { return operatorContext; } + @Override + public ListenableFuture isBlocked() { + return inputOperator.isBlocked(); + } + @Override public TsBlock next() throws Exception { long startTime = System.nanoTime(); @@ -148,9 +154,6 @@ public TsBlock next() throws Exception { // In this case, all partition executors are done } - if (!inputOperator.isBlocked().isDone()) { - return null; - } if (inputOperator.hasNextWithTimer()) { // This TsBlock is pre-sorted with PARTITION BY and ORDER BY channels TsBlock preSortedBlock = inputOperator.nextWithTimer(); From 32213310e667c363d871b0d26d2d81bf21ae5880 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 28 May 2025 02:36:21 +0800 Subject: [PATCH 32/54] Fix IT errors. --- .../partition/frame/RangeFrameTest.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java index 4c2885fccc138..0b18c388ad93c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java @@ -156,7 +156,12 @@ public void testUnboundPrecedingAndCurrentRow() { new FrameInfo( FrameInfo.FrameType.RANGE, FrameInfo.FrameBoundType.UNBOUNDED_PRECEDING, - FrameInfo.FrameBoundType.CURRENT_ROW); + -1, + FrameInfo.FrameBoundType.CURRENT_ROW, + -1, + 0, + SortOrder.ASC_NULLS_LAST // Sort channel does not matter + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); @@ -752,7 +757,12 @@ public void testCurrentRowAndCurrentRow() { new FrameInfo( FrameInfo.FrameType.RANGE, FrameInfo.FrameBoundType.CURRENT_ROW, - FrameInfo.FrameBoundType.CURRENT_ROW); + -1, + FrameInfo.FrameBoundType.CURRENT_ROW, + -1, + 0, + SortOrder.ASC_NULLS_LAST // Sort Order doest not matter + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); @@ -884,7 +894,11 @@ public void testCurrentRowAndUnboundFollowing() { new FrameInfo( FrameInfo.FrameType.RANGE, FrameInfo.FrameBoundType.CURRENT_ROW, - FrameInfo.FrameBoundType.UNBOUNDED_FOLLOWING); + -1, + FrameInfo.FrameBoundType.UNBOUNDED_FOLLOWING, + 0, + SortOrder.ASC_NULLS_FIRST // Sort channel does not matter + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); From 605b6a442e49864cdf6f8ecd60b3dac8354c647f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 28 May 2025 02:38:37 +0800 Subject: [PATCH 33/54] Add prune window column optimization. --- .../relational/planner/SymbolsExtractor.java | 13 ++-- .../iterative/rule/PruneWindowColumns.java | 65 +++++++++++++++++++ .../relational/planner/node/Patterns.java | 2 +- .../optimizations/LogicalOptimizeFactory.java | 2 + .../planner/assertions/PlanMatchPattern.java | 5 ++ 5 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java index 19d6bd058b7e0..92287dba72b40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.Lookup; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DefaultExpressionTraversalVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DefaultTraversalVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; @@ -87,11 +88,9 @@ public static Set extractUnique(AggregationNode.Aggregation aggregation) return ImmutableSet.copyOf(extractAll(aggregation)); } - /* - public static Set extractUnique(WindowNode.Function function) - { + public static Set extractUnique(WindowNode.Function function) { return ImmutableSet.copyOf(extractAll(function)); - }*/ + } public static List extractAll(Expression expression) { ImmutableList.Builder builder = ImmutableList.builder(); @@ -110,9 +109,7 @@ public static List extractAll(AggregationNode.Aggregation aggregation) { return builder.build(); } - /* - public static List extractAll(WindowNode.Function function) - { + public static List extractAll(WindowNode.Function function) { ImmutableList.Builder builder = ImmutableList.builder(); for (Expression argument : function.getArguments()) { builder.addAll(extractAll(argument)); @@ -122,7 +119,7 @@ public static List extractAll(WindowNode.Function function) function.getFrame().getStartValue().ifPresent(builder::add); function.getFrame().getSortKeyCoercedForFrameStartComparison().ifPresent(builder::add); return builder.build(); - }*/ + } // to extract qualified name with prefix public static Set extractNames( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java new file mode 100644 index 0000000000000..86a00196cd727 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java @@ -0,0 +1,65 @@ +package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.Util.restrictOutputs; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.Patterns.window; + +public class PruneWindowColumns extends ProjectOffPushDownRule { + public PruneWindowColumns() { + super(window()); + } + + @Override + protected Optional pushDownProjectOff(Context context, WindowNode windowNode, Set referencedOutputs) { + Map referencedFunctions = Maps.filterKeys( + windowNode.getWindowFunctions(), + referencedOutputs::contains); + + if (referencedFunctions.isEmpty()) { + return Optional.of(windowNode.getChild()); + } + + ImmutableSet.Builder referencedInputs = ImmutableSet.builder() + .addAll(windowNode.getChild().getOutputSymbols().stream() + .filter(referencedOutputs::contains) + .iterator()) + .addAll(windowNode.getSpecification().getPartitionBy()); + + windowNode.getSpecification().getOrderingScheme().ifPresent( + orderingScheme -> orderingScheme + .getOrderBy() + .forEach(referencedInputs::add)); + windowNode.getHashSymbol().ifPresent(referencedInputs::add); + + for (WindowNode.Function windowFunction : referencedFunctions.values()) { + referencedInputs.addAll(SymbolsExtractor.extractUnique(windowFunction)); + } + + PlanNode prunedWindowNode = new WindowNode( + windowNode.getPlanNodeId(), + restrictOutputs(context.getIdAllocator(), windowNode.getChild(), referencedInputs.build()) + .orElse(windowNode.getChild()), + windowNode.getSpecification(), + referencedFunctions, + windowNode.getHashSymbol(), + windowNode.getPrePartitionedInputs(), + windowNode.getPreSortedOrderPrefix()); + + if (prunedWindowNode.getOutputSymbols().size() == windowNode.getOutputSymbols().size()) { + // Neither function pruning nor input pruning was successful. + return Optional.empty(); + } + + return Optional.of(prunedWindowNode); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index 56992a5241d63..3440e791fd852 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -225,7 +225,7 @@ public static Pattern tableFunctionProcessor() { return typeOf(TableFunctionProcessorNode.class); } - public static Pattern windowFunction() { + public static Pattern window() { return typeOf(WindowNode.class); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 59df50bd8614b..764d27f3648c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -59,6 +59,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableFunctionProcessorSourceColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableScanColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTopKColumns; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneWindowColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughOffset; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PushLimitThroughProject; import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.RemoveDuplicateConditions; @@ -124,6 +125,7 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { new PruneTableFunctionProcessorSourceColumns(), new PruneTableScanColumns(plannerContext.getMetadata()), new PruneTopKColumns(), + new PruneWindowColumns(), new PruneJoinColumns(), new PruneJoinChildrenColumns()); IterativeOptimizer columnPruningOptimizer = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index ef051873c3515..09af61a3db81c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -52,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -426,6 +427,10 @@ public static PlanMatchPattern aggregationTableScan() { return node(AggregationTableScanNode.class); } + public static PlanMatchPattern window(PlanMatchPattern source) { + return node(WindowNode.class, source); + } + public static PlanMatchPattern markDistinct( String markerSymbol, List distinctSymbols, PlanMatchPattern source) { return node(MarkDistinctNode.class, source) From aeb67f7ac6b7c17e5dafcf79920ac6338a432dff Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 28 May 2025 02:39:08 +0800 Subject: [PATCH 34/54] Mvn spotless apply. --- .../relational/planner/SymbolsExtractor.java | 20 +++---- .../iterative/rule/PruneWindowColumns.java | 53 ++++++++++--------- .../partition/frame/RangeFrameTest.java | 10 ++-- 3 files changed, 44 insertions(+), 39 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java index 92287dba72b40..ce54b2b503a33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/SymbolsExtractor.java @@ -89,7 +89,7 @@ public static Set extractUnique(AggregationNode.Aggregation aggregation) } public static Set extractUnique(WindowNode.Function function) { - return ImmutableSet.copyOf(extractAll(function)); + return ImmutableSet.copyOf(extractAll(function)); } public static List extractAll(Expression expression) { @@ -110,15 +110,15 @@ public static List extractAll(AggregationNode.Aggregation aggregation) { } public static List extractAll(WindowNode.Function function) { - ImmutableList.Builder builder = ImmutableList.builder(); - for (Expression argument : function.getArguments()) { - builder.addAll(extractAll(argument)); - } - function.getFrame().getEndValue().ifPresent(builder::add); - function.getFrame().getSortKeyCoercedForFrameEndComparison().ifPresent(builder::add); - function.getFrame().getStartValue().ifPresent(builder::add); - function.getFrame().getSortKeyCoercedForFrameStartComparison().ifPresent(builder::add); - return builder.build(); + ImmutableList.Builder builder = ImmutableList.builder(); + for (Expression argument : function.getArguments()) { + builder.addAll(extractAll(argument)); + } + function.getFrame().getEndValue().ifPresent(builder::add); + function.getFrame().getSortKeyCoercedForFrameEndComparison().ifPresent(builder::add); + function.getFrame().getStartValue().ifPresent(builder::add); + function.getFrame().getSortKeyCoercedForFrameStartComparison().ifPresent(builder::add); + return builder.build(); } // to extract qualified name with prefix diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java index 86a00196cd727..f94685a8e8e32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java @@ -1,12 +1,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + import java.util.Map; import java.util.Optional; import java.util.Set; @@ -20,40 +21,44 @@ public PruneWindowColumns() { } @Override - protected Optional pushDownProjectOff(Context context, WindowNode windowNode, Set referencedOutputs) { - Map referencedFunctions = Maps.filterKeys( - windowNode.getWindowFunctions(), - referencedOutputs::contains); + protected Optional pushDownProjectOff( + Context context, WindowNode windowNode, Set referencedOutputs) { + Map referencedFunctions = + Maps.filterKeys(windowNode.getWindowFunctions(), referencedOutputs::contains); if (referencedFunctions.isEmpty()) { return Optional.of(windowNode.getChild()); } - ImmutableSet.Builder referencedInputs = ImmutableSet.builder() - .addAll(windowNode.getChild().getOutputSymbols().stream() - .filter(referencedOutputs::contains) - .iterator()) - .addAll(windowNode.getSpecification().getPartitionBy()); + ImmutableSet.Builder referencedInputs = + ImmutableSet.builder() + .addAll( + windowNode.getChild().getOutputSymbols().stream() + .filter(referencedOutputs::contains) + .iterator()) + .addAll(windowNode.getSpecification().getPartitionBy()); - windowNode.getSpecification().getOrderingScheme().ifPresent( - orderingScheme -> orderingScheme - .getOrderBy() - .forEach(referencedInputs::add)); + windowNode + .getSpecification() + .getOrderingScheme() + .ifPresent(orderingScheme -> orderingScheme.getOrderBy().forEach(referencedInputs::add)); windowNode.getHashSymbol().ifPresent(referencedInputs::add); for (WindowNode.Function windowFunction : referencedFunctions.values()) { referencedInputs.addAll(SymbolsExtractor.extractUnique(windowFunction)); } - PlanNode prunedWindowNode = new WindowNode( - windowNode.getPlanNodeId(), - restrictOutputs(context.getIdAllocator(), windowNode.getChild(), referencedInputs.build()) - .orElse(windowNode.getChild()), - windowNode.getSpecification(), - referencedFunctions, - windowNode.getHashSymbol(), - windowNode.getPrePartitionedInputs(), - windowNode.getPreSortedOrderPrefix()); + PlanNode prunedWindowNode = + new WindowNode( + windowNode.getPlanNodeId(), + restrictOutputs( + context.getIdAllocator(), windowNode.getChild(), referencedInputs.build()) + .orElse(windowNode.getChild()), + windowNode.getSpecification(), + referencedFunctions, + windowNode.getHashSymbol(), + windowNode.getPrePartitionedInputs(), + windowNode.getPreSortedOrderPrefix()); if (prunedWindowNode.getOutputSymbols().size() == windowNode.getOutputSymbols().size()) { // Neither function pruning nor input pruning was successful. diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java index 0b18c388ad93c..fe8a5edb7fb59 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/partition/frame/RangeFrameTest.java @@ -160,8 +160,8 @@ public void testUnboundPrecedingAndCurrentRow() { FrameInfo.FrameBoundType.CURRENT_ROW, -1, 0, - SortOrder.ASC_NULLS_LAST // Sort channel does not matter - ); + SortOrder.ASC_NULLS_LAST // Sort channel does not matter + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); @@ -761,8 +761,8 @@ public void testCurrentRowAndCurrentRow() { FrameInfo.FrameBoundType.CURRENT_ROW, -1, 0, - SortOrder.ASC_NULLS_LAST // Sort Order doest not matter - ); + SortOrder.ASC_NULLS_LAST // Sort Order doest not matter + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); @@ -898,7 +898,7 @@ public void testCurrentRowAndUnboundFollowing() { FrameInfo.FrameBoundType.UNBOUNDED_FOLLOWING, 0, SortOrder.ASC_NULLS_FIRST // Sort channel does not matter - ); + ); FrameTestUtils utils = new FrameTestUtils(tsBlock, dataType, frameInfo); utils.processAllRows(); From 735b97e92c69361842346f7afc8d805d15978808 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 28 May 2025 02:42:04 +0800 Subject: [PATCH 35/54] Remove all TODOs. --- .../queryengine/plan/relational/analyzer/StatementAnalyzer.java | 1 - .../plan/relational/metadata/FunctionNullability.java | 1 - .../db/queryengine/plan/relational/planner/QueryPlanner.java | 1 - 3 files changed, 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index ddb76e9d83ba6..812f09f0d2c15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1106,7 +1106,6 @@ private List analyzeWindowFunctions( } Analysis.ResolvedWindow window = analysis.getWindow(windowFunction); - // TODO get function requirements from window function metadata when we have it String name = windowFunction.getName().toString().toLowerCase(ENGLISH); if (name.equals("lag") || name.equals("lead")) { if (!window.getOrderBy().isPresent()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java index 7dabe159f4931..5d847e074f1f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/FunctionNullability.java @@ -51,7 +51,6 @@ public static FunctionNullability getScalarFunctionNullability(int argumentsNumb return new FunctionNullability(true, Collections.nCopies(argumentsNumber, true)); } - // TODO modify for each window function public static FunctionNullability getWindowFunctionNullability(int argumentsNumber) { return new FunctionNullability(true, Collections.nCopies(argumentsNumber, true)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 647afa8ef2c76..f23b8e0a78a1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -663,7 +663,6 @@ private FrameBoundPlanAndSymbols planFrameBound( coercions.get(sortKey).toSymbolReference(), toSqlType(expectedType), false, - // TODO: type coercion analysis.getType(sortKey).equals(expectedType)); Symbol castSymbol = symbolAllocator.newSymbol(cast, expectedType); sortKeyCoercions.put(expectedType, castSymbol); From 0cfbdb8d51c36d7545d36b73ce4a7e27f6ce1462 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Wed, 28 May 2025 23:48:41 +0800 Subject: [PATCH 36/54] Eliminate constant CASE WHEN clause. --- .../planner/IrExpressionInterpreter.java | 16 ++++++++++++++++ .../optimizations/LogicalOptimizeFactory.java | 3 +++ 2 files changed, 19 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java index d07be977756fe..22a370de3a2a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java @@ -57,6 +57,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.read.common.type.Type; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -640,6 +641,21 @@ private Object processComparisonExpression( toExpression(left, type(leftExpression)), toExpression(right, type(rightExpression))); } else { + if (!(left instanceof Number) || !(right instanceof Number)) { + throw new IllegalArgumentException("Both object must be type of number"); + } + BigDecimal leftNum = new BigDecimal(left.toString()); + BigDecimal rightNum = new BigDecimal(right.toString()); + + int compareResult = leftNum.compareTo(rightNum); + if (operator == ComparisonExpression.Operator.LESS_THAN) { + return compareResult < 0; + } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { + return compareResult < 0 || compareResult == 0; + } else if (operator == ComparisonExpression.Operator.EQUAL) { + return compareResult == 0; + } + return new ComparisonExpression( operator, toExpression(left, type(leftExpression)), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 764d27f3648c6..ef8c574a7ac7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -282,6 +282,9 @@ public LogicalOptimizeFactory(PlannerContext plannerContext) { plannerContext, ruleStats, ImmutableSet.of(new PruneDistinctAggregation())), simplifyOptimizer, new PushPredicateIntoTableScan(plannerContext, typeAnalyzer), + // Currently, we inline symbols but do not simplify them in predicate push down. + // So we have to add extra simplifyOptimizer here + simplifyOptimizer, // Currently, Distinct is not supported, so we cant use this rule for now. // new IterativeOptimizer( // plannerContext, From 7d4c97eb6f9c94f4f15de5272e5359e4a366d164 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 00:52:10 +0800 Subject: [PATCH 37/54] Mvn spotless apply UT. --- .../planner/WindowFunctionTest.java | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java new file mode 100644 index 0000000000000..c6c234c4e643b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java @@ -0,0 +1,230 @@ +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.Test; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.sort; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.window; + +public class WindowFunctionTest { + @Test + public void testSimpleWindowFunction() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT sum(s1) OVER(PARTITION BY tag1) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan("testdb.table1", ImmutableList.of("tag1", "s1"), ImmutableSet.of("tag1", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(sort(tableScan))))); + + /* ProjectNode + * └──WindowNode + * └──MergeSort + * ├──ExchangeNode + * │ └──TableScan + * ├──ExchangeNode + * │ └──TableScan + * └──ExchangeNode + * └──TableScan + */ + assertPlan( + planTester.getFragmentPlan(0), + output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + assertPlan(planTester.getFragmentPlan(1), tableScan); + assertPlan(planTester.getFragmentPlan(2), tableScan); + assertPlan(planTester.getFragmentPlan(3), tableScan); + } + + @Test + public void testWindowFunctionWithOrderBy() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT sum(s1) OVER(PARTITION BY tag1 ORDER BY s1) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan("testdb.table1", ImmutableList.of("tag1", "s1"), ImmutableSet.of("tag1", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(sort(tableScan))))); + + /* ProjectNode + * └──WindowNode + * └──MergeSort + * ├──ExchangeNode + * │ └──SortNode + * │ └──TableScan + * ├──ExchangeNode + * │ └──SortNode + * │ └──TableScan + * └──ExchangeNode + * └──SortNode + * └──TableScan + */ + assertPlan( + planTester.getFragmentPlan(0), + output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + assertPlan(planTester.getFragmentPlan(1), sort(tableScan)); + assertPlan(planTester.getFragmentPlan(2), sort(tableScan)); + assertPlan(planTester.getFragmentPlan(3), sort(tableScan)); + } + + @Test + public void testWindowFunctionOrderByPartitionByDup() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT sum(s1) OVER(PARTITION BY tag1 ORDER BY tag1) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan("testdb.table1", ImmutableList.of("tag1", "s1"), ImmutableSet.of("tag1", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(sort(tableScan))))); + + /* ProjectNode + * └──WindowNode + * └──MergeSort + * ├──ExchangeNode + * │ └──TableScan + * ├──ExchangeNode + * │ └──TableScan + * └──ExchangeNode + * └──TableScan + */ + assertPlan( + planTester.getFragmentPlan(0), + output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + assertPlan(planTester.getFragmentPlan(1), tableScan); + assertPlan(planTester.getFragmentPlan(2), tableScan); + assertPlan(planTester.getFragmentPlan(3), tableScan); + } + + @Test + public void testWindowFunctionWithFrame() { + PlanTester planTester = new PlanTester(); + + String sql = + "SELECT sum(s1) OVER(PARTITION BY tag1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan("testdb.table1", ImmutableList.of("tag1", "s1"), ImmutableSet.of("tag1", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──ProjectNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(sort(project(tableScan)))))); + + /* ProjectNode + * └──WindowNode + * └──MergeSort + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──TableScan + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──TableScan + * └──ExchangeNode + * └──ProjectNode + * └──TableScan + */ + assertPlan( + planTester.getFragmentPlan(0), + output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + assertPlan(planTester.getFragmentPlan(1), project(tableScan)); + assertPlan(planTester.getFragmentPlan(2), project(tableScan)); + assertPlan(planTester.getFragmentPlan(3), project(tableScan)); + } + + // @Test + // public void testWindowFunctionWithPushDown() { + // PlanTester planTester = new PlanTester(); + // + // String sql = + // "SELECT sum(s1) OVER(PARTITION BY (tag1,tag2,tag3)) FROM table1"; + // + // LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + // + // PlanMatchPattern tableScan = + // tableScan( + // "testdb.table1", + // ImmutableList.of("tag1", "tag2", "tag3", "s1"), + // ImmutableSet.of("tag1", "tag2", "tag3", "s1")); + // + // // Verify full LogicalPlan + // /* + // * └──OutputNode + // * └──ProjectNode + // * └──WindowNode + // * └──SortNode + // * └──ProjectNode + // * └──TableScanNode + // */ + // assertPlan(logicalQueryPlan, output(project(window(group(tableScan))))); + // + // /* └──ProjectNode + // * └──WindowNode + // * └──MergeSort + // * ├──ExchangeNode + // * │ └──ProjectNode + // * │ └──TableScan + // * ├──ExchangeNode + // * │ └──ProjectNode + // * │ └──TableScan + // * └──ExchangeNode + // * └──ProjectNode + // * └──TableScan + // */ + // assertPlan( + // planTester.getFragmentPlan(0), + // output(project(window(mergeSort(exchange(), exchange(), exchange()))))); + // assertPlan(planTester.getFragmentPlan(1), project(tableScan)); + // assertPlan(planTester.getFragmentPlan(2), project(tableScan)); + // assertPlan(planTester.getFragmentPlan(3), project(tableScan)); + // } +} From eeca348ba239960d30e9b932464db85f42bffa3f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 11:05:28 +0800 Subject: [PATCH 38/54] Add window node push down case in UT. --- .../planner/WindowFunctionTest.java | 104 ++++++++++-------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java index c6c234c4e643b..4f27875368693 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java @@ -8,7 +8,9 @@ import org.junit.Test; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.collect; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.group; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; @@ -181,50 +183,60 @@ public void testWindowFunctionWithFrame() { assertPlan(planTester.getFragmentPlan(3), project(tableScan)); } - // @Test - // public void testWindowFunctionWithPushDown() { - // PlanTester planTester = new PlanTester(); - // - // String sql = - // "SELECT sum(s1) OVER(PARTITION BY (tag1,tag2,tag3)) FROM table1"; - // - // LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); - // - // PlanMatchPattern tableScan = - // tableScan( - // "testdb.table1", - // ImmutableList.of("tag1", "tag2", "tag3", "s1"), - // ImmutableSet.of("tag1", "tag2", "tag3", "s1")); - // - // // Verify full LogicalPlan - // /* - // * └──OutputNode - // * └──ProjectNode - // * └──WindowNode - // * └──SortNode - // * └──ProjectNode - // * └──TableScanNode - // */ - // assertPlan(logicalQueryPlan, output(project(window(group(tableScan))))); - // - // /* └──ProjectNode - // * └──WindowNode - // * └──MergeSort - // * ├──ExchangeNode - // * │ └──ProjectNode - // * │ └──TableScan - // * ├──ExchangeNode - // * │ └──ProjectNode - // * │ └──TableScan - // * └──ExchangeNode - // * └──ProjectNode - // * └──TableScan - // */ - // assertPlan( - // planTester.getFragmentPlan(0), - // output(project(window(mergeSort(exchange(), exchange(), exchange()))))); - // assertPlan(planTester.getFragmentPlan(1), project(tableScan)); - // assertPlan(planTester.getFragmentPlan(2), project(tableScan)); - // assertPlan(planTester.getFragmentPlan(3), project(tableScan)); - // } + @Test + public void testWindowFunctionWithPushDown() { + PlanTester planTester = new PlanTester(); + + String sql = + "SELECT sum(s1) OVER(PARTITION BY tag1,tag2,tag3) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan( + "testdb.table1", + ImmutableList.of("tag1", "tag2", "tag3", "s1"), + ImmutableSet.of("tag1", "tag2", "tag3", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──ProjectNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(group(tableScan))))); + + // Verify DistributionPlan + /* + * └──OutputNode + * └──CollectNode + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──WindowNode + * │ └──TableScanNode + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──WindowNode + * │ └──TableScanNode + * └──ExchangeNode + * └──ProjectNode + * └──WindowNode + * └──MergeSortNode + * ├──ExchangeNode + * │ └──TableScan + * └──ExchangeNode + * └──TableScan + */ + assertPlan( + planTester.getFragmentPlan(0), + output(collect(exchange(), exchange(), exchange()))); + assertPlan(planTester.getFragmentPlan(1), project(window(tableScan))); + assertPlan(planTester.getFragmentPlan(2), project(window(tableScan))); + assertPlan(planTester.getFragmentPlan(3), project(window(mergeSort(exchange(), exchange())))); + assertPlan(planTester.getFragmentPlan(4), tableScan); + assertPlan(planTester.getFragmentPlan(5), tableScan); + } } From 30f16085ecf6a75cd0fccfdd0ed10bcbe22f9ed8 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 16:21:11 +0800 Subject: [PATCH 39/54] Remove function calling in window function query planner. --- .../plan/relational/planner/QueryPlanner.java | 37 ++----------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index f23b8e0a78a1c..2f468c1f7669e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -28,7 +28,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis.GroupingSetAnalysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.FieldId; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; import org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GapFillStartAndEndTimeExtractVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Aggregation; @@ -564,21 +563,11 @@ private FrameBoundPlanAndSymbols planFrameBound( Optional frameOffset, Analysis.ResolvedWindow window, Map sortKeyCoercions) { - Optional frameBoundCalculationFunction = - frameOffset.map(analysis::getFrameBoundCalculation); - - // Empty frameBoundCalculationFunction indicates that frame bound type is CURRENT ROW or - // UNBOUNDED. - // Handling it doesn't require any additional symbols. - if (!frameBoundCalculationFunction.isPresent()) { + // We don't need frameBoundCalculationFunction + if (!frameOffset.isPresent()) { return new FrameBoundPlanAndSymbols(subPlan, Optional.empty(), Optional.empty()); } - // Present frameBoundCalculationFunction indicates that frame bound type is - // PRECEDING or FOLLOWING. - // It requires adding certain projections to the plan so that the operator can determine frame - // bounds. - // First, append filter to validate offset values. They mustn't be negative or null. Symbol offsetSymbol = coercions.get(frameOffset.get()); Expression zeroOffset = zeroOfType(symbolAllocator.getTypes().getTableModelType(offsetSymbol)); @@ -627,27 +616,6 @@ private FrameBoundPlanAndSymbols planFrameBound( } } - // Next, pre-project the function which combines sortKey with the offset. - // Note: if frameOffset needs a coercion, it was added before by a call to coerce() method. - ResolvedFunction function = frameBoundCalculationFunction.get(); - Expression functionCall = - new FunctionCall( - function.toQualifiedName(), - ImmutableList.of( - sortKeyCoercedForFrameBoundCalculation.toSymbolReference(), - offsetSymbol.toSymbolReference())); - Symbol frameBoundSymbol = - symbolAllocator.newSymbol(functionCall, function.getSignature().getReturnType()); - subPlan = - subPlan.withNewRoot( - new ProjectNode( - queryIdAllocator.genPlanNodeId(), - subPlan.getRoot(), - Assignments.builder() - .putIdentities(subPlan.getRoot().getOutputSymbols()) - .put(frameBoundSymbol, functionCall) - .build())); - // Finally, coerce the sortKey to the type of frameBound so that the operator can perform // comparisons on them Optional sortKeyCoercedForFrameBoundComparison = Optional.of(coercions.get(sortKey)); @@ -679,6 +647,7 @@ private FrameBoundPlanAndSymbols planFrameBound( } } + Symbol frameBoundSymbol = coercions.get(frameOffset.get()); return new FrameBoundPlanAndSymbols( subPlan, Optional.of(frameBoundSymbol), sortKeyCoercedForFrameBoundComparison); } From 165b91197d3804a9c9d25c59dc27dd545ac4c9e4 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 16:21:53 +0800 Subject: [PATCH 40/54] Mvn spotless apply UT. --- .../planner/WindowFunctionTest.java | 109 +++++++++--------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java index 4f27875368693..707497eb0359d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java @@ -183,60 +183,57 @@ public void testWindowFunctionWithFrame() { assertPlan(planTester.getFragmentPlan(3), project(tableScan)); } - @Test - public void testWindowFunctionWithPushDown() { - PlanTester planTester = new PlanTester(); - - String sql = - "SELECT sum(s1) OVER(PARTITION BY tag1,tag2,tag3) FROM table1"; - - LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); - - PlanMatchPattern tableScan = - tableScan( - "testdb.table1", - ImmutableList.of("tag1", "tag2", "tag3", "s1"), - ImmutableSet.of("tag1", "tag2", "tag3", "s1")); - - // Verify full LogicalPlan - /* - * └──OutputNode - * └──ProjectNode - * └──WindowNode - * └──SortNode - * └──ProjectNode - * └──TableScanNode - */ - assertPlan(logicalQueryPlan, output(project(window(group(tableScan))))); - - // Verify DistributionPlan - /* - * └──OutputNode - * └──CollectNode - * ├──ExchangeNode - * │ └──ProjectNode - * │ └──WindowNode - * │ └──TableScanNode - * ├──ExchangeNode - * │ └──ProjectNode - * │ └──WindowNode - * │ └──TableScanNode - * └──ExchangeNode - * └──ProjectNode - * └──WindowNode - * └──MergeSortNode - * ├──ExchangeNode - * │ └──TableScan - * └──ExchangeNode - * └──TableScan - */ - assertPlan( - planTester.getFragmentPlan(0), - output(collect(exchange(), exchange(), exchange()))); - assertPlan(planTester.getFragmentPlan(1), project(window(tableScan))); - assertPlan(planTester.getFragmentPlan(2), project(window(tableScan))); - assertPlan(planTester.getFragmentPlan(3), project(window(mergeSort(exchange(), exchange())))); - assertPlan(planTester.getFragmentPlan(4), tableScan); - assertPlan(planTester.getFragmentPlan(5), tableScan); - } + @Test + public void testWindowFunctionWithPushDown() { + PlanTester planTester = new PlanTester(); + + String sql = "SELECT sum(s1) OVER(PARTITION BY tag1,tag2,tag3) FROM table1"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan( + "testdb.table1", + ImmutableList.of("tag1", "tag2", "tag3", "s1"), + ImmutableSet.of("tag1", "tag2", "tag3", "s1")); + + // Verify full LogicalPlan + /* + * └──OutputNode + * └──ProjectNode + * └──WindowNode + * └──SortNode + * └──ProjectNode + * └──TableScanNode + */ + assertPlan(logicalQueryPlan, output(project(window(group(tableScan))))); + + // Verify DistributionPlan + /* + * └──OutputNode + * └──CollectNode + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──WindowNode + * │ └──TableScanNode + * ├──ExchangeNode + * │ └──ProjectNode + * │ └──WindowNode + * │ └──TableScanNode + * └──ExchangeNode + * └──ProjectNode + * └──WindowNode + * └──MergeSortNode + * ├──ExchangeNode + * │ └──TableScan + * └──ExchangeNode + * └──TableScan + */ + assertPlan(planTester.getFragmentPlan(0), output(collect(exchange(), exchange(), exchange()))); + assertPlan(planTester.getFragmentPlan(1), project(window(tableScan))); + assertPlan(planTester.getFragmentPlan(2), project(window(tableScan))); + assertPlan(planTester.getFragmentPlan(3), project(window(mergeSort(exchange(), exchange())))); + assertPlan(planTester.getFragmentPlan(4), tableScan); + assertPlan(planTester.getFragmentPlan(5), tableScan); + } } From 569511010388053ca4778cda2e10ae3f7e40facf Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 16:22:38 +0800 Subject: [PATCH 41/54] Finish IT prototype. --- .../it/db/it/IoTDBWindowFunctionIT.java | 383 ++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java new file mode 100644 index 0000000000000..30614251908b8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -0,0 +1,383 @@ +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBWindowFunctionIT { + private static final String DATABASE_NAME = "test"; + private static final String[] sqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table demo (device string tag, value double field)", + "insert into demo values (2021-01-01T09:05:00, 'd1', 3)", + "insert into demo values (2021-01-01T09:07:00, 'd1', 5)", + "insert into demo values (2021-01-01T09:09:00, 'd1', 3)", + "insert into demo values (2021-01-01T09:10:00, 'd1', 1)", + "insert into demo values (2021-01-01T09:08:00, 'd2', 2)", + "insert into demo values (2021-01-01T09:15:00, 'd2', 4)", + "FLUSH", + "CLEAR ATTRIBUTE CACHE", + }; + + private static void insertData() { + try (Connection connection = EnvFactory.getEnv().getTableConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + fail("insertData failed."); + } + } + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testPartitionBy() { + String[] expectedHeader = new String[] {"time", "device", "value", "cnt"}; + String[] retArray = + new String[] { + "2021-01-01T09:05:00.000Z,d1,3.0,4,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:09:00.000Z,d1,3.0,4,", + "2021-01-01T09:10:00.000Z,d1,1.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,2,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, count(value) OVER (PARTITION BY device) AS cnt FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testOrderBy() { + String[] expectedHeader = new String[] {"time", "device", "value", "rnk"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, rank() OVER (PARTITION BY device ORDER BY value) AS rnk FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testRowsFraming() { + String[] expectedHeader = new String[] {"time", "device", "value", "cnt"}; + String[] retArray = + new String[] { + "2021-01-01T09:05:00.000Z,d1,3.0,1,", + "2021-01-01T09:07:00.000Z,d1,5.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:10:00.000Z,d1,1.0,2,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, count(value) OVER (PARTITION BY device ROWS 1 PRECEDING) AS cnt FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testGroupsFraming() { + String[] expectedHeader = new String[] {"time", "device", "value", "cnt"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,3,", + "2021-01-01T09:09:00.000Z,d1,3.0,3,", + "2021-01-01T09:07:00.000Z,d1,5.0,3,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) AS cnt FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testRangeFraming() { + String[] expectedHeader = new String[] {"time", "device", "value", "cnt"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,3,", + "2021-01-01T09:09:00.000Z,d1,3.0,3,", + "2021-01-01T09:07:00.000Z,d1,5.0,3,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, count(value) OVER (PARTITION BY device ORDER BY value RANGE BETWEEN 2 PRECEDING AND CURRENT ROW) AS cnt FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testAggregation() { + String[] expectedHeader = new String[] {"time", "device", "value", "sum"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,7.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,7.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,12.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,6.0,", + }; + tableResultSetEqualTest( + "SELECT *, sum(value) OVER (PARTITION BY device ORDER BY value) AS sum FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testFirstValue() { + String[] expectedHeader = new String[] {"time", "device", "value", "fv"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,1.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", + }; + tableResultSetEqualTest( + "SELECT *, first_value(value) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS fv FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testLastValue() { + String[] expectedHeader = new String[] {"time", "device", "value", "lv"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", + }; + tableResultSetEqualTest( + "SELECT *, last_value(value) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS lv FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNthValue() { + String[] expectedHeader = new String[] {"time", "device", "value", "nv"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", + }; + tableResultSetEqualTest( + "SELECT *, nth_value(value, 2) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS nv FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testLead() { + String[] expectedHeader = new String[] {"time", "device", "value", "ld"}; + String[] retArray = + new String[] { + "2021-01-01T09:05:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,1.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,null,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,null,", + }; + tableResultSetEqualTest( + "SELECT *, lead(value) OVER (PARTITION BY device ORDER BY time) AS ld FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testLag() { + String[] expectedHeader = new String[] {"time", "device", "value", "lg"}; + String[] retArray = + new String[] { + "2021-01-01T09:05:00.000Z,d1,3.0,null,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,null,", + "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", + }; + tableResultSetEqualTest( + "SELECT *, lag(value) OVER (PARTITION BY device ORDER BY time) AS lg FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testRank() { + String[] expectedHeader = new String[] {"time", "device", "value", "rk"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, rank() OVER (PARTITION BY device ORDER BY value) AS rk FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testDenseRank() { + String[] expectedHeader = new String[] {"time", "device", "value", "rk"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,3,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, dense_rank() OVER (PARTITION BY device ORDER BY value) AS rk FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testRowNumber() { + String[] expectedHeader = new String[] {"time", "device", "value", "rn"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,3,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, row_number() OVER (PARTITION BY device ORDER BY value) AS rn FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testPercentRank() { + String[] expectedHeader = new String[] {"time", "device", "value", "pr"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,0.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,0.3333333333333333,", + "2021-01-01T09:09:00.000Z,d1,3.0,0.3333333333333333,", + "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,0.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", + }; + tableResultSetEqualTest( + "SELECT *, percent_rank() OVER (PARTITION BY device ORDER BY value) AS pr FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testCumeDist() { + String[] expectedHeader = new String[] {"time", "device", "value", "cd"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,0.25,", + "2021-01-01T09:05:00.000Z,d1,3.0,0.75,", + "2021-01-01T09:09:00.000Z,d1,3.0,0.75,", + "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,0.5,", + "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", + }; + tableResultSetEqualTest( + "SELECT *, cume_dist() OVER (PARTITION BY device ORDER BY value) AS cd FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } + + @Test + public void testNTile() { + String[] expectedHeader = new String[] {"time", "device", "value", "nt"}; + String[] retArray = + new String[] { + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,1,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,2,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", + }; + tableResultSetEqualTest( + "SELECT *, ntile(2) OVER (PARTITION BY device ORDER BY value) AS nt FROM demo ORDER BY device", + expectedHeader, + retArray, + DATABASE_NAME); + } +} From 967ed5a9c8910c170737f5318eff53fe7f6587fa Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 16:23:17 +0800 Subject: [PATCH 42/54] Mvn spotless apply IT. --- .../it/db/it/IoTDBWindowFunctionIT.java | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java index 30614251908b8..613cc159d769f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -158,12 +158,12 @@ public void testAggregation() { String[] expectedHeader = new String[] {"time", "device", "value", "sum"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", - "2021-01-01T09:05:00.000Z,d1,3.0,7.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,7.0,", - "2021-01-01T09:07:00.000Z,d1,5.0,12.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,6.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,7.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,7.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,12.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,6.0,", }; tableResultSetEqualTest( "SELECT *, sum(value) OVER (PARTITION BY device ORDER BY value) AS sum FROM demo ORDER BY device", @@ -177,12 +177,12 @@ public void testFirstValue() { String[] expectedHeader = new String[] {"time", "device", "value", "fv"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", - "2021-01-01T09:05:00.000Z,d1,3.0,1.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", - "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,1.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,1.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,2.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", }; tableResultSetEqualTest( "SELECT *, first_value(value) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS fv FROM demo ORDER BY device", @@ -196,12 +196,12 @@ public void testLastValue() { String[] expectedHeader = new String[] {"time", "device", "value", "lv"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", - "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", - "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", }; tableResultSetEqualTest( "SELECT *, last_value(value) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS lv FROM demo ORDER BY device", @@ -215,12 +215,12 @@ public void testNthValue() { String[] expectedHeader = new String[] {"time", "device", "value", "nv"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", - "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", - "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,3.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,5.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,4.0,", }; tableResultSetEqualTest( "SELECT *, nth_value(value, 2) OVER (PARTITION BY device ORDER BY value ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS nv FROM demo ORDER BY device", @@ -234,12 +234,12 @@ public void testLead() { String[] expectedHeader = new String[] {"time", "device", "value", "ld"}; String[] retArray = new String[] { - "2021-01-01T09:05:00.000Z,d1,3.0,5.0,", - "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,1.0,", - "2021-01-01T09:10:00.000Z,d1,1.0,null,", - "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,null,", + "2021-01-01T09:05:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,1.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,null,", + "2021-01-01T09:08:00.000Z,d2,2.0,4.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,null,", }; tableResultSetEqualTest( "SELECT *, lead(value) OVER (PARTITION BY device ORDER BY time) AS ld FROM demo ORDER BY device", @@ -253,12 +253,12 @@ public void testLag() { String[] expectedHeader = new String[] {"time", "device", "value", "lg"}; String[] retArray = new String[] { - "2021-01-01T09:05:00.000Z,d1,3.0,null,", - "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", - "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", - "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,null,", - "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,null,", + "2021-01-01T09:07:00.000Z,d1,5.0,3.0,", + "2021-01-01T09:09:00.000Z,d1,3.0,5.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,3.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,null,", + "2021-01-01T09:15:00.000Z,d2,4.0,2.0,", }; tableResultSetEqualTest( "SELECT *, lag(value) OVER (PARTITION BY device ORDER BY time) AS lg FROM demo ORDER BY device", @@ -272,12 +272,12 @@ public void testRank() { String[] expectedHeader = new String[] {"time", "device", "value", "rk"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1,", - "2021-01-01T09:05:00.000Z,d1,3.0,2,", - "2021-01-01T09:09:00.000Z,d1,3.0,2,", - "2021-01-01T09:07:00.000Z,d1,5.0,4,", - "2021-01-01T09:08:00.000Z,d2,2.0,1,", - "2021-01-01T09:15:00.000Z,d2,4.0,2,", + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( "SELECT *, rank() OVER (PARTITION BY device ORDER BY value) AS rk FROM demo ORDER BY device", @@ -291,12 +291,12 @@ public void testDenseRank() { String[] expectedHeader = new String[] {"time", "device", "value", "rk"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1,", - "2021-01-01T09:05:00.000Z,d1,3.0,2,", - "2021-01-01T09:09:00.000Z,d1,3.0,2,", - "2021-01-01T09:07:00.000Z,d1,5.0,3,", - "2021-01-01T09:08:00.000Z,d2,2.0,1,", - "2021-01-01T09:15:00.000Z,d2,4.0,2,", + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,3,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( "SELECT *, dense_rank() OVER (PARTITION BY device ORDER BY value) AS rk FROM demo ORDER BY device", @@ -310,12 +310,12 @@ public void testRowNumber() { String[] expectedHeader = new String[] {"time", "device", "value", "rn"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1,", - "2021-01-01T09:05:00.000Z,d1,3.0,2,", - "2021-01-01T09:09:00.000Z,d1,3.0,3,", - "2021-01-01T09:07:00.000Z,d1,5.0,4,", - "2021-01-01T09:08:00.000Z,d2,2.0,1,", - "2021-01-01T09:15:00.000Z,d2,4.0,2,", + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,2,", + "2021-01-01T09:09:00.000Z,d1,3.0,3,", + "2021-01-01T09:07:00.000Z,d1,5.0,4,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( "SELECT *, row_number() OVER (PARTITION BY device ORDER BY value) AS rn FROM demo ORDER BY device", @@ -329,12 +329,12 @@ public void testPercentRank() { String[] expectedHeader = new String[] {"time", "device", "value", "pr"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,0.0,", - "2021-01-01T09:05:00.000Z,d1,3.0,0.3333333333333333,", - "2021-01-01T09:09:00.000Z,d1,3.0,0.3333333333333333,", - "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,0.0,", - "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,0.0,", + "2021-01-01T09:05:00.000Z,d1,3.0,0.3333333333333333,", + "2021-01-01T09:09:00.000Z,d1,3.0,0.3333333333333333,", + "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,0.0,", + "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", }; tableResultSetEqualTest( "SELECT *, percent_rank() OVER (PARTITION BY device ORDER BY value) AS pr FROM demo ORDER BY device", @@ -348,12 +348,12 @@ public void testCumeDist() { String[] expectedHeader = new String[] {"time", "device", "value", "cd"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,0.25,", - "2021-01-01T09:05:00.000Z,d1,3.0,0.75,", - "2021-01-01T09:09:00.000Z,d1,3.0,0.75,", - "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", - "2021-01-01T09:08:00.000Z,d2,2.0,0.5,", - "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", + "2021-01-01T09:10:00.000Z,d1,1.0,0.25,", + "2021-01-01T09:05:00.000Z,d1,3.0,0.75,", + "2021-01-01T09:09:00.000Z,d1,3.0,0.75,", + "2021-01-01T09:07:00.000Z,d1,5.0,1.0,", + "2021-01-01T09:08:00.000Z,d2,2.0,0.5,", + "2021-01-01T09:15:00.000Z,d2,4.0,1.0,", }; tableResultSetEqualTest( "SELECT *, cume_dist() OVER (PARTITION BY device ORDER BY value) AS cd FROM demo ORDER BY device", @@ -367,12 +367,12 @@ public void testNTile() { String[] expectedHeader = new String[] {"time", "device", "value", "nt"}; String[] retArray = new String[] { - "2021-01-01T09:10:00.000Z,d1,1.0,1,", - "2021-01-01T09:05:00.000Z,d1,3.0,1,", - "2021-01-01T09:09:00.000Z,d1,3.0,2,", - "2021-01-01T09:07:00.000Z,d1,5.0,2,", - "2021-01-01T09:08:00.000Z,d2,2.0,1,", - "2021-01-01T09:15:00.000Z,d2,4.0,2,", + "2021-01-01T09:10:00.000Z,d1,1.0,1,", + "2021-01-01T09:05:00.000Z,d1,3.0,1,", + "2021-01-01T09:09:00.000Z,d1,3.0,2,", + "2021-01-01T09:07:00.000Z,d1,5.0,2,", + "2021-01-01T09:08:00.000Z,d2,2.0,1,", + "2021-01-01T09:15:00.000Z,d2,4.0,2,", }; tableResultSetEqualTest( "SELECT *, ntile(2) OVER (PARTITION BY device ORDER BY value) AS nt FROM demo ORDER BY device", From d9f66ce3fe274df4be8a1b645508f9e60ff577b6 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 16:28:35 +0800 Subject: [PATCH 43/54] Add license header to all files. --- .../it/db/it/IoTDBWindowFunctionIT.java | 19 +++++++++++++++++++ .../iterative/rule/PruneWindowColumns.java | 19 +++++++++++++++++++ .../planner/WindowFunctionTest.java | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java index 613cc159d769f..10856354638d8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.relational.it.db.it; import org.apache.iotdb.it.env.EnvFactory; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java index f94685a8e8e32..7401a65fe07c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneWindowColumns.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java index 707497eb0359d..4c10a9961dc89 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/WindowFunctionTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.relational.planner; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; From 863b576c17119290e2f2d15efeb515650f34f45b Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 18:58:36 +0800 Subject: [PATCH 44/54] Fix other UT after introducing expression simplifier. --- .../plan/relational/planner/CorrelatedSubqueryTest.java | 5 ++--- .../relational/planner/UncorrelatedSubqueryTest.java | 9 +++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java index d7fc9575f198d..e7baf8078f112 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CorrelatedSubqueryTest.java @@ -74,8 +74,7 @@ public void testCorrelatedExistsSubquery() { * | └──TableScanNode * ├──SortNode * │ └──AggregationNode - * | └──FilterNode - * | └──TableScanNode + * | └──TableScanNode */ assertPlan( @@ -95,7 +94,7 @@ public void testCorrelatedExistsSubquery() { Collections.emptyList(), Optional.empty(), SINGLE, - filter(filterPredicate, tableScan2))))))); + tableScan2)))))); // not exists with other filter sql = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/UncorrelatedSubqueryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/UncorrelatedSubqueryTest.java index 1884a6b3406a7..eebd4e1584398 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/UncorrelatedSubqueryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/UncorrelatedSubqueryTest.java @@ -60,6 +60,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.SINGLE; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.EQUAL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.GREATER_THAN; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.LESS_THAN_OR_EQUAL; public class UncorrelatedSubqueryTest { @@ -579,9 +580,8 @@ public void testUncorrelatedNotExistsSubquery() { PlanMatchPattern tableScan2 = tableScan("testdb.table2", ImmutableList.of(), ImmutableSet.of()); Expression filterPredicate = - new NotExpression( - new ComparisonExpression( - GREATER_THAN, new SymbolReference("count"), new LongLiteral("0"))); + new ComparisonExpression( + LESS_THAN_OR_EQUAL, new SymbolReference("count"), new LongLiteral("0")); // Verify full LogicalPlan /* * └──OutputNode @@ -590,7 +590,8 @@ public void testUncorrelatedNotExistsSubquery() { * | * ├──ProjectNode * │ └──FilterNode - * | └──TableScanNode + * │ └──Aggregation + * | └──TableScanNode */ assertPlan( From 7ffdec611bbe88aff788cc05ef0fdf5350eebdf4 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 19:06:37 +0800 Subject: [PATCH 45/54] Test throw exception when use frame and window function together. --- .../relational/it/db/it/IoTDBWindowFunctionIT.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java index 10856354638d8..2f5b04a11f508 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBWindowFunctionIT.java @@ -33,6 +33,7 @@ import java.sql.Connection; import java.sql.Statement; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; import static org.junit.Assert.fail; @@ -265,6 +266,11 @@ public void testLead() { expectedHeader, retArray, DATABASE_NAME); + + tableAssertTestFail( + "SELECT *, lead(value) OVER (PARTITION BY device ORDER BY time ROWS 1 PRECEDING) AS ld FROM demo", + "Cannot specify window frame for lead function", + DATABASE_NAME); } @Test @@ -284,6 +290,11 @@ public void testLag() { expectedHeader, retArray, DATABASE_NAME); + + tableAssertTestFail( + "SELECT *, lag(value) OVER (PARTITION BY device ORDER BY time ROWS 1 PRECEDING) AS lg FROM demo", + "Cannot specify window frame for lag function", + DATABASE_NAME); } @Test From 86af25b797407827b2896df45cc5e8921733565d Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 22:22:29 +0800 Subject: [PATCH 46/54] Refactor Expression Interpreter. --- .../planner/IrExpressionInterpreter.java | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java index 22a370de3a2a8..f3dcfc9d58fd3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java @@ -644,16 +644,53 @@ private Object processComparisonExpression( if (!(left instanceof Number) || !(right instanceof Number)) { throw new IllegalArgumentException("Both object must be type of number"); } - BigDecimal leftNum = new BigDecimal(left.toString()); - BigDecimal rightNum = new BigDecimal(right.toString()); - - int compareResult = leftNum.compareTo(rightNum); - if (operator == ComparisonExpression.Operator.LESS_THAN) { - return compareResult < 0; - } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { - return compareResult < 0 || compareResult == 0; - } else if (operator == ComparisonExpression.Operator.EQUAL) { - return compareResult == 0; + + if (left instanceof Integer && right instanceof Integer) { + Integer leftNum = (Integer) left; + Integer rightNum = (Integer) right; + if (operator == ComparisonExpression.Operator.LESS_THAN) { + return leftNum < rightNum; + } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { + return leftNum <= rightNum; + } else if (operator == ComparisonExpression.Operator.EQUAL) { + return leftNum.equals(rightNum); + } + } + + if (left instanceof Long && right instanceof Long) { + Long leftNum = (Long) left; + Long rightNum = (Long) right; + if (operator == ComparisonExpression.Operator.LESS_THAN) { + return leftNum < rightNum; + } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { + return leftNum <= rightNum; + } else if (operator == ComparisonExpression.Operator.EQUAL) { + return leftNum.equals(rightNum); + } + } + + if (left instanceof Float && right instanceof Float) { + Float leftNum = (Float) left; + Float rightNum = (Float) right; + if (operator == ComparisonExpression.Operator.LESS_THAN) { + return leftNum < rightNum; + } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { + return leftNum <= rightNum; + } else if (operator == ComparisonExpression.Operator.EQUAL) { + return leftNum.equals(rightNum); + } + } + + if (left instanceof Double && right instanceof Double) { + Double leftNum = (Double) left; + Double rightNum = (Double) right; + if (operator == ComparisonExpression.Operator.LESS_THAN) { + return leftNum < rightNum; + } else if (operator == ComparisonExpression.Operator.LESS_THAN_OR_EQUAL) { + return leftNum <= rightNum; + } else if (operator == ComparisonExpression.Operator.EQUAL) { + return leftNum.equals(rightNum); + } } return new ComparisonExpression( From 406f90f873fbe6926db78ac4d8db1a491ca0ffdf Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Thu, 29 May 2025 22:23:14 +0800 Subject: [PATCH 47/54] Maybe last mvn spotless apply. --- .../plan/relational/planner/IrExpressionInterpreter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java index f3dcfc9d58fd3..208da696a0c13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/IrExpressionInterpreter.java @@ -57,7 +57,6 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.read.common.type.Type; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; From 9278053b7060ef6987736b15ef30591234c05bf5 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:01:41 +0800 Subject: [PATCH 48/54] Add serialize to buffer method in Identifier class. --- .../db/queryengine/plan/relational/sql/ast/Identifier.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java index 0b14e47bab68c..1a1c8c38b74da 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java @@ -160,6 +160,12 @@ public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(this.delimited, stream); } + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(this.value, buffer); + ReadWriteIOUtils.write(this.delimited, buffer); + } + public Identifier(ByteBuffer byteBuffer) { super(null); this.value = ReadWriteIOUtils.readString(byteBuffer); From ca65c87fac5bdabcbd2156cde78d8984868434ae Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:02:16 +0800 Subject: [PATCH 49/54] Support serialize/deserialize for window node. --- .../plan/planner/plan/node/PlanNodeType.java | 3 + .../relational/planner/node/WindowNode.java | 247 +++++++++++++++++- .../plan/relational/sql/ast/FrameBound.java | 10 + .../plan/relational/sql/ast/WindowFrame.java | 11 + 4 files changed, 269 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 690b67c8358a0..26d8db4dd565d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -131,6 +131,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; @@ -678,6 +679,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TableFunctionProcessorNode.deserialize(buffer); case 1030: return GroupNode.deserialize(buffer); + case 1032: + return WindowNode.deserialize(buffer); case 2000: return RelationalInsertTabletNode.deserialize(buffer); case 2001: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java index b2ab99ee933be..021d506054097 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/WindowNode.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; @@ -36,10 +37,14 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.errorprone.annotations.Immutable; +import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -94,6 +99,21 @@ public WindowNode( this.preSortedOrderPrefix = preSortedOrderPrefix; } + public WindowNode( + PlanNodeId id, + DataOrganizationSpecification specification, + Map windowFunctions, + Optional hashSymbol, + Set prePartitionedInputs, + int preSortedOrderPrefix) { + super(id); + this.prePartitionedInputs = ImmutableSet.copyOf(prePartitionedInputs); + this.specification = specification; + this.windowFunctions = ImmutableMap.copyOf(windowFunctions); + this.hashSymbol = hashSymbol; + this.preSortedOrderPrefix = preSortedOrderPrefix; + } + @Override public PlanNode clone() { return new WindowNode( @@ -134,10 +154,81 @@ public R accept(PlanVisitor visitor, C context) { } @Override - protected void serializeAttributes(ByteBuffer byteBuffer) {} + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_WINDOW_FUNCTION.serialize(byteBuffer); + ReadWriteIOUtils.write(prePartitionedInputs.size(), byteBuffer); + prePartitionedInputs.forEach(symbol -> Symbol.serialize(symbol, byteBuffer)); + specification.serialize(byteBuffer); + ReadWriteIOUtils.write(preSortedOrderPrefix, byteBuffer); + ReadWriteIOUtils.write(windowFunctions.size(), byteBuffer); + windowFunctions.forEach( + (symbol, function) -> { + Symbol.serialize(symbol, byteBuffer); + function.serialize(byteBuffer); + }); + if (hashSymbol.isPresent()) { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + Symbol.serialize(hashSymbol.get(), byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } + } @Override - protected void serializeAttributes(DataOutputStream stream) throws IOException {} + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_WINDOW_FUNCTION.serialize(stream); + ReadWriteIOUtils.write(prePartitionedInputs.size(), stream); + for (Symbol symbol : prePartitionedInputs) { + Symbol.serialize(symbol, stream); + } + specification.serialize(stream); + ReadWriteIOUtils.write(preSortedOrderPrefix, stream); + ReadWriteIOUtils.write(windowFunctions.size(), stream); + for (Map.Entry entry : windowFunctions.entrySet()) { + Symbol symbol = entry.getKey(); + Function function = entry.getValue(); + Symbol.serialize(symbol, stream); + function.serialize(stream); + } + if (hashSymbol.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Symbol.serialize(hashSymbol.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + } + + public static WindowNode deserialize(ByteBuffer buffer) { + int size = ReadWriteIOUtils.readInt(buffer); + Set prePartitionedInputs = new HashSet<>(size); + for (int i = 0; i < size; i++) { + prePartitionedInputs.add(Symbol.deserialize(buffer)); + } + DataOrganizationSpecification specification = DataOrganizationSpecification.deserialize(buffer); + int preSortedOrderPrefix = ReadWriteIOUtils.readInt(buffer); + size = ReadWriteIOUtils.readInt(buffer); + Map windowFunctions = new HashMap<>(size); + for (int i = 0; i < size; i++) { + Symbol symbol = Symbol.deserialize(buffer); + Function function = new Function(buffer); + windowFunctions.put(symbol, function); + } + Optional hashSymbol; + if (ReadWriteIOUtils.readByte(buffer) == 1) { + hashSymbol = Optional.of(Symbol.deserialize(buffer)); + } else { + hashSymbol = Optional.empty(); + } + + PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new WindowNode( + planNodeId, + specification, + windowFunctions, + hashSymbol, + prePartitionedInputs, + preSortedOrderPrefix); + } public Set getPrePartitionedInputs() { return prePartitionedInputs; @@ -300,6 +391,129 @@ public int hashCode() { endValue, sortKeyCoercedForFrameEndComparison); } + + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write((byte) type.ordinal(), buffer); + ReadWriteIOUtils.write((byte) startType.ordinal(), buffer); + if (startValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Symbol.serialize(startValue.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + if (sortKeyCoercedForFrameStartComparison.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Symbol.serialize(sortKeyCoercedForFrameStartComparison.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + ReadWriteIOUtils.write((byte) endType.ordinal(), buffer); + if (endValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Symbol.serialize(endValue.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + if (sortKeyCoercedForFrameEndComparison.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Symbol.serialize(sortKeyCoercedForFrameEndComparison.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + + if (originalStartValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Expression.serialize(originalStartValue.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + if (originalEndValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Expression.serialize(originalEndValue.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + } + + public void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write((byte) type.ordinal(), stream); + ReadWriteIOUtils.write((byte) startType.ordinal(), stream); + if (startValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Symbol.serialize(startValue.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + if (sortKeyCoercedForFrameStartComparison.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Symbol.serialize(sortKeyCoercedForFrameStartComparison.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + ReadWriteIOUtils.write((byte) endType.ordinal(), stream); + if (endValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Symbol.serialize(endValue.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + if (sortKeyCoercedForFrameEndComparison.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Symbol.serialize(sortKeyCoercedForFrameEndComparison.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + + if (originalStartValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Expression.serialize(originalStartValue.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + if (originalEndValue.isPresent()) { + ReadWriteIOUtils.write((byte) 1, stream); + Expression.serialize(originalEndValue.get(), stream); + } else { + ReadWriteIOUtils.write((byte) 0, stream); + } + } + + public Frame(ByteBuffer byteBuffer) { + type = WindowFrame.Type.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + startType = FrameBound.Type.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + startValue = Optional.of(Symbol.deserialize(byteBuffer)); + } else { + startValue = Optional.empty(); + } + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + sortKeyCoercedForFrameStartComparison = Optional.of(Symbol.deserialize(byteBuffer)); + } else { + sortKeyCoercedForFrameStartComparison = Optional.empty(); + } + endType = FrameBound.Type.values()[ReadWriteIOUtils.readByte(byteBuffer)]; + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + endValue = Optional.of(Symbol.deserialize(byteBuffer)); + } else { + endValue = Optional.empty(); + } + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + sortKeyCoercedForFrameEndComparison = Optional.of(Symbol.deserialize(byteBuffer)); + } else { + sortKeyCoercedForFrameEndComparison = Optional.empty(); + } + + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + originalStartValue = Optional.of(Expression.deserialize(byteBuffer)); + } else { + originalStartValue = Optional.empty(); + } + if (ReadWriteIOUtils.readByte(byteBuffer) == 1) { + originalEndValue = Optional.of(Expression.deserialize(byteBuffer)); + } else { + originalEndValue = Optional.empty(); + } + } } @Immutable @@ -355,5 +569,34 @@ public boolean equals(Object obj) { public boolean isIgnoreNulls() { return ignoreNulls; } + + public void serialize(ByteBuffer byteBuffer) { + resolvedFunction.serialize(byteBuffer); + ReadWriteIOUtils.write(arguments.size(), byteBuffer); + arguments.forEach(argument -> Expression.serialize(argument, byteBuffer)); + frame.serialize(byteBuffer); + ReadWriteIOUtils.write(ignoreNulls, byteBuffer); + } + + public void serialize(DataOutputStream stream) throws IOException { + resolvedFunction.serialize(stream); + ReadWriteIOUtils.write(arguments.size(), stream); + for (Expression argument : arguments) { + Expression.serialize(argument, stream); + } + frame.serialize(stream); + ReadWriteIOUtils.write(ignoreNulls, stream); + } + + public Function(ByteBuffer buffer) { + resolvedFunction = ResolvedFunction.deserialize(buffer); + int size = ReadWriteIOUtils.readInt(buffer); + arguments = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + arguments.add(Expression.deserialize(buffer)); + } + frame = new Frame(buffer); + ignoreNulls = ReadWriteIOUtils.readBool(buffer); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java index 2296a19e8db0b..f204106a41085 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -106,6 +106,16 @@ public boolean shallowEquals(Node other) { return type == otherNode.type; } + public void serialize(ByteBuffer buffer) throws IOException { + ReadWriteIOUtils.write((byte) type.ordinal(), buffer); + if (value.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + Expression.serialize(value.get(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + } + public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((byte) type.ordinal(), stream); if (value.isPresent()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java index 37165fc7e4e7f..b05f6bb3438f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java @@ -107,6 +107,17 @@ public boolean shallowEquals(Node other) { return type == otherNode.type; } + public void serialize(ByteBuffer buffer) throws IOException { + ReadWriteIOUtils.write((byte) type.ordinal(), buffer); + start.serialize(buffer); + if (end.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + end.get().serialize(buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + } + public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write((byte) type.ordinal(), stream); start.serialize(stream); From 03145ef955ac0dfc5fa8add2c1881abe8535cd77 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:02:34 +0800 Subject: [PATCH 50/54] Add UT for window node serialize/deserialize. --- .../node/process/WindowNodeSerdeTest.java | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java new file mode 100644 index 0000000000000..680444e429613 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java @@ -0,0 +1,99 @@ +package org.apache.iotdb.db.queryengine.plan.planner.node.process; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.queryengine.plan.planner.node.PlanNodeDeserializeHelper; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.function.BoundSignature; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionId; +import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionKind; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.FunctionNullability; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.planner.DataOrganizationSpecification; +import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.SortOrder; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.WindowNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.DoubleType; +import org.apache.tsfile.read.common.type.LongType; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +public class WindowNodeSerdeTest { + @Test + public void testSerializeAndDeserialize() throws IllegalPathException, IOException { + // Construct other parameters + SeriesScanNode child = + new SeriesScanNode( + new PlanNodeId("TestSeriesScanNode"), + new MeasurementPath("root.sg.d1.s1", TSDataType.INT32), + Ordering.DESC, + null, + 100, + 100, + null); + PlanNodeId nodeId = new PlanNodeId("testWindowFunctionNode"); + DataOrganizationSpecification specification = getDataOrganizationSpecification(); + Optional hashSymbol = Optional.of(new Symbol("hash_col")); + + // Construct window function + BoundSignature signature = + new BoundSignature("count", LongType.INT64, Collections.singletonList(DoubleType.DOUBLE)); + FunctionId functionId = new FunctionId("count_id"); + FunctionKind functionKind = FunctionKind.AGGREGATE; + FunctionNullability nullability = FunctionNullability.getAggregationFunctionNullability(1); + ResolvedFunction resolvedFunction = + new ResolvedFunction(signature, functionId, functionKind, true, nullability); + + WindowNode.Frame frame = WindowNode.Frame.DEFAULT_FRAME; + Expression arguments = new Identifier("col1"); + WindowNode.Function function = + new WindowNode.Function( + resolvedFunction, Collections.singletonList(arguments), frame, true); + + Symbol symbol = new Symbol("cnt"); + HashMap map = new HashMap<>(); + map.put(symbol, function); + + WindowNode windowNode = + new WindowNode(nodeId, child, specification, map, hashSymbol, new HashSet<>(), 0); + + // Serialize and deserialize the node + ByteBuffer buffer = ByteBuffer.allocate(8196); + windowNode.serialize(buffer); + buffer.flip(); + PlanNode deserialized = PlanNodeDeserializeHelper.deserialize(buffer); + assertEquals(windowNode, deserialized); + } + + private static DataOrganizationSpecification getDataOrganizationSpecification() { + // Partition By + ImmutableList partitionBy = ImmutableList.of(new Symbol("col1"), new Symbol("col2")); + + // Order By + Symbol col3 = new Symbol("col3"); + List orderBy = Collections.singletonList(col3); + Map orderings = Collections.singletonMap(col3, SortOrder.ASC_NULLS_LAST); + Optional orderingScheme = Optional.of(new OrderingScheme(orderBy, orderings)); + + return new DataOrganizationSpecification(partitionBy, orderingScheme); + } +} From e8c3ad8c7cf6eaec32dcb90ec1ae2eb1894e1027 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:02:57 +0800 Subject: [PATCH 51/54] Add license header to new UT. --- .../node/process/WindowNodeSerdeTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java index 680444e429613..baf95593e3730 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/process/WindowNodeSerdeTest.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.plan.planner.node.process; import org.apache.iotdb.commons.exception.IllegalPathException; From 0013c1e2ab99bc7b848a776078c0aad8ff11f10f Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:44:22 +0800 Subject: [PATCH 52/54] Perhaps last mvn spotless apply. --- .../plan/planner/TableOperatorGenerator.java | 12 ++++++------ .../plan/relational/analyzer/Analysis.java | 1 - .../plan/relational/planner/QueryPlanner.java | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index b28ac913d7e2a..9b976f7f9df08 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -82,12 +82,6 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.SingleColumnMerger; import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.comparator.JoinKeyComparatorFactory; import org.apache.iotdb.db.queryengine.execution.operator.process.last.LastQueryUtil; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperator; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunction; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunctionFactory; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.AggregationWindowFunction; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.WindowAggregator; -import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.frame.FrameInfo; import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.LogicalIndexNavigation; import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.PatternVariableRecognizer; import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.PhysicalValueAccessor; @@ -97,6 +91,12 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.IrRowPatternToProgramRewriter; import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.Matcher; import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.Program; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.TableWindowOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.WindowFunctionFactory; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.AggregationWindowFunction; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.function.aggregate.WindowAggregator; +import org.apache.iotdb.db.queryengine.execution.operator.process.window.partition.frame.FrameInfo; import org.apache.iotdb.db.queryengine.execution.operator.schema.CountMergeOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaCountOperator; import org.apache.iotdb.db.queryengine.execution.operator.schema.SchemaQueryScanOperator; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index ecf3fb487a7f2..256d2150f3d88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -1302,7 +1302,6 @@ public Optional getSubqueryCoercion() { } } -<<<<<<< HEAD public void addWindowDefinition( QuerySpecification query, CanonicalizationAware name, ResolvedWindow window) { windowDefinitions diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 55ed491c72d8f..4df3d4f6562a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -61,8 +61,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.VariableDefinition; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WindowFrame; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; From 771056b75a675fc5e8c2c4ab5a02a62037fa03c0 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 17:57:01 +0800 Subject: [PATCH 53/54] Fix after compilation. --- .../process/window/utils/RowComparator.java | 1 - .../analyzer/ExpressionAnalyzer.java | 3 ++- .../plan/relational/sql/ast/FunctionCall.java | 23 +++++++++++++++++++ .../plan/relational/sql/ast/Identifier.java | 6 ----- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index 193a09e55a1cc..cd0c2ffdcff58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -90,7 +90,6 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse break; case STRING: case TEXT: - case STRING: case BLOB: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 5ddc3db288b22..92db153ea9f4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -2240,7 +2240,8 @@ public static ExpressionAnalysis analyzePatternRecognitionExpression( analyzer.getSubqueries(), analyzer.getExistsSubqueries(), analyzer.getColumnReferences(), - analyzer.getQuantifiedComparisons()); + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions()); } public static ExpressionAnalysis analyzeExpressions( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 0cb6649721e6c..8c3ad1b1971f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -64,6 +64,8 @@ public FunctionCall( QualifiedName name, Optional processingMode, List arguments) { super(null); this.name = requireNonNull(name, "name is null"); + this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); this.distinct = false; this.processingMode = requireNonNull(processingMode, "processingMode is null"); this.arguments = requireNonNull(arguments, "arguments is null"); @@ -76,6 +78,8 @@ public FunctionCall( List arguments) { super(null); this.name = requireNonNull(name, "name is null"); + this.window = Optional.empty(); + this.nullTreatment = Optional.empty(); this.distinct = distinct; this.processingMode = requireNonNull(processingMode, "processingMode is null"); this.arguments = requireNonNull(arguments, "arguments is null"); @@ -113,6 +117,25 @@ public FunctionCall( this.window = window; this.nullTreatment = nullTreatment; this.distinct = distinct; + this.processingMode = Optional.empty(); + this.arguments = arguments; + } + + public FunctionCall( + NodeLocation location, + QualifiedName name, + Optional window, + Optional nullTreatment, + boolean distinct, + Optional processingMode, + List arguments) { + super(requireNonNull(location, "location is null")); + + this.name = name; + this.window = window; + this.nullTreatment = nullTreatment; + this.distinct = distinct; + this.processingMode = requireNonNull(processingMode, "processingMode is null"); this.arguments = arguments; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java index 999c6e1c4012c..3d3a260bc0fdf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Identifier.java @@ -166,12 +166,6 @@ public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(this.delimited, stream); } - @Override - public void serialize(ByteBuffer buffer) { - ReadWriteIOUtils.write(this.value, buffer); - ReadWriteIOUtils.write(this.delimited, buffer); - } - public Identifier(ByteBuffer byteBuffer) { super(null); this.value = ReadWriteIOUtils.readString(byteBuffer); From 5224b64ada7683d20a1b7ecdd06f4677013f2340 Mon Sep 17 00:00:00 2001 From: Sh-Zh-7 Date: Fri, 30 May 2025 19:58:13 +0800 Subject: [PATCH 54/54] Fix newly come up UT error. --- .../plan/relational/sql/ast/FrameBound.java | 2 +- .../plan/relational/sql/ast/FunctionCall.java | 22 ++++++++++++-- .../plan/relational/sql/ast/OrderBy.java | 7 +++++ .../plan/relational/sql/ast/SortItem.java | 6 ++++ .../plan/relational/sql/ast/Window.java | 3 ++ .../plan/relational/sql/ast/WindowFrame.java | 2 +- .../relational/sql/ast/WindowReference.java | 5 ++++ .../sql/ast/WindowSpecification.java | 29 +++++++++++++++++++ 8 files changed, 72 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java index f204106a41085..04aa993e99d86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FrameBound.java @@ -106,7 +106,7 @@ public boolean shallowEquals(Node other) { return type == otherNode.type; } - public void serialize(ByteBuffer buffer) throws IOException { + public void serialize(ByteBuffer buffer) { ReadWriteIOUtils.write((byte) type.ordinal(), buffer); if (value.isPresent()) { ReadWriteIOUtils.write((byte) 1, buffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 8c3ad1b1971f3..0ef3e51adebad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -222,14 +222,32 @@ public TableExpressionType getExpressionType() { @Override public void serialize(ByteBuffer buffer) { this.name.serialize(buffer); - ReadWriteIOUtils.write(this.distinct, buffer); - ReadWriteIOUtils.write(arguments.size(), buffer); for (Expression argument : arguments) { Expression.serialize(argument, buffer); } + if (nullTreatment.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + ReadWriteIOUtils.write((byte) nullTreatment.get().ordinal(), buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + + if (window.isPresent()) { + ReadWriteIOUtils.write((byte) 1, buffer); + // Window type + if (window.get() instanceof WindowReference) { + ReadWriteIOUtils.write((byte) 0, buffer); + } else { + ReadWriteIOUtils.write((byte) 1, buffer); + } + window.get().serialize(buffer); + } else { + ReadWriteIOUtils.write((byte) 0, buffer); + } + if (processingMode.isPresent()) { ReadWriteIOUtils.write(true, buffer); ProcessingMode mode = processingMode.get(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java index 31a0d1acadfa3..56bb95bac2a52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OrderBy.java @@ -92,6 +92,13 @@ public boolean shallowEquals(Node other) { return sameClass(this, other); } + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(sortItems.size(), buffer); + for (SortItem sortItem : sortItems) { + sortItem.serialize(buffer); + } + } + public void serialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(sortItems.size(), stream); for (SortItem sortItem : sortItems) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java index df9fe6d4559f1..f32b5ee356a6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SortItem.java @@ -124,6 +124,12 @@ public boolean shallowEquals(Node other) { return ordering == otherItem.ordering && nullOrdering == otherItem.nullOrdering; } + void serialize(ByteBuffer buffer) { + Expression.serialize(sortKey, buffer); + ReadWriteIOUtils.write((byte) ordering.ordinal(), buffer); + ReadWriteIOUtils.write((byte) nullOrdering.ordinal(), buffer); + } + void serialize(DataOutputStream stream) throws IOException { Expression.serialize(sortKey, stream); ReadWriteIOUtils.write((byte) ordering.ordinal(), stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java index b9c34a9496cf6..edbe81408fa84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Window.java @@ -21,7 +21,10 @@ import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; public interface Window { + void serialize(ByteBuffer byteBuffer); + void serialize(DataOutputStream stream) throws IOException; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java index b05f6bb3438f3..6cb790f99959f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowFrame.java @@ -107,7 +107,7 @@ public boolean shallowEquals(Node other) { return type == otherNode.type; } - public void serialize(ByteBuffer buffer) throws IOException { + public void serialize(ByteBuffer buffer) { ReadWriteIOUtils.write((byte) type.ordinal(), buffer); start.serialize(buffer); if (end.isPresent()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java index dcf91c59aefc1..6f54fee59ba54 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowReference.java @@ -79,6 +79,11 @@ public boolean shallowEquals(Node other) { return sameClass(this, other); } + @Override + public void serialize(ByteBuffer byteBuffer) { + name.serialize(byteBuffer); + } + @Override public void serialize(DataOutputStream stream) throws IOException { name.serialize(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java index 39e42a3fbab65..6982a46aea972 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WindowSpecification.java @@ -118,6 +118,35 @@ public boolean shallowEquals(Node other) { return sameClass(this, other); } + @Override + public void serialize(ByteBuffer byteBuffer) { + if (existingWindowName.isPresent()) { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + existingWindowName.get().serialize(byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } + + ReadWriteIOUtils.write(partitionBy.size(), byteBuffer); + for (Expression expression : partitionBy) { + Expression.serialize(expression, byteBuffer); + } + + if (orderBy.isPresent()) { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + orderBy.get().serialize(byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } + + if (frame.isPresent()) { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + frame.get().serialize(byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } + } + @Override public void serialize(DataOutputStream stream) throws IOException { if (existingWindowName.isPresent()) {