From 1797bea66ba94b19d7628dc78c12c770add49fd4 Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Fri, 18 Jun 2021 16:31:25 +0800 Subject: [PATCH 1/9] add ComputeManager --- .../core/graph/value/IdValueList.java | 9 + .../computer/core/output/ComputerOutput.java | 0 .../computer/core/compute/ComputeManager.java | 144 +++++++ .../core/compute/FileGraphPartition.java | 373 ++++++++++++++++++ .../core/compute/input/EdgesInput.java | 253 ++++++++++++ .../core/compute/input/EmptyEdges.java | 55 +++ .../core/compute/input/MessageInput.java | 112 ++++++ .../core/compute/input/ReusablePointer.java | 101 +++++ .../core/compute/input/VertexInput.java | 86 ++++ .../computer/core/config/ComputerOptions.java | 10 + .../core/graph/partition/PartitionStat.java | 8 + .../computer/core/output/LogOutput.java | 56 +++ .../core/receiver/MessageRecvManager.java | 23 +- .../core/receiver/MessageRecvPartition.java | 4 + .../core/receiver/MessageRecvPartitions.java | 10 +- .../computer/core/receiver/RecvStat.java | 43 ++ .../store/hgkvfile/entry/InlinePointer.java | 3 +- .../computer/core/worker/WorkerService.java | 43 +- .../core/compute/ComputeManagerTest.java | 248 ++++++++++++ .../core/compute/ComputeTestSuite.java | 37 ++ .../core/compute/MockComputation.java | 85 ++++ .../core/compute/MockMessageSender.java | 41 ++ .../core/compute/input/EdgesInputTest.java | 300 ++++++++++++++ .../core/compute/input/EmptyEdgesTest.java | 45 +++ .../core/compute/input/MessageInputTest.java | 177 +++++++++ .../graph/partition/PartitionStatTest.java | 22 ++ .../computer/core/receiver/ReceiverUtil.java | 19 + .../ComputeMessageRecvPartitionTest.java | 27 +- .../computer/suite/unit/UnitTestSuite.java | 4 +- 29 files changed, 2266 insertions(+), 72 deletions(-) rename {computer-core => computer-api}/src/main/java/com/baidu/hugegraph/computer/core/output/ComputerOutput.java (100%) create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java create mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManagerTest.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockComputation.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockMessageSender.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java diff --git a/computer-api/src/main/java/com/baidu/hugegraph/computer/core/graph/value/IdValueList.java b/computer-api/src/main/java/com/baidu/hugegraph/computer/core/graph/value/IdValueList.java index 99b9a26db..68a603523 100644 --- a/computer-api/src/main/java/com/baidu/hugegraph/computer/core/graph/value/IdValueList.java +++ b/computer-api/src/main/java/com/baidu/hugegraph/computer/core/graph/value/IdValueList.java @@ -44,4 +44,13 @@ public void read(RandomAccessInput in) throws IOException { public void write(RandomAccessOutput out) throws IOException { this.write(out, false); } + + @Override + public IdValueList copy() { + IdValueList values = new IdValueList(); + for (IdValue value : this.values()) { + values.add(value); + } + return values; + } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/ComputerOutput.java b/computer-api/src/main/java/com/baidu/hugegraph/computer/core/output/ComputerOutput.java similarity index 100% rename from computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/ComputerOutput.java rename to computer-api/src/main/java/com/baidu/hugegraph/computer/core/output/ComputerOutput.java diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java new file mode 100644 index 000000000..c1e77b793 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java @@ -0,0 +1,144 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; + +import com.baidu.hugegraph.computer.core.common.ComputerContext; +import com.baidu.hugegraph.computer.core.graph.partition.PartitionStat; +import com.baidu.hugegraph.computer.core.graph.value.Value; +import com.baidu.hugegraph.computer.core.manager.Managers; +import com.baidu.hugegraph.computer.core.network.message.MessageType; +import com.baidu.hugegraph.computer.core.receiver.MessageRecvManager; +import com.baidu.hugegraph.computer.core.receiver.RecvStat; +import com.baidu.hugegraph.computer.core.sender.MessageSendManager; +import com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; +import com.baidu.hugegraph.computer.core.worker.Computation; +import com.baidu.hugegraph.computer.core.worker.ComputationContext; +import com.baidu.hugegraph.computer.core.worker.WorkerStat; +import com.baidu.hugegraph.util.Log; + +public class ComputeManager> { + + private static final Logger LOG = Log.logger(ComputeManager.class); + + private ComputerContext context; + private final Managers managers; + + private final Map> partitions; + private final Computation computation; + private final MessageRecvManager messageRecvManager; + private final MessageSendManager sendManager; + + public ComputeManager(ComputerContext context, Managers managers, + Computation computation) { + this.context = context; + this.managers = managers; + this.computation = computation; + this.partitions = new HashMap<>(); + this.messageRecvManager = this.managers.get(MessageRecvManager.NAME); + this.sendManager = this.managers.get(MessageSendManager.NAME); + } + + public WorkerStat input() { + WorkerStat workerStat = new WorkerStat(); + MessageRecvManager messageRecvManager = this.managers.get( + MessageRecvManager.NAME); + messageRecvManager.waitReceivedAllMessages(); + Map> vertices; + vertices = messageRecvManager.vertexPartitions(); + Map> edges; + edges = messageRecvManager.edgePartitions(); + // TODO: parallel input process + for (Map.Entry> entry : + vertices.entrySet()) { + FileGraphPartition partition = new FileGraphPartition<>( + this.context, this.managers, + entry.getKey()); + PartitionStat partitionStat = partition.init(entry.getValue(), + edges.get(entry.getKey())); + workerStat.add(partitionStat); + this.partitions.put(entry.getKey(), partition); + } + return workerStat; + } + + /** + * Get compute-messages from MessageRecvManager. Be called before + * {@link MessageRecvManager#beforeSuperstep} is called. + */ + public void takeComputeMessages() { + Map> messages; + messages = this.messageRecvManager.messagePartitions(); + for (FileGraphPartition partition : this.partitions.values()) { + PeekableIterator m = messages.get(partition.partition()); + if (m == null) { + m = PeekableIterator.emptyIterator(); + } + partition.messages(m); + } + } + + public WorkerStat compute(ComputationContext context, int superstep) { + this.sendManager.startSend(MessageType.MSG); + WorkerStat workerStat = new WorkerStat(); + Map partitionStats = new HashMap<>( + this.partitions.size()); + if (superstep == 0) { + // TODO: parallel compute process. + for (FileGraphPartition partition : this.partitions.values()) { + PartitionStat stat = partition.compute0(context, + this.computation); + partitionStats.put(stat.partitionId(), stat); + } + } else { + // TODO: parallel compute process. + for (FileGraphPartition partition : this.partitions.values()) { + PartitionStat stat = partition.compute(context, + this.computation, + superstep); + partitionStats.put(stat.partitionId(), stat); + } + } + this.sendManager.finishSend(MessageType.MSG); + // After compute and send finish signal. + Map recvStats = this.messageRecvManager.recvStats(); + for (Map.Entry entry : + partitionStats.entrySet()) { + PartitionStat partitionStat = entry.getValue(); + partitionStat.merge(recvStats.get(partitionStat.partitionId())); + workerStat.add(partitionStat); + } + return workerStat; + } + + public void output() { + // TODO: Write results back parallel + for (FileGraphPartition partition : this.partitions.values()) { + PartitionStat stat = partition.output(); + LOG.info("Output partition {} complete, stat='{}'", + partition.partition(), stat); + } + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java new file mode 100644 index 000000000..b89f3959e --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java @@ -0,0 +1,373 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import com.baidu.hugegraph.computer.core.common.ComputerContext; +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.compute.input.EdgesInput; +import com.baidu.hugegraph.computer.core.compute.input.MessageInput; +import com.baidu.hugegraph.computer.core.compute.input.VertexInput; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; +import com.baidu.hugegraph.computer.core.graph.partition.PartitionStat; +import com.baidu.hugegraph.computer.core.graph.value.Value; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.computer.core.io.BufferedFileInput; +import com.baidu.hugegraph.computer.core.io.BufferedFileOutput; +import com.baidu.hugegraph.computer.core.manager.Managers; +import com.baidu.hugegraph.computer.core.output.ComputerOutput; +import com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator; +import com.baidu.hugegraph.computer.core.store.FileGenerator; +import com.baidu.hugegraph.computer.core.store.FileManager; +import com.baidu.hugegraph.computer.core.store.hgkvfile.buffer.EntryIterator; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntriesUtil; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.Pointer; +import com.baidu.hugegraph.computer.core.worker.Computation; +import com.baidu.hugegraph.computer.core.worker.ComputationContext; +import com.baidu.hugegraph.util.E; + +public class FileGraphPartition> { + + private final ComputerContext context; + private static final String VERTEX = "vertex"; + private static final String EDGE = "edge"; + private static final String STATUS = "status"; + private static final String VALUE = "value"; + + private final FileGenerator fileGenerator; + private final int partition; + + private File vertexFile; + private File edgeFile; + private File preStatusFile; + private File curStatusFile; + private File preValueFile; + private File curValueFile; + + private long vertexCount; + private long edgeCount; + + private BufferedFileOutput curStatusOut; + private BufferedFileOutput curValueOut; + private BufferedFileInput preStatusIn; + private BufferedFileInput preValueIn; + private VertexInput vertexInput; + private EdgesInput edgesInput; + private MessageInput messageInput; + private final Value result; + + public FileGraphPartition(ComputerContext context, Managers managers, + int partition) { + this.context = context; + this.fileGenerator = managers.get(FileManager.NAME); + this.partition = partition; + this.vertexFile = new File(this.fileGenerator.randomDirectory(VERTEX)); + this.edgeFile = new File(this.fileGenerator.randomDirectory(EDGE)); + this.result = this.context.config().createObject( + ComputerOptions.ALGORITHM_RESULT_CLASS); + this.vertexCount = 0L; + this.edgeCount = 0L; + } + + public PartitionStat init(PeekableIterator vertices, + PeekableIterator edges) { + if (edges == null) { + edges = PeekableIterator.emptyIterator(); + } + try { + createFile(this.vertexFile); + createFile(this.edgeFile); + BufferedFileOutput vertexOut = new BufferedFileOutput( + this.vertexFile); + BufferedFileOutput edgeOut = new BufferedFileOutput(this.edgeFile); + while (vertices.hasNext()) { + KvEntry entry = vertices.next(); + Pointer key = entry.key(); + vertexOut.writeFixedInt(key.bytes().length); + vertexOut.write(key.bytes()); + Pointer value = entry.value(); + vertexOut.writeFixedInt(value.bytes().length); + vertexOut.write(value.bytes()); + this.writeEdges(key, edges, edgeOut); + this.vertexCount++; + } + vertexOut.close(); + edgeOut.close(); + } catch (IOException e) { + e.printStackTrace(); + throw new ComputerException("Init vertex input failed", e); + } + + return new PartitionStat(this.partition, this.vertexCount, + this.edgeCount); + } + + // Package access + PartitionStat compute0(ComputationContext context, + Computation computation) { + long activeVertexCount = 0; + this.beforeCompute(0); + + while (this.vertexInput.hasNext()) { + Vertex vertex = this.vertexInput.next(); + Edges edges = this.edgesInput.edges(this.vertexInput.idPointer()); + vertex.edges(edges); + computation.compute0(context, vertex); + if (vertex.active()) { + activeVertexCount++; + } + this.saveVertex(vertex); + } + this.afterCompute(0); + + return new PartitionStat(this.partition, this.vertexCount, + this.edgeCount, + this.vertexCount - activeVertexCount, 0L, 0L); + } + + // Package access + PartitionStat compute(ComputationContext context, + Computation computation, + int superstep) { + this.beforeCompute(superstep); + long activeVertexCount = 0L; + while (this.vertexInput.hasNext()) { + Vertex vertex = this.vertexInput.next(); + Iterator messageIt = this.messageInput.iterator( + this.vertexInput.idPointer()); + try { + boolean status = this.preStatusIn.readBoolean(); + this.result.read(this.preValueIn); + vertex.value(this.result); + if (status || messageIt.hasNext()) { + vertex.reactivate(); + } else { + vertex.inactivate(); + } + } catch (IOException e) { + throw new ComputerException("Can't read status or result", e); + } + + /* + * If the vertex is inactive, it's edges will be skipped + * automatically at the next vertex. + */ + if (vertex.active()) { + Edges edges = this.edgesInput.edges( + this.vertexInput.idPointer()); + vertex.edges(edges); + computation.compute(context, vertex, messageIt); + } + + // Computation may change vertex status. + if (vertex.active()) { + activeVertexCount++; + } + this.saveVertex(vertex); + } + this.afterCompute(superstep); + return new PartitionStat(this.partition, this.vertexCount, + this.edgeCount, + this.vertexCount - activeVertexCount, 0L, 0L); + } + + public PartitionStat output() { + ComputerOutput output = this.context.config().createObject( + ComputerOptions.OUTPUT_CLASS); + output.init(this.context.config(), this.partition); + this.beforeOutput(); + while (this.vertexInput.hasNext()) { + Vertex vertex = this.vertexInput.next(); + try { + boolean status = this.preStatusIn.readBoolean(); + this.result.read(this.preValueIn); + vertex.value(this.result); + if (status) { + vertex.reactivate(); + } else { + vertex.inactivate(); + } + } catch (IOException e) { + throw new ComputerException("Can't read status or result", e); + } + + Edges edges = this.edgesInput.edges(this.vertexInput.idPointer()); + vertex.edges(edges); + output.write(vertex); + } + this.afterOutput(); + output.close(); + return new PartitionStat(this.partition, this.vertexCount, + this.edgeCount); + } + + public void messages(PeekableIterator messages) { + this.messageInput = new MessageInput<>(this.context, messages); + } + + public int partition() { + return this.partition; + } + + private void saveVertex(Vertex vertex) { + try { + this.curStatusOut.writeBoolean(vertex.active()); + Value value = vertex.value(); + E.checkNotNull(value, "Vertex's value can't be null"); + vertex.value().write(this.curValueOut); + } catch (IOException e) { + throw new ComputerException("Save vertex failed, vertex id {}", + vertex.id()); + } + } + + private void writeEdges(Pointer id, PeekableIterator edges, + BufferedFileOutput edgeOut) + throws IOException { + while (edges.hasNext()) { + KvEntry entry = edges.peek(); + Pointer key = entry.key(); + int status = id.compareTo(key); + if (status < 0) { + return; + } else if (status == 0) { + edges.next(); + edgeOut.writeFixedInt(id.bytes().length); + edgeOut.write(id.bytes()); + + long valuePosition = edgeOut.position(); + edgeOut.writeFixedInt(0); + + this.edgeCount += entry.numSubEntries(); + edgeOut.writeFixedInt((int) entry.numSubEntries()); + EntryIterator subKvIt = EntriesUtil.subKvIterFromEntry(entry); + while (subKvIt.hasNext()) { + KvEntry subEntry = subKvIt.next(); + // Not write sub-key length + edgeOut.write(subEntry.key().bytes()); + // Not write sub-value length + edgeOut.write(subEntry.value().bytes()); + } + int valueLength = (int) (edgeOut.position() - valuePosition - + Constants.INT_LEN); + edgeOut.writeFixedInt(valuePosition, valueLength); + } else { // status > 0 + edges.next(); + } + } + } + + private void beforeCompute(int superstep) { + this.vertexInput = new VertexInput(this.context, + this.vertexFile, + this.vertexCount); + this.edgesInput = new EdgesInput(this.context, this.edgeFile); + // Inputs + try { + this.vertexInput.init(); + this.edgesInput.init(); + if (superstep != 0) { + this.preStatusFile = this.curStatusFile; + this.preValueFile = this.curValueFile; + this.preStatusIn = new BufferedFileInput(this.preStatusFile); + this.preValueIn = new BufferedFileInput(this.preValueFile); + } + + // Outputs + String statusPath = this.fileGenerator.randomDirectory( + STATUS, Integer.toString(superstep), + Integer.toString(this.partition)); + String valuePath = this.fileGenerator.randomDirectory( + VALUE, Integer.toString(superstep), + Integer.toString(this.partition)); + this.curStatusFile = new File(statusPath); + this.curValueFile = new File(valuePath); + createFile(this.curStatusFile); + createFile(this.curValueFile); + + this.curStatusOut = new BufferedFileOutput(this.curStatusFile); + this.curValueOut = new BufferedFileOutput(this.curValueFile); + + } catch (IOException e) { + throw new ComputerException("Before compute call failed", e); + } + } + + private void afterCompute(int superstep) { + try { + this.vertexInput.close(); + this.edgesInput.close(); + if (superstep != 0) { + this.preStatusIn.close(); + this.preValueIn.close(); + this.preStatusFile.delete(); + this.preValueFile.delete(); + } + this.curStatusOut.close(); + this.curValueOut.close(); + } catch (IOException e) { + throw new ComputerException("After compute call failed", e); + } + } + + private void beforeOutput() { + this.vertexInput = new VertexInput(this.context, + this.vertexFile, + this.vertexCount); + this.edgesInput = new EdgesInput(this.context, this.edgeFile); + try { + this.vertexInput.init(); + this.edgesInput.init(); + + this.preStatusFile = this.curStatusFile; + this.preValueFile = this.curValueFile; + this.preStatusIn = new BufferedFileInput(this.preStatusFile); + this.preValueIn = new BufferedFileInput(this.preValueFile); + } catch (IOException e) { + throw new ComputerException("Before output call failed", e); + } + } + + private void afterOutput() { + try { + this.vertexInput.close(); + this.edgesInput.close(); + this.preStatusIn.close(); + this.preValueIn.close(); + this.preStatusFile.delete(); + this.preValueFile.delete(); + this.vertexInput.close(); + this.edgesInput.close(); + } catch (IOException e) { + throw new ComputerException("After output call failed", e); + } + } + + private static void createFile(File file) throws IOException { + file.getParentFile().mkdirs(); + file.createNewFile(); + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java new file mode 100644 index 000000000..9db003fac --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java @@ -0,0 +1,253 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import javax.annotation.Nonnull; + +import com.baidu.hugegraph.computer.core.common.ComputerContext; +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.config.EdgeFrequency; +import com.baidu.hugegraph.computer.core.graph.GraphFactory; +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; +import com.baidu.hugegraph.computer.core.graph.properties.Properties; +import com.baidu.hugegraph.computer.core.io.BufferedFileInput; +import com.baidu.hugegraph.computer.core.io.RandomAccessInput; +import com.baidu.hugegraph.computer.core.io.StreamGraphInput; + +public class EdgesInput { + + private RandomAccessInput input; + private final ReusablePointer idPointer; + private final ReusablePointer valuePointer; + private final File edgeFile; + private final GraphFactory graphFactory; + private final int flushThreshold; + private final EdgeFrequency frequency; + + public EdgesInput(ComputerContext context, File edgeFile) { + this.graphFactory = context.graphFactory(); + this.idPointer = new ReusablePointer(); + this.valuePointer = new ReusablePointer(); + this.edgeFile = edgeFile; + this.flushThreshold = context.config().get( + ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX); + this.frequency = context.config().get(ComputerOptions.INPUT_EDGE_FREQ); + } + + public void init() throws IOException { + this.input = new BufferedFileInput(this.edgeFile); + } + + public void close() throws IOException { + this.input.close(); + } + + public Edges edges(ReusablePointer vidPointer) { + try { + while (this.input.available() > 0) { + long startPosition = this.input.position(); + this.idPointer.read(this.input); + int status = vidPointer.compareTo(this.idPointer); + if (status < 0) { // No edges + /* + * The current batch belong to vertex that vertex id is + * bigger than specified id. + */ + this.input.seek(startPosition); + return EmptyEdges.instance(); + } else if (status == 0) { // Has edges + this.valuePointer.read(this.input); + Edges edges = this.readEdges(this.valuePointer.input()); + if (edges.size() < this.flushThreshold) { + return edges; + } else { + return new SuperEdges(vidPointer, edges, startPosition); + } + } else { + /* + * The current batch belong to vertex that vertex id is + * smaller than specified id. + */ + int valueLength = this.input.readFixedInt(); + this.input.skip(valueLength); + } + } + return EmptyEdges.instance(); + } catch (IOException e) { + throw new ComputerException("Can't read from input", e); + } + } + + private class SuperEdges implements Edges { + + private ReusablePointer vid; + private final long startPosition; + private Iterator currentEdgeIt; + private int size; + private int iterationTime; + + SuperEdges(ReusablePointer vid, + Edges edges, + long startPosition) { + this.vid = vid; + this.startPosition = startPosition; + this.currentEdgeIt = edges.iterator(); + this.size = 0; + this.iterationTime = 0; + } + + @Override + public int size() { + if (this.size == 0) { + this.computeSize(); + } + return this.size; + } + + private void computeSize() { + long currentPosition = EdgesInput.this.input.position(); + try { + EdgesInput.this.input.seek(this.startPosition); + EdgesInput.this.idPointer.read(EdgesInput.this.input); + while (EdgesInput.this.idPointer.compareTo(this.vid) == 0) { + long valueLength = EdgesInput.this.input.readFixedInt(); + this.size += EdgesInput.this.input.readInt(); + long bytesToSkip = valueLength - Constants.INT_LEN; + EdgesInput.this.input.skip(bytesToSkip); + if (EdgesInput.this.input.available() > 0) { + EdgesInput.this.idPointer.read(EdgesInput.this.input); + } else { + break; + } + } + EdgesInput.this.input.seek(currentPosition); + } catch (IOException e) { + throw new ComputerException("Compute size of edges failed", e); + } + } + + @Override + public void add(Edge edge) { + throw new ComputerException("Can't add edge"); + } + + @Override + @Nonnull + public Iterator iterator() { + try { + if (this.iterationTime != 0) { + EdgesInput.this.input.seek(this.startPosition); + } + this.iterationTime++; + } catch (IOException e) { + throw new ComputerException("Can't seek to %s", + e, this.startPosition); + } + return new Iterator() { + + @Override + public boolean hasNext() { + if (currentEdgeIt.hasNext()) { + return true; + } else { + long currentPosition = input.position(); + try { + if (input.available() > 0) { + idPointer.read(input); + if (idPointer.compareTo(vid) == 0) { + valuePointer.read(EdgesInput.this.input); + currentEdgeIt = readEdges( + valuePointer.input()) + .iterator(); + } else { + input.seek(currentPosition); + } + } + } catch (IOException e) { + throw new ComputerException( + "Read Edges from position %s failed", + currentPosition); + } + } + return currentEdgeIt.hasNext(); + } + + @Override + public Edge next() { + return currentEdgeIt.next(); + } + }; + } + } + + // TODO: use one batch of edges to read batches for each vertex. + private Edges readEdges(RandomAccessInput in) { + try { + int count = in.readFixedInt(); + Edges edges = this.graphFactory.createEdges(count); + if (this.frequency == EdgeFrequency.SINGLE) { + for (int i = 0; i < count; i++) { + Edge edge = this.graphFactory.createEdge(); + // Only use targetId as subKey, use props as subValue + edge.targetId(StreamGraphInput.readId(in)); + Properties props = this.graphFactory.createProperties(); + props.read(in); + edges.add(edge); + } + } else if (this.frequency == EdgeFrequency.SINGLE_PER_LABEL) { + for (int i = 0; i < count; i++) { + Edge edge = this.graphFactory.createEdge(); + // Use label + targetId as subKey, use props as subValue + edge.label(in.readUTF()); + edge.targetId(StreamGraphInput.readId(in)); + Properties props = this.graphFactory.createProperties(); + props.read(in); + edge.properties(props); + edges.add(edge); + } + } else { + assert this.frequency == EdgeFrequency.MULTIPLE; + for (int i = 0; i < count; i++) { + Edge edge = this.graphFactory.createEdge(); + /* + * Use label + sortValues + targetId as subKey, + * use properties as subValue + */ + edge.label(in.readUTF()); + edge.name(in.readUTF()); + edge.targetId(StreamGraphInput.readId(in)); + Properties props = this.graphFactory.createProperties(); + props.read(in); + edges.add(edge); + } + } + return edges; + } catch (IOException e) { + throw new ComputerException("Read edges failed", e); + } + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java new file mode 100644 index 000000000..1ac808cef --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.util.Collections; +import java.util.Iterator; + +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; + +public class EmptyEdges implements Edges { + + private static final EmptyEdges INSTANCE = new EmptyEdges(); + + private EmptyEdges() { + // pass + } + + public static EmptyEdges instance() { + return INSTANCE; + } + + @Override + public int size() { + return 0; + } + + @Override + public void add(Edge edge) { + throw new ComputerException("Can't add edge '{}'", edge); + } + + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java new file mode 100644 index 000000000..fe226a7b9 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java @@ -0,0 +1,112 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.baidu.hugegraph.computer.core.common.ComputerContext; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.graph.value.Value; +import com.baidu.hugegraph.computer.core.io.BytesInput; +import com.baidu.hugegraph.computer.core.io.IOFactory; +import com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.Pointer; + +public class MessageInput> { + + private final PeekableIterator messages; + private final T value; + + public MessageInput(ComputerContext context, + PeekableIterator messages) { + if (messages == null) { + this.messages = PeekableIterator.emptyIterator(); + } else { + this.messages = messages; + } + + this.value = context.config().createObject( + ComputerOptions.ALGORITHM_MESSAGE_CLASS); + } + + public Iterator iterator(ReusablePointer vid) { + while (this.messages.hasNext()) { + KvEntry entry = this.messages.peek(); + Pointer key = entry.key(); + int status = vid.compareTo(key); + if (status < 0) { + return Collections.emptyIterator(); + } else if (status == 0) { + break; + } else { + this.messages.next(); + } + } + + return new Iterator() { + + // It indicates whether the value can be returned to client. + boolean valueValid = false; + + @Override + public boolean hasNext() { + if (this.valueValid) { + return true; + } + if (MessageInput.this.messages.hasNext()) { + KvEntry entry = MessageInput.this.messages.peek(); + Pointer key = entry.key(); + int status = vid.compareTo(key); + if (status == 0) { + MessageInput.this.messages.next(); + this.valueValid = true; + try { + BytesInput in = IOFactory.createBytesInput( + entry.value().bytes()); + MessageInput.this.value.read(in); + } catch (IOException e) { + throw new ComputerException("Can't read value", e); + } + return true; + } else { + return false; + } + } else { + return false; + } + } + + @Override + public T next() { + if (this.valueValid) { + this.valueValid = false; + return MessageInput.this.value; + } else { + throw new NoSuchElementException(); + } + } + }; + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java new file mode 100644 index 000000000..fcd235073 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java @@ -0,0 +1,101 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.IOException; + +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.io.IOFactory; +import com.baidu.hugegraph.computer.core.io.RandomAccessInput; +import com.baidu.hugegraph.computer.core.io.RandomAccessOutput; +import com.baidu.hugegraph.computer.core.io.Readable; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.Pointer; +import com.baidu.hugegraph.computer.core.util.BytesUtil; + +public class ReusablePointer implements Pointer, Readable { + + private int length; + private byte[] bytes; + private RandomAccessInput input; + + public ReusablePointer() { + this.bytes = Constants.EMPTY_BYTES; + this.length = 0; + this.input = IOFactory.createBytesInput(this.bytes); + } + + public ReusablePointer(byte[] bytes, int length) { + this.bytes = bytes; + this.length = length; + this.input = IOFactory.createBytesInput(this.bytes); + } + + @Override + public void read(RandomAccessInput in) throws IOException { + this.length = in.readInt(); + if (this.bytes.length < this.length) { + this.bytes = new byte[this.length]; + this.input = IOFactory.createBytesInput(this.bytes); + } + in.readFully(this.bytes, 0, this.length); + } + + @Override + public RandomAccessInput input() { + try { + this.input.seek(0L); + } catch (IOException e) { + throw new ComputerException("Seek to position 0 failed", e); + } + return this.input; + } + + @Override + public byte[] bytes() { + return this.bytes; + } + + @Override + public void write(RandomAccessOutput output) throws IOException { + output.writeFixedInt(this.length); + output.write(this.bytes(), 0, this.length); + } + + @Override + public long offset() { + return 0L; + } + + @Override + public long length() { + return this.length; + } + + @Override + public int compareTo(Pointer other) { + try { + return BytesUtil.compare(this.bytes(), this.length, + other.bytes(), (int) other.length()); + } catch (IOException e) { + throw new ComputerException(e.getMessage(), e); + } + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java new file mode 100644 index 000000000..f511327b8 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java @@ -0,0 +1,86 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.File; +import java.io.IOException; + +import com.baidu.hugegraph.computer.core.common.ComputerContext; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.graph.properties.Properties; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.computer.core.io.BufferedFileInput; +import com.baidu.hugegraph.computer.core.io.RandomAccessInput; +import com.baidu.hugegraph.computer.core.io.StreamGraphInput; + +public class VertexInput { + + private final long vertexCount; + private long readCount; + private RandomAccessInput input; + private final Vertex vertex; + private final ReusablePointer idPointer; + private final ReusablePointer valuePointer; + private final Properties properties; + private final File vertexFile; + + public VertexInput(ComputerContext context, + File vertexFile, + long vertexCount) { + this.vertexFile = vertexFile; + this.vertexCount = vertexCount; + this.readCount = 0L; + this.vertex = context.graphFactory().createVertex(); + this.idPointer = new ReusablePointer(); + this.valuePointer = new ReusablePointer(); + this.properties = context.graphFactory().createProperties(); + this.readCount = 0; + } + + public void init() throws IOException { + this.input = new BufferedFileInput(this.vertexFile); + } + + public void close() throws IOException { + this.input.close(); + } + + public boolean hasNext() { + return this.readCount < this.vertexCount; + } + + public Vertex next() { + this.readCount++; + try { + this.idPointer.read(this.input); + this.valuePointer.read(this.input); + this.properties.read(this.valuePointer.input()); + this.vertex.id(StreamGraphInput.readId(this.idPointer.input())); + this.vertex.properties(this.properties); + } catch (IOException e) { + throw new ComputerException("Can't read from input", e); + } + return this.vertex; + } + + public ReusablePointer idPointer() { + return this.idPointer; + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java index 10741f2d8..b271d4869 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/config/ComputerOptions.java @@ -33,6 +33,7 @@ import com.baidu.hugegraph.computer.core.master.DefaultMasterComputation; import com.baidu.hugegraph.computer.core.network.TransportConf; import com.baidu.hugegraph.computer.core.network.netty.NettyTransportProvider; +import com.baidu.hugegraph.computer.core.output.LogOutput; import com.baidu.hugegraph.config.ConfigConvOption; import com.baidu.hugegraph.config.ConfigListOption; import com.baidu.hugegraph.config.ConfigOption; @@ -165,6 +166,15 @@ public static synchronized ComputerOptions instance() { 4 ); + public static final ConfigOption> OUTPUT_CLASS = + new ConfigOption<>( + "output.output_class", + "The class to output the computation result of each " + + "vertex. Be called after iteration computation.", + disallowEmpty(), + LogOutput.class + ); + public static final ConfigOption OUTPUT_RESULT_NAME = new ConfigOption<>( "output.result_name", diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java index df7338cf2..e12d27919 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java @@ -25,6 +25,7 @@ import com.baidu.hugegraph.computer.core.io.RandomAccessOutput; import com.baidu.hugegraph.computer.core.io.Readable; import com.baidu.hugegraph.computer.core.io.Writable; +import com.baidu.hugegraph.computer.core.receiver.RecvStat; import com.baidu.hugegraph.computer.core.util.JsonUtil; public class PartitionStat implements Readable, Writable { @@ -102,6 +103,13 @@ public void write(RandomAccessOutput out) throws IOException { out.writeLong(this.messageBytes); } + public void merge(RecvStat recvStat) { + if (recvStat != null) { + this.messageCount += recvStat.messageCount(); + this.messageBytes += recvStat.messageBytes(); + } + } + @Override public boolean equals(Object obj) { if (!(obj instanceof PartitionStat)) { diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java new file mode 100644 index 000000000..d7aeebd84 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java @@ -0,0 +1,56 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.output; + +import org.slf4j.Logger; + +import com.baidu.hugegraph.computer.core.config.Config; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.util.Log; + +/** + * LogOutput print the computation result to log file. + * It can't be used on production environment. + * Be used for test or development only. + */ +public class LogOutput implements ComputerOutput { + + private static final Logger LOG = Log.logger(LogOutput.class); + + private int partition; + + @Override + public void init(Config config, int partition) { + this.partition = partition; + LOG.info("begin output partition {}", this.partition); + } + + @Override + public void write(Vertex vertex) { + LOG.info("id='{}', result='{}'", + vertex.id().toString(), + vertex.value().toString()); + } + + @Override + public void close() { + LOG.info("close output partition {}", partition); + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java index 81d226c7f..405c840f6 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java @@ -31,7 +31,6 @@ import com.baidu.hugegraph.computer.core.common.exception.TransportException; import com.baidu.hugegraph.computer.core.config.ComputerOptions; import com.baidu.hugegraph.computer.core.config.Config; -import com.baidu.hugegraph.computer.core.graph.partition.PartitionStat; import com.baidu.hugegraph.computer.core.manager.Manager; import com.baidu.hugegraph.computer.core.network.ConnectionId; import com.baidu.hugegraph.computer.core.network.MessageHandler; @@ -45,7 +44,6 @@ import com.baidu.hugegraph.computer.core.store.FileManager; import com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator; import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; -import com.baidu.hugegraph.computer.core.worker.WorkerStat; import com.baidu.hugegraph.util.E; import com.baidu.hugegraph.util.Log; @@ -115,7 +113,6 @@ public void beforeSuperstep(Config config, int superstep) { @Override public void afterSuperstep(Config config, int superstep) { - // pass } @Override @@ -136,19 +133,6 @@ public void exceptionCaught(TransportException cause, connectionId, cause); } - /** - * Merge vertices and edges in each partition parallel, and get the - * workerStat. Be called at the end of input superstep. - */ - public WorkerStat mergeGraph() { - // TODO: Merge vertices and edges in each partition parallel - PartitionStat stat1 = new PartitionStat(0, 100L, 200L, - 50L, 60L, 70L); - WorkerStat workerStat = new WorkerStat(); - workerStat.add(stat1); - return workerStat; - } - public void waitReceivedAllMessages() { try { boolean status = this.finishMessagesLatch.await( @@ -233,4 +217,11 @@ public Map> messagePartitions() { this.messagePartitions = null; return partitions.iterators(); } + + public Map recvStats() { + this.waitReceivedAllMessages(); + E.checkState(this.messagePartitions != null, + "The messagePartitions can't be null"); + return this.messagePartitions.recvStats(); + } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java index a30b8f885..fa29c4343 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java @@ -115,6 +115,10 @@ public long totalBytes() { return this.totalBytes; } + public RecvStat recvStat() { + return new RecvStat(0L, this.totalBytes); + } + protected abstract OuterSortFlusher outerSortFlusher(); protected abstract String type(); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java index a390eeff9..3406f35b1 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java @@ -50,7 +50,7 @@ public MessageRecvPartitions(ComputerContext context, this.partitions = new HashMap<>(); } - public abstract P createPartition(); + protected abstract P createPartition(); public void addBuffer(int partitionId, ManagedBuffer buffer) { P partition = this.partition(partitionId); @@ -82,4 +82,12 @@ public Map> iterators() { } return entries; } + + public Map recvStats() { + Map entries = new HashMap<>(); + for (Map.Entry entry : this.partitions.entrySet()) { + entries.put(entry.getKey(), entry.getValue().recvStat()); + } + return entries; + } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java new file mode 100644 index 000000000..e65356909 --- /dev/null +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.receiver; + +/** + * Received message stat for a partition. + */ +public class RecvStat { + + private final long messageCount; + + private final long messageBytes; + + public RecvStat(long messageCount, long messageBytes) { + this.messageCount = messageCount; + this.messageBytes = messageBytes; + } + + public long messageCount() { + return this.messageCount; + } + + public long messageBytes() { + return this.messageBytes; + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InlinePointer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InlinePointer.java index 916e4c590..a37dd0385 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InlinePointer.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/store/hgkvfile/entry/InlinePointer.java @@ -71,7 +71,8 @@ public long length() { @Override public int compareTo(Pointer other) { try { - return BytesUtil.compare(this.bytes(), other.bytes()); + return BytesUtil.compare(this.bytes(), (int) this.length, + other.bytes(), (int) other.length()); } catch (IOException e) { throw new ComputerException(e.getMessage(), e); } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java index 3278f5069..c89ec2471 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java @@ -33,12 +33,12 @@ import com.baidu.hugegraph.computer.core.common.ComputerContext; import com.baidu.hugegraph.computer.core.common.Constants; import com.baidu.hugegraph.computer.core.common.ContainerInfo; +import com.baidu.hugegraph.computer.core.compute.ComputeManager; import com.baidu.hugegraph.computer.core.config.ComputerOptions; import com.baidu.hugegraph.computer.core.config.Config; import com.baidu.hugegraph.computer.core.graph.SuperstepStat; import com.baidu.hugegraph.computer.core.graph.edge.Edge; import com.baidu.hugegraph.computer.core.graph.id.Id; -import com.baidu.hugegraph.computer.core.graph.partition.PartitionStat; import com.baidu.hugegraph.computer.core.graph.value.Value; import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; import com.baidu.hugegraph.computer.core.input.WorkerInputManager; @@ -66,6 +66,7 @@ public class WorkerService { private boolean inited; private Config config; private Bsp4Worker bsp4Worker; + private ComputeManager computeManager; private ContainerInfo workerInfo; private Computation computation; @@ -127,6 +128,8 @@ public void init(Config config) { dm.connect(worker.id(), worker.hostname(), worker.dataPort()); } + this.computeManager = new ComputeManager(this.context, this.managers, + this.computation); this.managers.initedAll(this.config); LOG.info("{} WorkerService initialized", this); this.inited = true; @@ -184,7 +187,9 @@ public void execute() { WorkerContext context = new SuperstepContext(superstep, superstepStat); LOG.info("Start computation of superstep {}", superstep); - + if (superstep > 0) { + this.computeManager.takeComputeMessages(); + } /* * Call beforeSuperstep() before all workers compute() called. * @@ -200,14 +205,8 @@ public void execute() { */ this.bsp4Worker.workerStepPrepareDone(superstep); this.bsp4Worker.waitMasterStepPrepareDone(superstep); - - WorkerStat workerStat = this.compute(); - - /* - * Wait for all workers to do compute() - */ - MessageRecvManager messageRecvManager = - this.managers.get(MessageRecvManager.NAME); + WorkerStat workerStat = this.computeManager.compute(context, + superstep); this.bsp4Worker.workerStepComputeDone(superstep); this.bsp4Worker.waitMasterStepComputeDone(superstep); @@ -314,10 +313,7 @@ private SuperstepStat inputstep() { this.bsp4Worker.workerInputDone(); this.bsp4Worker.waitMasterInputDone(); - MessageRecvManager recvManager = - this.managers.get(MessageRecvManager.NAME); - - WorkerStat workerStat = recvManager.mergeGraph(); + WorkerStat workerStat = this.computeManager.input(); this.bsp4Worker.workerStepDone(Constants.INPUT_SUPERSTEP, workerStat); @@ -333,28 +329,11 @@ private SuperstepStat inputstep() { * can exit successfully. */ private void outputstep() { - /* - * Write results back parallel - */ - // TODO: output the vertices in partitions parallel + this.computeManager.output(); this.bsp4Worker.workerOutputDone(); LOG.info("{} WorkerService outputstep finished", this); } - /** - * Compute vertices of all partitions parallel in this worker. - * Be called one time for a superstep. - * @return WorkerStat - */ - private WorkerStat compute() { - // TODO: compute partitions parallel and get workerStat - PartitionStat stat1 = new PartitionStat(0, 100L, 200L, - 50L, 60L, 70L); - WorkerStat workerStat = new WorkerStat(); - workerStat.add(stat1); - return workerStat; - } - private class SuperstepContext implements WorkerContext { private final int superstep; diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManagerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManagerTest.java new file mode 100644 index 000000000..af9d0bba2 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManagerTest.java @@ -0,0 +1,248 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Random; +import java.util.function.Consumer; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.config.Config; +import com.baidu.hugegraph.computer.core.config.Null; +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; +import com.baidu.hugegraph.computer.core.graph.id.Id; +import com.baidu.hugegraph.computer.core.graph.id.LongId; +import com.baidu.hugegraph.computer.core.graph.properties.Properties; +import com.baidu.hugegraph.computer.core.graph.value.IdValueList; +import com.baidu.hugegraph.computer.core.graph.value.IdValueListList; +import com.baidu.hugegraph.computer.core.graph.value.LongValue; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.computer.core.io.BytesOutput; +import com.baidu.hugegraph.computer.core.io.IOFactory; +import com.baidu.hugegraph.computer.core.manager.Managers; +import com.baidu.hugegraph.computer.core.network.ConnectionId; +import com.baidu.hugegraph.computer.core.network.buffer.ManagedBuffer; +import com.baidu.hugegraph.computer.core.network.message.MessageType; +import com.baidu.hugegraph.computer.core.receiver.MessageRecvManager; +import com.baidu.hugegraph.computer.core.receiver.ReceiverUtil; +import com.baidu.hugegraph.computer.core.sender.MessageSendManager; +import com.baidu.hugegraph.computer.core.sort.sorting.SortManager; +import com.baidu.hugegraph.computer.core.store.FileManager; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutput; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutputImpl; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntryWriter; +import com.baidu.hugegraph.computer.core.worker.Computation; +import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; + +public class ComputeManagerTest extends UnitTestBase { + + private static final Random RANDOM = new Random(1); + + private Config config; + private Managers managers; + private ConnectionId connectionId; + private ComputeManager computeManager; + + @Before + public void setup() { + this.config = UnitTestBase.updateWithRequiredOptions( + ComputerOptions.JOB_ID, "local_001", + ComputerOptions.JOB_WORKERS_COUNT, "1", + ComputerOptions.JOB_PARTITIONS_COUNT, "2", + ComputerOptions.BSP_MAX_SUPER_STEP, "2", + ComputerOptions.WORKER_COMBINER_CLASS, + Null.class.getName(), // Can't combine + ComputerOptions.ALGORITHM_RESULT_CLASS, + IdValueListList.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, + IdValueList.class.getName(), + ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", + ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", + ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", + ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", + ComputerOptions.WORKER_COMPUTATION_CLASS, + MockComputation.class.getName() + ); + + this.managers = new Managers(); + FileManager fileManager = new FileManager(); + this.managers.add(fileManager); + SortManager sortManager = new SortManager(context()); + this.managers.add(sortManager); + + MessageSendManager sendManager = new MessageSendManager( + context(), sortManager, + new MockMessageSender()); + this.managers.add(sendManager); + MessageRecvManager receiveManager = new MessageRecvManager(context(), + fileManager, + sortManager); + this.managers.add(receiveManager); + this.managers.initAll(this.config); + this.connectionId = new ConnectionId(new InetSocketAddress("localhost", + 8081), + 0); + Computation computation = this.config.createObject( + ComputerOptions.WORKER_COMPUTATION_CLASS); + this.computeManager = new ComputeManager(context(), this.managers, + computation); + } + + @After + public void teardown() { + this.managers.closeAll(this.config); + } + + @Test + public void testProcess() throws IOException { + MessageRecvManager receiveManager = this.managers.get( + MessageRecvManager.NAME); + receiveManager.onStarted(this.connectionId); + add200VertexBuffer((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.VERTEX, 0, buffer); + }); + // Partition 1 only has vertex. + add200VertexBuffer((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.VERTEX, 1, buffer); + }); + receiveManager.onFinished(this.connectionId); + receiveManager.onStarted(this.connectionId); + addSingleFreqEdgeBuffer((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.EDGE, 0, buffer); + }); + receiveManager.onFinished(this.connectionId); + this.computeManager.input(); + + // Superstep 0 + receiveManager.beforeSuperstep(this.config, 0); + receiveManager.onStarted(this.connectionId); + addMessages((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.MSG, 0, buffer); + }); + receiveManager.onFinished(this.connectionId); + this.computeManager.compute(null, 0); + receiveManager.afterSuperstep(this.config, 0); + + // Superstep 1 + this.computeManager.takeComputeMessages(); + receiveManager.beforeSuperstep(this.config, 1); + receiveManager.onStarted(this.connectionId); + receiveManager.onFinished(this.connectionId); + this.computeManager.compute(null, 1); + receiveManager.afterSuperstep(this.config, 1); + + // Output + this.computeManager.output(); + } + + private static void add200VertexBuffer(Consumer consumer) + throws IOException { + for (long i = 0L; i < 200L; i += 2) { + Vertex vertex = graphFactory().createVertex(); + vertex.id(new LongId(i)); + vertex.properties(graphFactory().createProperties()); + ReceiverUtil.comsumeBuffer(writeVertex(vertex), consumer); + } + } + + private static byte[] writeVertex(Vertex vertex) throws IOException { + BytesOutput bytesOutput = IOFactory.createBytesOutput( + Constants.SMALL_BUF_SIZE); + EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); + + entryOutput.writeEntry(out -> { + out.writeByte(vertex.id().type().code()); + vertex.id().write(out); + }, out -> { + vertex.properties().write(out); + }); + + return bytesOutput.toByteArray(); + } + + private static void addSingleFreqEdgeBuffer( + Consumer consumer) throws IOException { + for (long i = 0L; i < 200L; i++) { + Vertex vertex = graphFactory().createVertex(); + vertex.id(new LongId(i)); + int count = RANDOM.nextInt(20); + if (count == 0) { + continue; + } + Edges edges = graphFactory().createEdges(count); + + for (long j = 0; j < count; j++) { + Edge edge = graphFactory().createEdge(); + edge.targetId(new LongId(RANDOM.nextInt(200))); + Properties properties = graphFactory().createProperties(); + properties.put("p1", new LongValue(i)); + edge.properties(properties); + edges.add(edge); + } + vertex.edges(edges); + ReceiverUtil.comsumeBuffer(writeEdges(vertex), consumer); + } + } + + private static byte[] writeEdges(Vertex vertex) throws IOException { + BytesOutput bytesOutput = IOFactory.createBytesOutput( + Constants.SMALL_BUF_SIZE); + EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); + + Id id = vertex.id(); + KvEntryWriter subKvWriter = entryOutput.writeEntry(out -> { + out.writeByte(id.type().code()); + id.write(out); + }); + for (Edge edge : vertex.edges()) { + Id targetId = edge.targetId(); + subKvWriter.writeSubKv(out -> { + out.writeByte(targetId.type().code()); + targetId.write(out); + }, out -> { + edge.properties().write(out); + }); + } + subKvWriter.writeFinish(); + return bytesOutput.toByteArray(); + } + + private static void addMessages(Consumer consumer) + throws IOException { + for (long i = 0L; i < 200L; i++) { + int count = RANDOM.nextInt(5); + for (int j = 0; j < count; j++) { + Id id = new LongId(i); + IdValueList message = new IdValueList(); + message.add(id.idValue()); + ReceiverUtil.comsumeBuffer(ReceiverUtil.writeMessage(id, + message), + consumer); + } + } + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java new file mode 100644 index 000000000..09164a24e --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +import com.baidu.hugegraph.computer.core.compute.input.EdgesInputTest; +import com.baidu.hugegraph.computer.core.compute.input.EmptyEdgesTest; +import com.baidu.hugegraph.computer.core.compute.input.MessageInputTest; + +@RunWith(Suite.class) +@Suite.SuiteClasses({ + EdgesInputTest.class, + EmptyEdgesTest.class, + MessageInputTest.class, + ComputeManagerTest.class +}) +public class ComputeTestSuite { +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockComputation.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockComputation.java new file mode 100644 index 000000000..9bb81d4d5 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockComputation.java @@ -0,0 +1,85 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import java.util.Iterator; +import java.util.Random; + +import org.junit.Assert; + +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; +import com.baidu.hugegraph.computer.core.graph.value.IdValueList; +import com.baidu.hugegraph.computer.core.graph.value.IdValueListList; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.computer.core.worker.Computation; +import com.baidu.hugegraph.computer.core.worker.ComputationContext; + +public class MockComputation implements Computation { + + private static final String NAME = "MockComputation"; + private static final String CATEGORY = "Mock"; + private static final Random RANDOM = new Random(1001L); + + @Override + public String name() { + return NAME; + } + + @Override + public String category() { + return CATEGORY; + } + + @Override + public void compute0(ComputationContext context, Vertex vertex) { + IdValueListList value = new IdValueListList(); + vertex.value(value); + Edges edges = vertex.edges(); + checkEdges(edges); + checkEdges(edges); + if (RANDOM.nextInt() % 10 == 0) { + vertex.inactivate(); + } + } + + @Override + public void compute(ComputationContext context, Vertex vertex, + Iterator messages) { + IdValueListList value = vertex.value(); + while (messages.hasNext()) { + Assert.assertTrue(messages.hasNext()); + value.add(messages.next().copy()); + } + Assert.assertFalse(messages.hasNext()); + if (RANDOM.nextInt() % 10 == 0) { + vertex.inactivate(); + } + } + + private static void checkEdges(Edges edges) { + int edgeSize = edges.size(); + int edgeIndex = 0; + for (Edge edge : edges) { + edgeIndex++; + } + Assert.assertEquals(edgeSize, edgeIndex); + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockMessageSender.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockMessageSender.java new file mode 100644 index 000000000..b0845fd9c --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/MockMessageSender.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute; + +import java.util.concurrent.CompletableFuture; + +import com.baidu.hugegraph.computer.core.network.message.MessageType; +import com.baidu.hugegraph.computer.core.sender.MessageSender; +import com.baidu.hugegraph.computer.core.sender.QueuedMessage; + +public class MockMessageSender implements MessageSender { + + @Override + public CompletableFuture send(int workerId, MessageType type) { + CompletableFuture future = new CompletableFuture<>(); + future.complete(null); + return future; + } + + @Override + public void send(int workerId, QueuedMessage message) { + // pass + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java new file mode 100644 index 000000000..9a9a92ae9 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java @@ -0,0 +1,300 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Iterator; +import java.util.function.Consumer; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.compute.FileGraphPartition; +import com.baidu.hugegraph.computer.core.compute.MockMessageSender; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.config.Config; +import com.baidu.hugegraph.computer.core.config.EdgeFrequency; +import com.baidu.hugegraph.computer.core.config.Null; +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.core.graph.edge.Edges; +import com.baidu.hugegraph.computer.core.graph.id.Id; +import com.baidu.hugegraph.computer.core.graph.id.LongId; +import com.baidu.hugegraph.computer.core.graph.properties.Properties; +import com.baidu.hugegraph.computer.core.graph.value.IdValueList; +import com.baidu.hugegraph.computer.core.graph.value.IdValueListList; +import com.baidu.hugegraph.computer.core.graph.value.LongValue; +import com.baidu.hugegraph.computer.core.graph.vertex.Vertex; +import com.baidu.hugegraph.computer.core.io.BytesOutput; +import com.baidu.hugegraph.computer.core.io.IOFactory; +import com.baidu.hugegraph.computer.core.manager.Managers; +import com.baidu.hugegraph.computer.core.network.ConnectionId; +import com.baidu.hugegraph.computer.core.network.buffer.ManagedBuffer; +import com.baidu.hugegraph.computer.core.network.message.MessageType; +import com.baidu.hugegraph.computer.core.receiver.MessageRecvManager; +import com.baidu.hugegraph.computer.core.receiver.ReceiverUtil; +import com.baidu.hugegraph.computer.core.sender.MessageSendManager; +import com.baidu.hugegraph.computer.core.sort.sorting.SortManager; +import com.baidu.hugegraph.computer.core.store.FileManager; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutput; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutputImpl; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntryWriter; +import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; +import com.baidu.hugegraph.testutil.Whitebox; + +public class EdgesInputTest extends UnitTestBase { + + private Config config; + private Managers managers; + + @After + public void teardown() { + this.managers.closeAll(this.config); + } + + @Test + public void testSingle() throws IOException { + this.testEdgeFreq(EdgeFrequency.SINGLE); + } + + @Test + public void testSinglePerLabel() throws IOException { + this.testEdgeFreq(EdgeFrequency.SINGLE_PER_LABEL); + } + + @Test + public void testMultiple() throws IOException { + this.testEdgeFreq(EdgeFrequency.MULTIPLE); + } + + private void testEdgeFreq(EdgeFrequency freq) + throws IOException { + this.config = UnitTestBase.updateWithRequiredOptions( + ComputerOptions.JOB_ID, "local_001", + ComputerOptions.JOB_WORKERS_COUNT, "1", + ComputerOptions.JOB_PARTITIONS_COUNT, "1", + ComputerOptions.WORKER_COMBINER_CLASS, + Null.class.getName(), // Can't combine + ComputerOptions.ALGORITHM_RESULT_CLASS, + IdValueListList.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, + IdValueList.class.getName(), + ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", + ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", + ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", + ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", + ComputerOptions.INPUT_EDGE_FREQ, freq.name() + ); + this.managers = new Managers(); + FileManager fileManager = new FileManager(); + this.managers.add(fileManager); + SortManager sortManager = new SortManager(context()); + this.managers.add(sortManager); + + MessageSendManager sendManager = new MessageSendManager( + context(), sortManager, + new MockMessageSender()); + this.managers.add(sendManager); + MessageRecvManager receiveManager = new MessageRecvManager(context(), + fileManager, + sortManager); + this.managers.add(receiveManager); + this.managers.initAll(this.config); + ConnectionId connectionId = new ConnectionId(new InetSocketAddress( + "localhost", 8081), + 0); + FileGraphPartition partition = new FileGraphPartition(context(), + this.managers, + 0); + receiveManager.onStarted(connectionId); + add200VertexBuffer((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.VERTEX, 0, buffer); + }); + receiveManager.onFinished(connectionId); + receiveManager.onStarted(connectionId); + addEdgeBuffer((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.EDGE, 0, buffer); + }, freq); + //recvManagerConsumer.accept(addEdgeBuffer); + + receiveManager.onFinished(connectionId); + partition.init(receiveManager.vertexPartitions().get(0), + receiveManager.edgePartitions().get(0)); + File edgeFile = Whitebox.getInternalState(partition, "edgeFile"); + EdgesInput edgesInput = new EdgesInput(context(), edgeFile); + edgesInput.init(); + this.checkEdgesInput(edgesInput, freq); + edgesInput.close(); + } + + private static void add200VertexBuffer(Consumer consumer) + throws IOException { + for (long i = 0L; i < 200L; i += 1) { + Vertex vertex = graphFactory().createVertex(); + vertex.id(new LongId(i)); + vertex.properties(graphFactory().createProperties()); + ReceiverUtil.comsumeBuffer(writeVertex(vertex), consumer); + } + } + + private static byte[] writeVertex(Vertex vertex) throws IOException { + BytesOutput bytesOutput = IOFactory.createBytesOutput( + Constants.SMALL_BUF_SIZE); + EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); + + entryOutput.writeEntry(out -> { + out.writeByte(vertex.id().type().code()); + vertex.id().write(out); + }, out -> { + vertex.properties().write(out); + }); + + return bytesOutput.toByteArray(); + } + + private static void addEdgeBuffer(Consumer consumer, + EdgeFrequency freq) throws IOException { + for (long i = 0L; i < 200L; i++) { + Vertex vertex = graphFactory().createVertex(); + vertex.id(new LongId(i)); + int count = (int) i; + if (count == 0) { + continue; + } + Edges edges = graphFactory().createEdges(count); + + for (long j = 0; j < count; j++) { + Edge edge = graphFactory().createEdge(); + switch (freq) { + case SINGLE: + edge.targetId(new LongId(j)); + break; + case SINGLE_PER_LABEL: + edge.label(String.valueOf(j)); + edge.targetId(new LongId(j)); + break; + case MULTIPLE: + edge.name(String.valueOf(j)); + edge.label(String.valueOf(j)); + edge.targetId(new LongId(j)); + break; + default: + throw new ComputerException( + "Illegal edge frequency %s", freq); + } + + Properties properties = graphFactory().createProperties(); + properties.put("p1", new LongValue(i)); + edge.properties(properties); + edges.add(edge); + } + vertex.edges(edges); + ReceiverUtil.comsumeBuffer(writeEdges(vertex, freq), consumer); + } + } + + private static byte[] writeEdges(Vertex vertex, EdgeFrequency freq) + throws IOException { + BytesOutput bytesOutput = IOFactory.createBytesOutput( + Constants.SMALL_BUF_SIZE); + EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); + + Id id = vertex.id(); + KvEntryWriter subKvWriter = entryOutput.writeEntry(out -> { + out.writeByte(id.type().code()); + id.write(out); + }); + for (Edge edge : vertex.edges()) { + Id targetId = edge.targetId(); + subKvWriter.writeSubKv(out -> { + switch (freq) { + case SINGLE: + out.writeByte(targetId.type().code()); + targetId.write(out); + break; + case SINGLE_PER_LABEL: + out.writeUTF(edge.label()); + out.writeByte(targetId.type().code()); + targetId.write(out); + break; + case MULTIPLE: + out.writeUTF(edge.label()); + out.writeUTF(edge.name()); + out.writeByte(targetId.type().code()); + targetId.write(out); + break; + default: + throw new ComputerException( + "Illegal edge frequency %s", freq); + } + }, out -> { + edge.properties().write(out); + }); + } + subKvWriter.writeFinish(); + return bytesOutput.toByteArray(); + } + + private void checkEdgesInput(EdgesInput edgesInput, EdgeFrequency freq) + throws IOException { + + for (long i = 0L; i < 200L; i++) { + LongId id = new LongId(i); + ReusablePointer idPointer = idToResablePpointer(id); + Edges edges = edgesInput.edges(idPointer); + Iterator edgesIt = edges.iterator(); + Assert.assertEquals(i, edges.size()); + for (int j = 0; j < edges.size(); j++) { + Assert.assertTrue(edgesIt.hasNext()); + Edge edge = edgesIt.next(); + switch (freq) { + case SINGLE: + Assert.assertEquals(new LongId(j), edge.targetId()); + break; + case SINGLE_PER_LABEL: + Assert.assertEquals(new LongId(j), edge.targetId()); + Assert.assertEquals(String.valueOf(j), edge.label()); + break; + case MULTIPLE: + Assert.assertEquals(new LongId(j), edge.targetId()); + Assert.assertEquals(String.valueOf(j), edge.label()); + Assert.assertEquals(String.valueOf(j), edge.name()); + break; + default: + throw new ComputerException( + "Illegal edge frequency %s", freq); + } + } + Assert.assertFalse(edgesIt.hasNext()); + } + } + + public static ReusablePointer idToResablePpointer(Id id) + throws IOException { + BytesOutput output = IOFactory.createBytesOutput(9); + output.writeByte(id.type().code()); + id.write(output); + return new ReusablePointer(output.buffer(), (int) output.position()); + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java new file mode 100644 index 000000000..64cf9ce79 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.util.Iterator; + +import org.junit.Test; + +import com.baidu.hugegraph.computer.core.common.exception.ComputerException; +import com.baidu.hugegraph.computer.core.graph.edge.Edge; +import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; +import com.baidu.hugegraph.testutil.Assert; + +public class EmptyEdgesTest extends UnitTestBase { + + @Test + public void testEmptyEdges() { + EmptyEdges edges = EmptyEdges.instance(); + Iterator it = edges.iterator(); + Assert.assertFalse(it.hasNext()); + Assert.assertEquals(0, edges.size()); + Assert.assertThrows(ComputerException.class, () -> { + edges.add(graphFactory().createEdge()); + }, e -> { + Assert.assertContains("Can't add edge", e.getMessage()); + }); + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java new file mode 100644 index 000000000..19b233751 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java @@ -0,0 +1,177 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Consumer; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.baidu.hugegraph.computer.core.compute.MockComputation; +import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.config.Config; +import com.baidu.hugegraph.computer.core.config.Null; +import com.baidu.hugegraph.computer.core.graph.id.Id; +import com.baidu.hugegraph.computer.core.graph.id.LongId; +import com.baidu.hugegraph.computer.core.graph.value.IdValueList; +import com.baidu.hugegraph.computer.core.graph.value.IdValueListList; +import com.baidu.hugegraph.computer.core.manager.Managers; +import com.baidu.hugegraph.computer.core.network.ConnectionId; +import com.baidu.hugegraph.computer.core.network.buffer.ManagedBuffer; +import com.baidu.hugegraph.computer.core.network.message.MessageType; +import com.baidu.hugegraph.computer.core.receiver.MessageRecvManager; +import com.baidu.hugegraph.computer.core.receiver.ReceiverUtil; +import com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator; +import com.baidu.hugegraph.computer.core.sort.sorting.SortManager; +import com.baidu.hugegraph.computer.core.store.FileManager; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; +import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; +import com.baidu.hugegraph.testutil.Assert; + +public class MessageInputTest extends UnitTestBase { + + private Config config; + private Managers managers; + private ConnectionId connectionId; + + @Before + public void setup() { + this.config = UnitTestBase.updateWithRequiredOptions( + ComputerOptions.JOB_ID, "local_001", + ComputerOptions.JOB_WORKERS_COUNT, "1", + ComputerOptions.JOB_PARTITIONS_COUNT, "2", + ComputerOptions.BSP_MAX_SUPER_STEP, "2", + ComputerOptions.WORKER_COMBINER_CLASS, + Null.class.getName(), // Can't combine + ComputerOptions.ALGORITHM_RESULT_CLASS, + IdValueListList.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, + IdValueList.class.getName(), + ComputerOptions.WORKER_DATA_DIRS, "[data_dir1, data_dir2]", + ComputerOptions.WORKER_RECEIVED_BUFFERS_BYTES_LIMIT, "10000", + ComputerOptions.WORKER_WAIT_FINISH_MESSAGES_TIMEOUT, "1000", + ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "10", + ComputerOptions.WORKER_COMPUTATION_CLASS, + MockComputation.class.getName() + ); + + this.managers = new Managers(); + FileManager fileManager = new FileManager(); + this.managers.add(fileManager); + SortManager sortManager = new SortManager(context()); + this.managers.add(sortManager); + + MessageRecvManager receiveManager = new MessageRecvManager(context(), + fileManager, + sortManager); + this.managers.add(receiveManager); + this.managers.initAll(this.config); + this.connectionId = new ConnectionId(new InetSocketAddress("localhost", + 8081), + 0); + } + + @After + public void teardown() { + this.managers.closeAll(this.config); + } + + @Test + public void testMessageInput() throws IOException { + MessageRecvManager receiveManager = this.managers.get( + MessageRecvManager.NAME); + receiveManager.onStarted(this.connectionId); + + // Superstep 0 + receiveManager.beforeSuperstep(this.config, 0); + receiveManager.onStarted(this.connectionId); + addMessages((ManagedBuffer buffer) -> { + receiveManager.handle(MessageType.MSG, 0, buffer); + }); + receiveManager.onFinished(this.connectionId); + PeekableIterator it = receiveManager.messagePartitions() + .get(0); + MessageInput input = new MessageInput<>(context(), it); + Map> expectedMessages = expectedMessages(); + checkMessages(expectedMessages, input); + } + + private void checkMessages(Map> expectedMessages, + MessageInput input) + throws IOException { + + for (long i = 0L; i < 200L; i++) { + List messages = expectedMessages.get(new LongId(i)); + LongId id = new LongId(i); + ReusablePointer idPointer = EdgesInputTest.idToResablePpointer(id); + Iterator mit = input.iterator(idPointer); + if (messages == null) { + Assert.assertFalse(mit.hasNext()); + } else { + for (int j = 0; j < messages.size();j++) { + Assert.assertTrue(mit.hasNext()); + Assert.assertTrue(messages.contains(mit.next())); + } + } + } + } + + private static void addMessages(Consumer consumer) + throws IOException { + Random random = new Random(1); + for (long i = 0L; i < 200L; i++) { + int count = random.nextInt(5); + for (int j = 0; j < count; j++) { + Id id = new LongId(random.nextInt(200)); + IdValueList message = new IdValueList(); + message.add(id.idValue()); + ReceiverUtil.comsumeBuffer(ReceiverUtil.writeMessage(id, + message), + consumer); + } + } + } + + private static Map> expectedMessages() { + Random random = new Random(1); + Map> globalMessages = new HashMap<>(); + for (long i = 0L; i < 200L; i++) { + int count = random.nextInt(5); + for (int j = 0; j < count; j++) { + Id id = new LongId(random.nextInt(200)); + IdValueList message = new IdValueList(); + message.add(id.idValue()); + List messages = globalMessages.computeIfAbsent( + id, nid -> new ArrayList<>()); + messages.add(message); + } + } + return globalMessages; + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java index 7da34256f..4594d5d18 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java @@ -23,6 +23,7 @@ import org.junit.Test; +import com.baidu.hugegraph.computer.core.receiver.RecvStat; import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; import com.baidu.hugegraph.testutil.Assert; @@ -55,6 +56,27 @@ public void testReadWrite() throws IOException { UnitTestBase.assertEqualAfterWriteAndRead(stat2, stat2ReadObj); } + @Test + public void testMerge() throws IOException { + PartitionStat stat1 = new PartitionStat(0, 1L, 2L); + stat1.merge(null); + Assert.assertEquals(1L, stat1.vertexCount()); + Assert.assertEquals(2L, stat1.edgeCount()); + Assert.assertEquals(0L, stat1.messageBytes()); + Assert.assertEquals(0L, stat1.messageCount()); + RecvStat recvStat = new RecvStat(100L, 400L); + stat1.merge(recvStat); + Assert.assertEquals(1L, stat1.vertexCount()); + Assert.assertEquals(2L, stat1.edgeCount()); + Assert.assertEquals(100L, stat1.messageBytes()); + Assert.assertEquals(400L, stat1.messageCount()); + stat1.merge(recvStat); + Assert.assertEquals(1L, stat1.vertexCount()); + Assert.assertEquals(2L, stat1.edgeCount()); + Assert.assertEquals(200L, stat1.messageBytes()); + Assert.assertEquals(800L, stat1.messageCount()); + } + @Test public void testEquals() { PartitionStat stat1 = new PartitionStat(0, 1L, 2L); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/ReceiverUtil.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/ReceiverUtil.java index d5264db13..3e9665f29 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/ReceiverUtil.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/ReceiverUtil.java @@ -22,12 +22,18 @@ import java.io.IOException; import java.util.function.Consumer; +import com.baidu.hugegraph.computer.core.common.Constants; import com.baidu.hugegraph.computer.core.graph.id.Id; +import com.baidu.hugegraph.computer.core.io.BytesOutput; +import com.baidu.hugegraph.computer.core.io.IOFactory; import com.baidu.hugegraph.computer.core.io.RandomAccessInput; import com.baidu.hugegraph.computer.core.io.Readable; import com.baidu.hugegraph.computer.core.io.StreamGraphInput; +import com.baidu.hugegraph.computer.core.io.Writable; import com.baidu.hugegraph.computer.core.network.buffer.ManagedBuffer; import com.baidu.hugegraph.computer.core.network.buffer.NettyManagedBuffer; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutput; +import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutputImpl; import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.Pointer; import io.netty.buffer.ByteBuf; @@ -61,4 +67,17 @@ public static void readValue(Pointer pointer, Readable value) value.read(input); input.seek(position); } + + public static byte[] writeMessage(Id id, Writable message) + throws IOException { + BytesOutput bytesOutput = IOFactory.createBytesOutput( + Constants.SMALL_BUF_SIZE); + EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); + + entryOutput.writeEntry(out -> { + out.writeByte(id.type().code()); + id.write(out); + }, message); + return bytesOutput.toByteArray(); + } } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/message/ComputeMessageRecvPartitionTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/message/ComputeMessageRecvPartitionTest.java index 8ccf928ba..7e046fac1 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/message/ComputeMessageRecvPartitionTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/receiver/message/ComputeMessageRecvPartitionTest.java @@ -27,7 +27,6 @@ import org.junit.Test; import com.baidu.hugegraph.computer.core.combiner.DoubleValueSumCombiner; -import com.baidu.hugegraph.computer.core.common.Constants; import com.baidu.hugegraph.computer.core.config.ComputerOptions; import com.baidu.hugegraph.computer.core.config.Config; import com.baidu.hugegraph.computer.core.config.Null; @@ -35,9 +34,6 @@ import com.baidu.hugegraph.computer.core.graph.id.LongId; import com.baidu.hugegraph.computer.core.graph.value.DoubleValue; import com.baidu.hugegraph.computer.core.graph.value.IdValueList; -import com.baidu.hugegraph.computer.core.io.BytesOutput; -import com.baidu.hugegraph.computer.core.io.IOFactory; -import com.baidu.hugegraph.computer.core.io.Writable; import com.baidu.hugegraph.computer.core.network.buffer.ManagedBuffer; import com.baidu.hugegraph.computer.core.network.message.MessageType; import com.baidu.hugegraph.computer.core.receiver.ReceiverUtil; @@ -45,8 +41,6 @@ import com.baidu.hugegraph.computer.core.sort.sorting.SortManager; import com.baidu.hugegraph.computer.core.store.FileManager; import com.baidu.hugegraph.computer.core.store.SuperstepFileGenerator; -import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutput; -import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutputImpl; import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; import com.baidu.hugegraph.testutil.Assert; @@ -128,7 +122,9 @@ public static void addTwentyCombineMessageBuffer( for (int j = 0; j < 2; j++) { Id id = new LongId(i); DoubleValue message = new DoubleValue(i); - ReceiverUtil.comsumeBuffer(writeMessage(id, message), consumer); + ReceiverUtil.comsumeBuffer(ReceiverUtil.writeMessage(id, + message), + consumer); } } } @@ -164,24 +160,13 @@ public static void checkTenCombineMessages(PeekableIterator it) Id id = new LongId(i); IdValueList message = new IdValueList(); message.add(id.idValue()); - ReceiverUtil.comsumeBuffer(writeMessage(id, message), consumer); + ReceiverUtil.comsumeBuffer(ReceiverUtil.writeMessage(id, + message), + consumer); } } } - private static byte[] writeMessage(Id id, Writable message) - throws IOException { - BytesOutput bytesOutput = IOFactory.createBytesOutput( - Constants.SMALL_BUF_SIZE); - EntryOutput entryOutput = new EntryOutputImpl(bytesOutput); - - entryOutput.writeEntry(out -> { - out.writeByte(id.type().code()); - id.write(out); - }, message); - return bytesOutput.toByteArray(); - } - private static void checkIdValueListMessages(PeekableIterator it) throws IOException { for (long i = 0L; i < 10L; i++) { diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java index 5caa8763d..f16deac8c 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java @@ -28,6 +28,7 @@ import com.baidu.hugegraph.computer.core.bsp.BspTestSuite; import com.baidu.hugegraph.computer.core.combiner.CombinerTestSuite; import com.baidu.hugegraph.computer.core.common.CommonTestSuite; +import com.baidu.hugegraph.computer.core.compute.ComputeTestSuite; import com.baidu.hugegraph.computer.core.config.ComputerOptions; import com.baidu.hugegraph.computer.core.config.ConfigTestSuite; import com.baidu.hugegraph.computer.core.graph.GraphTestSuite; @@ -63,7 +64,8 @@ SorterTestSuite.class, SortingTestSuite.class, SenderTestSuite.class, - ReceiverTestSuite.class + ReceiverTestSuite.class, + ComputeTestSuite.class }) public class UnitTestSuite { From f75e70de3700f3464f22286ccbe8923a988e9bc3 Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Thu, 1 Jul 2021 18:28:19 +0800 Subject: [PATCH 2/9] tiny improve --- .../computer/core/compute/ComputeManager.java | 28 ++++++++----------- .../core/compute/FileGraphPartition.java | 24 ++++++---------- .../core/compute/input/ReusablePointer.java | 2 +- .../computer/core/receiver/RecvStat.java | 1 - .../core/compute/input/EdgesInputTest.java | 11 ++++---- .../core/compute/input/MessageInputTest.java | 6 ++-- .../graph/partition/PartitionStatTest.java | 11 ++++---- .../core/worker/WorkerServiceTest.java | 10 +++---- 8 files changed, 39 insertions(+), 54 deletions(-) diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java index c1e77b793..961527a13 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java @@ -43,12 +43,12 @@ public class ComputeManager> { private static final Logger LOG = Log.logger(ComputeManager.class); - private ComputerContext context; + private final ComputerContext context; private final Managers managers; private final Map> partitions; private final Computation computation; - private final MessageRecvManager messageRecvManager; + private final MessageRecvManager recvManager; private final MessageSendManager sendManager; public ComputeManager(ComputerContext context, Managers managers, @@ -57,19 +57,17 @@ public ComputeManager(ComputerContext context, Managers managers, this.managers = managers; this.computation = computation; this.partitions = new HashMap<>(); - this.messageRecvManager = this.managers.get(MessageRecvManager.NAME); + this.recvManager = this.managers.get(MessageRecvManager.NAME); this.sendManager = this.managers.get(MessageSendManager.NAME); } public WorkerStat input() { WorkerStat workerStat = new WorkerStat(); - MessageRecvManager messageRecvManager = this.managers.get( - MessageRecvManager.NAME); - messageRecvManager.waitReceivedAllMessages(); + this.recvManager.waitReceivedAllMessages(); Map> vertices; - vertices = messageRecvManager.vertexPartitions(); + vertices = this.recvManager.vertexPartitions(); Map> edges; - edges = messageRecvManager.edgePartitions(); + edges = this.recvManager.edgePartitions(); // TODO: parallel input process for (Map.Entry> entry : vertices.entrySet()) { @@ -89,14 +87,10 @@ public WorkerStat input() { * {@link MessageRecvManager#beforeSuperstep} is called. */ public void takeComputeMessages() { - Map> messages; - messages = this.messageRecvManager.messagePartitions(); + Map> messages = + this.recvManager.messagePartitions(); for (FileGraphPartition partition : this.partitions.values()) { - PeekableIterator m = messages.get(partition.partition()); - if (m == null) { - m = PeekableIterator.emptyIterator(); - } - partition.messages(m); + partition.messages(messages.get(partition.partition())); } } @@ -123,9 +117,9 @@ public WorkerStat compute(ComputationContext context, int superstep) { } this.sendManager.finishSend(MessageType.MSG); // After compute and send finish signal. - Map recvStats = this.messageRecvManager.recvStats(); + Map recvStats = this.recvManager.recvStats(); for (Map.Entry entry : - partitionStats.entrySet()) { + partitionStats.entrySet()) { PartitionStat partitionStat = entry.getValue(); partitionStat.merge(recvStats.get(partitionStat.partitionId())); workerStat.add(partitionStat); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java index b89f3959e..77b468707 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java @@ -51,17 +51,17 @@ public class FileGraphPartition> { - private final ComputerContext context; private static final String VERTEX = "vertex"; private static final String EDGE = "edge"; private static final String STATUS = "status"; private static final String VALUE = "value"; + private final ComputerContext context; private final FileGenerator fileGenerator; private final int partition; + private final File vertexFile; + private final File edgeFile; - private File vertexFile; - private File edgeFile; private File preStatusFile; private File curStatusFile; private File preValueFile; @@ -117,8 +117,7 @@ public PartitionStat init(PeekableIterator vertices, vertexOut.close(); edgeOut.close(); } catch (IOException e) { - e.printStackTrace(); - throw new ComputerException("Init vertex input failed", e); + throw new ComputerException("File graph partition init failed", e); } return new PartitionStat(this.partition, this.vertexCount, @@ -128,7 +127,7 @@ public PartitionStat init(PeekableIterator vertices, // Package access PartitionStat compute0(ComputationContext context, Computation computation) { - long activeVertexCount = 0; + long activeVertexCount = 0L; this.beforeCompute(0); while (this.vertexInput.hasNext()) { @@ -168,7 +167,7 @@ PartitionStat compute(ComputationContext context, vertex.inactivate(); } } catch (IOException e) { - throw new ComputerException("Can't read status or result", e); + throw new ComputerException("Read status or result failed", e); } /* @@ -211,7 +210,7 @@ public PartitionStat output() { vertex.inactivate(); } } catch (IOException e) { - throw new ComputerException("Can't read status or result", e); + throw new ComputerException("Read status or result failed", e); } Edges edges = this.edgesInput.edges(this.vertexInput.idPointer()); @@ -245,8 +244,7 @@ private void saveVertex(Vertex vertex) { } private void writeEdges(Pointer id, PeekableIterator edges, - BufferedFileOutput edgeOut) - throws IOException { + BufferedFileOutput edgeOut) throws IOException { while (edges.hasNext()) { KvEntry entry = edges.peek(); Pointer key = entry.key(); @@ -281,8 +279,7 @@ private void writeEdges(Pointer id, PeekableIterator edges, } private void beforeCompute(int superstep) { - this.vertexInput = new VertexInput(this.context, - this.vertexFile, + this.vertexInput = new VertexInput(this.context, this.vertexFile, this.vertexCount); this.edgesInput = new EdgesInput(this.context, this.edgeFile); // Inputs @@ -310,7 +307,6 @@ private void beforeCompute(int superstep) { this.curStatusOut = new BufferedFileOutput(this.curStatusFile); this.curValueOut = new BufferedFileOutput(this.curValueFile); - } catch (IOException e) { throw new ComputerException("Before compute call failed", e); } @@ -359,8 +355,6 @@ private void afterOutput() { this.preValueIn.close(); this.preStatusFile.delete(); this.preValueFile.delete(); - this.vertexInput.close(); - this.edgesInput.close(); } catch (IOException e) { throw new ComputerException("After output call failed", e); } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java index fcd235073..bb6c9a106 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java @@ -50,7 +50,7 @@ public ReusablePointer(byte[] bytes, int length) { @Override public void read(RandomAccessInput in) throws IOException { - this.length = in.readInt(); + this.length = in.readFixedInt(); if (this.bytes.length < this.length) { this.bytes = new byte[this.length]; this.input = IOFactory.createBytesInput(this.bytes); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java index e65356909..71e9af79f 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java @@ -25,7 +25,6 @@ public class RecvStat { private final long messageCount; - private final long messageBytes; public RecvStat(long messageCount, long messageBytes) { diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java index 9a9a92ae9..066fb471f 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java @@ -136,7 +136,6 @@ private void testEdgeFreq(EdgeFrequency freq) addEdgeBuffer((ManagedBuffer buffer) -> { receiveManager.handle(MessageType.EDGE, 0, buffer); }, freq); - //recvManagerConsumer.accept(addEdgeBuffer); receiveManager.onFinished(connectionId); partition.init(receiveManager.vertexPartitions().get(0), @@ -150,7 +149,7 @@ private void testEdgeFreq(EdgeFrequency freq) private static void add200VertexBuffer(Consumer consumer) throws IOException { - for (long i = 0L; i < 200L; i += 1) { + for (long i = 0L; i < 200L; i += 2) { Vertex vertex = graphFactory().createVertex(); vertex.id(new LongId(i)); vertex.properties(graphFactory().createProperties()); @@ -259,9 +258,9 @@ private static byte[] writeEdges(Vertex vertex, EdgeFrequency freq) private void checkEdgesInput(EdgesInput edgesInput, EdgeFrequency freq) throws IOException { - for (long i = 0L; i < 200L; i++) { + for (long i = 0L; i < 200L; i+= 2) { LongId id = new LongId(i); - ReusablePointer idPointer = idToResablePpointer(id); + ReusablePointer idPointer = idToReusablePointer(id); Edges edges = edgesInput.edges(idPointer); Iterator edgesIt = edges.iterator(); Assert.assertEquals(i, edges.size()); @@ -290,8 +289,8 @@ private void checkEdgesInput(EdgesInput edgesInput, EdgeFrequency freq) } } - public static ReusablePointer idToResablePpointer(Id id) - throws IOException { + public static ReusablePointer idToReusablePointer(Id id) + throws IOException { BytesOutput output = IOFactory.createBytesOutput(9); output.writeByte(id.type().code()); id.write(output); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java index 19b233751..b7527877e 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInputTest.java @@ -123,13 +123,11 @@ public void testMessageInput() throws IOException { } private void checkMessages(Map> expectedMessages, - MessageInput input) - throws IOException { - + MessageInput input) throws IOException { for (long i = 0L; i < 200L; i++) { List messages = expectedMessages.get(new LongId(i)); LongId id = new LongId(i); - ReusablePointer idPointer = EdgesInputTest.idToResablePpointer(id); + ReusablePointer idPointer = EdgesInputTest.idToReusablePointer(id); Iterator mit = input.iterator(idPointer); if (messages == null) { Assert.assertFalse(mit.hasNext()); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java index 4594d5d18..fa8d9ce4f 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java @@ -62,19 +62,20 @@ public void testMerge() throws IOException { stat1.merge(null); Assert.assertEquals(1L, stat1.vertexCount()); Assert.assertEquals(2L, stat1.edgeCount()); - Assert.assertEquals(0L, stat1.messageBytes()); Assert.assertEquals(0L, stat1.messageCount()); + Assert.assertEquals(0L, stat1.messageBytes()); + RecvStat recvStat = new RecvStat(100L, 400L); stat1.merge(recvStat); Assert.assertEquals(1L, stat1.vertexCount()); Assert.assertEquals(2L, stat1.edgeCount()); - Assert.assertEquals(100L, stat1.messageBytes()); - Assert.assertEquals(400L, stat1.messageCount()); + Assert.assertEquals(100L, stat1.messageCount()); + Assert.assertEquals(400L, stat1.messageBytes()); stat1.merge(recvStat); Assert.assertEquals(1L, stat1.vertexCount()); Assert.assertEquals(2L, stat1.edgeCount()); - Assert.assertEquals(200L, stat1.messageBytes()); - Assert.assertEquals(800L, stat1.messageCount()); + Assert.assertEquals(200L, stat1.messageCount()); + Assert.assertEquals(800L, stat1.messageBytes()); } @Test diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java index ad402e2d0..d5362c534 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java @@ -67,8 +67,8 @@ public void testServiceWith1Worker() throws InterruptedException { LOG.error("Failed to start worker", e); exceptions[0] = e; } finally { - workerService.close(); countDownLatch.countDown(); + workerService.close(); } }); @@ -92,8 +92,8 @@ public void testServiceWith1Worker() throws InterruptedException { LOG.error("Failed to start master", e); exceptions[1] = e; } finally { - masterService.close(); countDownLatch.countDown(); + masterService.close(); } }); @@ -132,8 +132,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start worker", e); exceptions[0] = e; } finally { - workerService.close(); countDownLatch.countDown(); + workerService.close(); } }); @@ -159,8 +159,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start worker", e); exceptions[1] = e; } finally { - workerService.close(); countDownLatch.countDown(); + workerService.close(); } }); @@ -185,8 +185,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start master", e); exceptions[2] = e; } finally { - masterService.close(); countDownLatch.countDown(); + masterService.close(); } }); From 0bb8138552c18e7cb9c83b1ef1a6f9cd4f8e9361 Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Thu, 1 Jul 2021 20:23:35 +0800 Subject: [PATCH 3/9] SortManager & NettyManagedBuffer bug fix --- .../network/buffer/NettyManagedBuffer.java | 7 ++++--- .../core/sort/sorting/SortManager.java | 2 +- .../core/compute/input/EdgesInputTest.java | 2 +- .../core/worker/WorkerServiceTest.java | 21 ++++++++++++++----- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java index 8c662824a..0244d35af 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java @@ -69,9 +69,10 @@ public ByteBuf nettyByteBuf() { @Override public byte[] copyToByteArray() { - return ByteBufUtil.getBytes(this.buf, - this.buf.readerIndex(), - this.buf.readableBytes(), + ByteBuf buf = this.buf.duplicate(); + return ByteBufUtil.getBytes(buf, + buf.readerIndex(), + buf.readableBytes(), true); } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java index b20e3b228..1df083cb8 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java @@ -117,7 +117,7 @@ public CompletableFuture sort(MessageType type, throw new ComputerException("Failed to sort buffers of %s " + "message", e, type.name()); } - return ByteBuffer.wrap(output.buffer()); + return ByteBuffer.wrap(output.toByteArray()); }, this.sortExecutor); } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java index 066fb471f..0800730c6 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java @@ -258,7 +258,7 @@ private static byte[] writeEdges(Vertex vertex, EdgeFrequency freq) private void checkEdgesInput(EdgesInput edgesInput, EdgeFrequency freq) throws IOException { - for (long i = 0L; i < 200L; i+= 2) { + for (long i = 0L; i < 200L; i += 2) { LongId id = new LongId(i); ReusablePointer idPointer = idToReusablePointer(id); Edges edges = edgesInput.edges(idPointer); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java index d5362c534..e628e9b06 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java @@ -30,6 +30,7 @@ import com.baidu.hugegraph.computer.core.common.exception.ComputerException; import com.baidu.hugegraph.computer.core.config.ComputerOptions; import com.baidu.hugegraph.computer.core.config.Config; +import com.baidu.hugegraph.computer.core.graph.value.DoubleValue; import com.baidu.hugegraph.computer.core.master.MasterService; import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; import com.baidu.hugegraph.config.RpcOptions; @@ -56,7 +57,9 @@ public void testServiceWith1Worker() throws InterruptedException { ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, - MockComputation.class.getName() + MockComputation.class.getName(), + ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); try { @@ -82,7 +85,9 @@ public void testServiceWith1Worker() throws InterruptedException { ComputerOptions.BSP_LOG_INTERVAL, "30000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.MASTER_COMPUTATION_CLASS, - MockMasterComputation.class.getName() + MockMasterComputation.class.getName(), + ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName() ); MasterService masterService = new MasterService(); try { @@ -121,7 +126,9 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, - MockComputation2.class.getName() + MockComputation2.class.getName(), + ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); try { @@ -148,7 +155,9 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.WORKER_COMPUTATION_CLASS, - MockComputation2.class.getName() + MockComputation2.class.getName(), + ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); try { @@ -175,7 +184,9 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.BSP_LOG_INTERVAL, "10000", ComputerOptions.BSP_MAX_SUPER_STEP, "2", ComputerOptions.MASTER_COMPUTATION_CLASS, - MockMasterComputation2.class.getName() + MockMasterComputation2.class.getName(), + ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName() ); MasterService masterService = new MasterService(); try { From 408b416c568a328c67ba97994def90334e0fc7e9 Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Fri, 2 Jul 2021 10:18:45 +0800 Subject: [PATCH 4/9] tiny improve --- .../computer/core/worker/MockComputation.java | 9 +------ .../core/worker/MockComputation2.java | 6 ----- .../core/worker/MockMasterComputation.java | 10 +++---- .../core/worker/MockMasterComputation2.java | 9 ------- .../core/worker/WorkerServiceTest.java | 27 ++++++++++++++----- 5 files changed, 27 insertions(+), 34 deletions(-) diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation.java index 8f0e09010..348e24083 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation.java @@ -56,8 +56,6 @@ public void beforeSuperstep(WorkerContext context) { } else if (context.superstep() == 1) { this.assertStep1Aggregators(context); } - - this.assertStat(context); } @Override @@ -129,11 +127,6 @@ private void assertAggregateValueWithError(WorkerContext context) { }); } - protected void assertStat(WorkerContext context) { - Assert.assertEquals(100L, context.totalVertexCount()); - Assert.assertEquals(200L, context.totalEdgeCount()); - } - protected void createAndRunAggregators(WorkerContext context) { // AGGR_CUSTOM_INT this.aggrCustomInt = context.createAggregator( @@ -403,7 +396,7 @@ public String category() { @Override public void compute0(ComputationContext context, Vertex vertex) { - // pass + vertex.value(new DoubleValue(0.5D)); } @Override diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation2.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation2.java index 7fc34bbc0..1e665b8a8 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation2.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockComputation2.java @@ -27,12 +27,6 @@ public class MockComputation2 extends MockComputation { - @Override - protected void assertStat(WorkerContext context) { - Assert.assertEquals(200L, context.totalVertexCount()); - Assert.assertEquals(400L, context.totalEdgeCount()); - } - @Override protected void assertStep1Aggregators(WorkerContext context) { Assert.assertEquals(new IntValue(10), context.aggregatedValue( diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java index cd306261a..21a91d716 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java @@ -178,11 +178,11 @@ public boolean compute(MasterComputationContext context) { } protected void assertStat(MasterComputationContext context) { - Assert.assertEquals(100L, context.totalVertexCount()); - Assert.assertEquals(200L, context.totalEdgeCount()); - Assert.assertEquals(50L, context.finishedVertexCount()); - Assert.assertEquals(60L, context.messageCount()); - Assert.assertEquals(70L, context.messageBytes()); + Assert.assertEquals(6L, context.totalVertexCount()); + Assert.assertEquals(5L, context.totalEdgeCount()); + Assert.assertEquals(0L, context.finishedVertexCount()); + Assert.assertEquals(0L, context.messageCount()); + Assert.assertEquals(0L, context.messageBytes()); } protected void assertStep0Aggregators(MasterComputationContext context) { diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation2.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation2.java index 36c22cffe..140c641bc 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation2.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation2.java @@ -28,15 +28,6 @@ public class MockMasterComputation2 extends MockMasterComputation { - @Override - protected void assertStat(MasterComputationContext context) { - Assert.assertEquals(200L, context.totalVertexCount()); - Assert.assertEquals(400L, context.totalEdgeCount()); - Assert.assertEquals(100L, context.finishedVertexCount()); - Assert.assertEquals(120L, context.messageCount()); - Assert.assertEquals(140L, context.messageBytes()); - } - @Override protected void assertStep0Aggregators(MasterComputationContext context) { Assert.assertEquals(new IntValue(10), context.aggregatedValue( diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java index e628e9b06..bde7dc6ff 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java @@ -59,19 +59,21 @@ public void testServiceWith1Worker() throws InterruptedException { ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); try { - Thread.sleep(2000); + Thread.sleep(2000L); workerService.init(config); workerService.execute(); } catch (Throwable e) { LOG.error("Failed to start worker", e); exceptions[0] = e; } finally { - countDownLatch.countDown(); workerService.close(); + countDownLatch.countDown(); } }); @@ -87,6 +89,8 @@ public void testServiceWith1Worker() throws InterruptedException { ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName() ); MasterService masterService = new MasterService(); @@ -97,8 +101,13 @@ public void testServiceWith1Worker() throws InterruptedException { LOG.error("Failed to start master", e); exceptions[1] = e; } finally { - countDownLatch.countDown(); + /** + * It must close the service first. The pool will be shutdown + * if count down is executed first, and the server thread in + * master service will not be closed. + */ masterService.close(); + countDownLatch.countDown(); } }); @@ -128,6 +137,8 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); @@ -139,8 +150,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start worker", e); exceptions[0] = e; } finally { - countDownLatch.countDown(); workerService.close(); + countDownLatch.countDown(); } }); @@ -157,6 +168,8 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.WORKER_COMPUTATION_CLASS, MockComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName() ); WorkerService workerService = new MockWorkerService(); @@ -168,8 +181,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start worker", e); exceptions[1] = e; } finally { - countDownLatch.countDown(); workerService.close(); + countDownLatch.countDown(); } }); @@ -186,6 +199,8 @@ public void testServiceWith2Workers() throws InterruptedException { ComputerOptions.MASTER_COMPUTATION_CLASS, MockMasterComputation2.class.getName(), ComputerOptions.ALGORITHM_RESULT_CLASS, + DoubleValue.class.getName(), + ComputerOptions.ALGORITHM_MESSAGE_CLASS, DoubleValue.class.getName() ); MasterService masterService = new MasterService(); @@ -196,8 +211,8 @@ public void testServiceWith2Workers() throws InterruptedException { LOG.error("Failed to start master", e); exceptions[2] = e; } finally { - countDownLatch.countDown(); masterService.close(); + countDownLatch.countDown(); } }); From 0d7e4fd041f147c70dd0163a2c37dc4155e0e1de Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Fri, 2 Jul 2021 11:51:31 +0800 Subject: [PATCH 5/9] remove NetworkTestSuite --- .../com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java index f16deac8c..426eab7f9 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java @@ -36,7 +36,6 @@ import com.baidu.hugegraph.computer.core.graph.value.LongValue; import com.baidu.hugegraph.computer.core.input.InputTestSuite; import com.baidu.hugegraph.computer.core.io.IOTestSuite; -import com.baidu.hugegraph.computer.core.network.NetworkTestSuite; import com.baidu.hugegraph.computer.core.receiver.ReceiverTestSuite; import com.baidu.hugegraph.computer.core.sender.SenderTestSuite; import com.baidu.hugegraph.computer.core.sort.sorter.SorterTestSuite; @@ -59,7 +58,7 @@ IOTestSuite.class, InputTestSuite.class, WorkerTestSuite.class, - NetworkTestSuite.class, + // NetworkTestSuite.class, StoreTestSuite.class, SorterTestSuite.class, SortingTestSuite.class, From d788a8a0cd3180ea6491758d0d2f13820aead713 Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Mon, 5 Jul 2021 19:54:19 +0800 Subject: [PATCH 6/9] bug fix BufferedFileInput#seek can seek to the end of file --- .../hugegraph/computer/core/io/BufferedFileInput.java | 2 +- .../computer/core/network/buffer/NettyManagedBuffer.java | 7 +++---- .../baidu/hugegraph/computer/core/io/BufferedFileTest.java | 2 +- .../computer/core/worker/MockMasterComputation.java | 2 -- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileInput.java index bd556243c..d92127b8f 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileInput.java @@ -73,7 +73,7 @@ public void seek(long position) throws IOException { super.seek(position - bufferStart); return; } - if (position >= this.fileLength()) { + if (position > this.fileLength()) { throw new EOFException(String.format( "Can't seek to %s, reach the end of file", position)); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java index 0244d35af..8c662824a 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/network/buffer/NettyManagedBuffer.java @@ -69,10 +69,9 @@ public ByteBuf nettyByteBuf() { @Override public byte[] copyToByteArray() { - ByteBuf buf = this.buf.duplicate(); - return ByteBufUtil.getBytes(buf, - buf.readerIndex(), - buf.readableBytes(), + return ByteBufUtil.getBytes(this.buf, + this.buf.readerIndex(), + this.buf.readableBytes(), true); } } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileTest.java index 1590a3d89..30520ffca 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/io/BufferedFileTest.java @@ -247,7 +247,7 @@ public void testInputSeekOutRange() throws IOException { Assert.assertEquals(1, input.readInt()); input.skip(4); Assert.assertThrows(EOFException.class, () -> { - input.seek(12); // Out of range + input.seek(13); // Out of range }, e -> { Assert.assertContains("reach the end of file", e.getMessage()); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java index 21a91d716..79afbc8a7 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/MockMasterComputation.java @@ -165,8 +165,6 @@ private void registerAggregatorWithError(MasterContext context) { @Override public boolean compute(MasterComputationContext context) { - this.assertStat(context); - if (context.superstep() == 0) { this.assertStep0Aggregators(context); this.updateStep0Aggregators(context); From 040c6d639757db72a055b29dec4e0be57efe810a Mon Sep 17 00:00:00 2001 From: Hou Zhizhen Date: Mon, 5 Jul 2021 20:24:45 +0800 Subject: [PATCH 7/9] move EmptyEdges to EdgesInput and other improve --- .../computer/core/compute/ComputeManager.java | 7 +- .../core/compute/FileGraphPartition.java | 14 ++-- .../core/compute/input/EdgesInput.java | 29 +++++++ .../core/compute/input/EmptyEdges.java | 55 ------------- .../core/compute/input/MessageInput.java | 79 ++++++++++--------- .../core/compute/input/VertexInput.java | 3 +- .../core/graph/partition/PartitionStat.java | 10 +-- .../computer/core/output/LogOutput.java | 4 +- .../core/receiver/MessageRecvManager.java | 2 +- .../core/receiver/MessageRecvPartition.java | 4 +- .../core/receiver/MessageRecvPartitions.java | 4 +- .../{RecvStat.java => RecvMessageStat.java} | 4 +- .../core/sort/sorting/SortManager.java | 3 +- .../graph/partition/PartitionStatTest.java | 8 +- .../suite/integrate/MockComputation.java | 4 +- .../suite/integrate/SenderIntegrateTest.java | 52 +++++++----- 16 files changed, 143 insertions(+), 139 deletions(-) delete mode 100644 computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java rename computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/{RecvStat.java => RecvMessageStat.java} (92%) diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java index 961527a13..9c8eccbf6 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java @@ -30,7 +30,7 @@ import com.baidu.hugegraph.computer.core.manager.Managers; import com.baidu.hugegraph.computer.core.network.message.MessageType; import com.baidu.hugegraph.computer.core.receiver.MessageRecvManager; -import com.baidu.hugegraph.computer.core.receiver.RecvStat; +import com.baidu.hugegraph.computer.core.receiver.RecvMessageStat; import com.baidu.hugegraph.computer.core.sender.MessageSendManager; import com.baidu.hugegraph.computer.core.sort.flusher.PeekableIterator; import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntry; @@ -83,7 +83,8 @@ public WorkerStat input() { } /** - * Get compute-messages from MessageRecvManager. Be called before + * Get compute-messages from MessageRecvManager, then put message to + * corresponding partition. Be called before * {@link MessageRecvManager#beforeSuperstep} is called. */ public void takeComputeMessages() { @@ -117,7 +118,7 @@ public WorkerStat compute(ComputationContext context, int superstep) { } this.sendManager.finishSend(MessageType.MSG); // After compute and send finish signal. - Map recvStats = this.recvManager.recvStats(); + Map recvStats = this.recvManager.recvStats(); for (Map.Entry entry : partitionStats.entrySet()) { PartitionStat partitionStat = entry.getValue(); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java index 77b468707..4986b2597 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java @@ -62,6 +62,8 @@ public class FileGraphPartition> { private final File vertexFile; private final File edgeFile; + private final Value result; + private File preStatusFile; private File curStatusFile; private File preValueFile; @@ -77,7 +79,6 @@ public class FileGraphPartition> { private VertexInput vertexInput; private EdgesInput edgesInput; private MessageInput messageInput; - private final Value result; public FileGraphPartition(ComputerContext context, Managers managers, int partition) { @@ -124,8 +125,7 @@ public PartitionStat init(PeekableIterator vertices, this.edgeCount); } - // Package access - PartitionStat compute0(ComputationContext context, + protected PartitionStat compute0(ComputationContext context, Computation computation) { long activeVertexCount = 0L; this.beforeCompute(0); @@ -147,8 +147,7 @@ PartitionStat compute0(ComputationContext context, this.vertexCount - activeVertexCount, 0L, 0L); } - // Package access - PartitionStat compute(ComputationContext context, + protected PartitionStat compute(ComputationContext context, Computation computation, int superstep) { this.beforeCompute(superstep); @@ -223,6 +222,11 @@ public PartitionStat output() { this.edgeCount); } + /** + * Put the messages sent at previous superstep from MessageRecvManager to + * this partition. The messages is null if no messages sent to this + * partition at previous superstep. + */ public void messages(PeekableIterator messages) { this.messageInput = new MessageInput<>(this.context, messages); } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java index 9db003fac..6dd3bb198 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.Iterator; import javax.annotation.Nonnull; @@ -251,3 +252,31 @@ private Edges readEdges(RandomAccessInput in) { } } } + +class EmptyEdges implements Edges { + + private static final EmptyEdges INSTANCE = new EmptyEdges(); + + private EmptyEdges() { + // pass + } + + public static EmptyEdges instance() { + return INSTANCE; + } + + @Override + public int size() { + return 0; + } + + @Override + public void add(Edge edge) { + throw new ComputerException("Can't add edge '{}'", edge); + } + + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } +} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java deleted file mode 100644 index 1ac808cef..000000000 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdges.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.computer.core.compute.input; - -import java.util.Collections; -import java.util.Iterator; - -import com.baidu.hugegraph.computer.core.common.exception.ComputerException; -import com.baidu.hugegraph.computer.core.graph.edge.Edge; -import com.baidu.hugegraph.computer.core.graph.edge.Edges; - -public class EmptyEdges implements Edges { - - private static final EmptyEdges INSTANCE = new EmptyEdges(); - - private EmptyEdges() { - // pass - } - - public static EmptyEdges instance() { - return INSTANCE; - } - - @Override - public int size() { - return 0; - } - - @Override - public void add(Edge edge) { - throw new ComputerException("Can't add edge '{}'", edge); - } - - @Override - public Iterator iterator() { - return Collections.emptyIterator(); - } -} diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java index fe226a7b9..517feef19 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java @@ -51,11 +51,11 @@ public MessageInput(ComputerContext context, ComputerOptions.ALGORITHM_MESSAGE_CLASS); } - public Iterator iterator(ReusablePointer vid) { + public Iterator iterator(ReusablePointer vidPointer) { while (this.messages.hasNext()) { KvEntry entry = this.messages.peek(); Pointer key = entry.key(); - int status = vid.compareTo(key); + int status = vidPointer.compareTo(key); if (status < 0) { return Collections.emptyIterator(); } else if (status == 0) { @@ -65,48 +65,55 @@ public Iterator iterator(ReusablePointer vid) { } } - return new Iterator() { + return new MessageIterator(vidPointer); + } - // It indicates whether the value can be returned to client. - boolean valueValid = false; + private class MessageIterator implements Iterator { - @Override - public boolean hasNext() { - if (this.valueValid) { - return true; - } - if (MessageInput.this.messages.hasNext()) { - KvEntry entry = MessageInput.this.messages.peek(); - Pointer key = entry.key(); - int status = vid.compareTo(key); - if (status == 0) { - MessageInput.this.messages.next(); - this.valueValid = true; - try { - BytesInput in = IOFactory.createBytesInput( - entry.value().bytes()); - MessageInput.this.value.read(in); - } catch (IOException e) { - throw new ComputerException("Can't read value", e); - } - return true; - } else { - return false; + // It indicates whether the value can be returned to client. + boolean valueValid = false; + private ReusablePointer vidPointer; + + private MessageIterator(ReusablePointer vidPointer) { + this.vidPointer = vidPointer; + } + + @Override + public boolean hasNext() { + if (this.valueValid) { + return true; + } + if (MessageInput.this.messages.hasNext()) { + KvEntry entry = MessageInput.this.messages.peek(); + Pointer key = entry.key(); + int status = this.vidPointer.compareTo(key); + if (status == 0) { + MessageInput.this.messages.next(); + this.valueValid = true; + try { + BytesInput in = IOFactory.createBytesInput( + entry.value().bytes()); + MessageInput.this.value.read(in); + } catch (IOException e) { + throw new ComputerException("Can't read value", e); } + return true; } else { return false; } + } else { + return false; } + } - @Override - public T next() { - if (this.valueValid) { - this.valueValid = false; - return MessageInput.this.value; - } else { - throw new NoSuchElementException(); - } + @Override + public T next() { + if (this.valueValid) { + this.valueValid = false; + return MessageInput.this.value; + } else { + throw new NoSuchElementException(); } - }; + } } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java index f511327b8..56c0490dc 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/VertexInput.java @@ -75,7 +75,8 @@ public Vertex next() { this.vertex.id(StreamGraphInput.readId(this.idPointer.input())); this.vertex.properties(this.properties); } catch (IOException e) { - throw new ComputerException("Can't read from input", e); + throw new ComputerException("Can't read vertex from input '%s'", + e, this.vertexFile.getAbsolutePath()); } return this.vertex; } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java index e12d27919..a6eacab6f 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStat.java @@ -25,7 +25,7 @@ import com.baidu.hugegraph.computer.core.io.RandomAccessOutput; import com.baidu.hugegraph.computer.core.io.Readable; import com.baidu.hugegraph.computer.core.io.Writable; -import com.baidu.hugegraph.computer.core.receiver.RecvStat; +import com.baidu.hugegraph.computer.core.receiver.RecvMessageStat; import com.baidu.hugegraph.computer.core.util.JsonUtil; public class PartitionStat implements Readable, Writable { @@ -103,10 +103,10 @@ public void write(RandomAccessOutput out) throws IOException { out.writeLong(this.messageBytes); } - public void merge(RecvStat recvStat) { - if (recvStat != null) { - this.messageCount += recvStat.messageCount(); - this.messageBytes += recvStat.messageBytes(); + public void merge(RecvMessageStat recvMessageStat) { + if (recvMessageStat != null) { + this.messageCount += recvMessageStat.messageCount(); + this.messageBytes += recvMessageStat.messageBytes(); } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java index d7aeebd84..97885c475 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/output/LogOutput.java @@ -39,7 +39,7 @@ public class LogOutput implements ComputerOutput { @Override public void init(Config config, int partition) { this.partition = partition; - LOG.info("begin output partition {}", this.partition); + LOG.info("Start write back partition {}", this.partition); } @Override @@ -51,6 +51,6 @@ public void write(Vertex vertex) { @Override public void close() { - LOG.info("close output partition {}", partition); + LOG.info("End write back partition {}", this.partition); } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java index 405c840f6..122798e67 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java @@ -218,7 +218,7 @@ public Map> messagePartitions() { return partitions.iterators(); } - public Map recvStats() { + public Map recvStats() { this.waitReceivedAllMessages(); E.checkState(this.messagePartitions != null, "The messagePartitions can't be null"); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java index fa29c4343..2991a43c3 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java @@ -115,8 +115,8 @@ public long totalBytes() { return this.totalBytes; } - public RecvStat recvStat() { - return new RecvStat(0L, this.totalBytes); + public RecvMessageStat recvStat() { + return new RecvMessageStat(0L, this.totalBytes); } protected abstract OuterSortFlusher outerSortFlusher(); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java index 3406f35b1..d04f09ff6 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java @@ -83,8 +83,8 @@ public Map> iterators() { return entries; } - public Map recvStats() { - Map entries = new HashMap<>(); + public Map recvStats() { + Map entries = new HashMap<>(); for (Map.Entry entry : this.partitions.entrySet()) { entries.put(entry.getKey(), entry.getValue().recvStat()); } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvMessageStat.java similarity index 92% rename from computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java rename to computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvMessageStat.java index 71e9af79f..01bf2f3e0 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvStat.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/RecvMessageStat.java @@ -22,12 +22,12 @@ /** * Received message stat for a partition. */ -public class RecvStat { +public class RecvMessageStat { private final long messageCount; private final long messageBytes; - public RecvStat(long messageCount, long messageBytes) { + public RecvMessageStat(long messageCount, long messageBytes) { this.messageCount = messageCount; this.messageBytes = messageBytes; } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java index 1df083cb8..2a67ec7cb 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/sort/sorting/SortManager.java @@ -117,7 +117,8 @@ public CompletableFuture sort(MessageType type, throw new ComputerException("Failed to sort buffers of %s " + "message", e, type.name()); } - return ByteBuffer.wrap(output.toByteArray()); + + return ByteBuffer.wrap(output.buffer(), 0, (int) output.position()); }, this.sortExecutor); } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java index fa8d9ce4f..ce99bc774 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/graph/partition/PartitionStatTest.java @@ -23,7 +23,7 @@ import org.junit.Test; -import com.baidu.hugegraph.computer.core.receiver.RecvStat; +import com.baidu.hugegraph.computer.core.receiver.RecvMessageStat; import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; import com.baidu.hugegraph.testutil.Assert; @@ -65,13 +65,13 @@ public void testMerge() throws IOException { Assert.assertEquals(0L, stat1.messageCount()); Assert.assertEquals(0L, stat1.messageBytes()); - RecvStat recvStat = new RecvStat(100L, 400L); - stat1.merge(recvStat); + RecvMessageStat recvMessageStat = new RecvMessageStat(100L, 400L); + stat1.merge(recvMessageStat); Assert.assertEquals(1L, stat1.vertexCount()); Assert.assertEquals(2L, stat1.edgeCount()); Assert.assertEquals(100L, stat1.messageCount()); Assert.assertEquals(400L, stat1.messageBytes()); - stat1.merge(recvStat); + stat1.merge(recvMessageStat); Assert.assertEquals(1L, stat1.vertexCount()); Assert.assertEquals(2L, stat1.edgeCount()); Assert.assertEquals(200L, stat1.messageCount()); diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/MockComputation.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/MockComputation.java index 492e2eded..ee763c6b5 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/MockComputation.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/MockComputation.java @@ -40,12 +40,12 @@ public String category() { @Override public void compute0(ComputationContext context, Vertex vertex) { - + vertex.value(new DoubleValue(0.5D)); } @Override public void compute(ComputationContext context, Vertex vertex, Iterator messages) { - + // pass } } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/SenderIntegrateTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/SenderIntegrateTest.java index 20aa4a4ad..50b69c7ee 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/SenderIntegrateTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/integrate/SenderIntegrateTest.java @@ -31,6 +31,7 @@ import com.baidu.hugegraph.computer.core.common.ComputerContext; import com.baidu.hugegraph.computer.core.common.exception.TransportException; import com.baidu.hugegraph.computer.core.config.ComputerOptions; +import com.baidu.hugegraph.computer.core.graph.value.DoubleValue; import com.baidu.hugegraph.computer.core.manager.Managers; import com.baidu.hugegraph.computer.core.master.MasterService; import com.baidu.hugegraph.computer.core.network.DataClientManager; @@ -63,8 +64,9 @@ public void testOneWorker() throws InterruptedException { Thread masterThread = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_002") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(1) @@ -88,8 +90,9 @@ public void testOneWorker() throws InterruptedException { Thread workerThread = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_002") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(1) @@ -123,8 +126,9 @@ public void testTwoWorkers() throws InterruptedException { Thread masterThread = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_003") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(2) @@ -143,8 +147,9 @@ public void testTwoWorkers() throws InterruptedException { Thread workerThread1 = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_003") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(2) @@ -164,8 +169,9 @@ public void testTwoWorkers() throws InterruptedException { Thread workerThread2 = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_003") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(2) @@ -197,8 +203,9 @@ public void testOneWorkerWithBusyClient() throws InterruptedException { Thread masterThread = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_002") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(1) @@ -222,8 +229,9 @@ public void testOneWorkerWithBusyClient() throws InterruptedException { Thread workerThread = new Thread(() -> { String[] args = OptionsBuilder.newInstance() .withJobId("local_002") - .withValueName("rank") - .withValueType("DOUBLE") + .withResultName("rank") + .withResultClass(DoubleValue.class) + .withMessageClass(DoubleValue.class) .withMaxSuperStep(3) .withComputationClass(COMPUTATION) .withWorkerCount(1) @@ -317,13 +325,21 @@ public OptionsBuilder withJobId(String jobId) { return this; } - public OptionsBuilder withValueName(String name) { - this.options.add(name); + public OptionsBuilder withResultClass(Class clazz) { + this.options.add(ComputerOptions.ALGORITHM_RESULT_CLASS.name()); + this.options.add(clazz.getName()); return this; } - public OptionsBuilder withValueType(String type) { - this.options.add(type); + public OptionsBuilder withMessageClass(Class clazz) { + this.options.add(ComputerOptions.ALGORITHM_MESSAGE_CLASS.name()); + this.options.add(clazz.getName()); + return this; + } + + public OptionsBuilder withResultName(String name) { + this.options.add(ComputerOptions.OUTPUT_RESULT_NAME.name()); + this.options.add(name); return this; } From 7def657bf3bc1345cf729d5ea425394a64313dc9 Mon Sep 17 00:00:00 2001 From: houzhizhen Date: Wed, 7 Jul 2021 11:32:42 +0800 Subject: [PATCH 8/9] delete EmptyEdgesTest,add ResuablePointerTest --- .../computer/core/compute/ComputeManager.java | 3 +- .../core/compute/FileGraphPartition.java | 208 ++++++++++-------- .../core/compute/input/EdgesInput.java | 121 +++++----- .../core/compute/input/MessageInput.java | 3 +- .../core/compute/input/ReusablePointer.java | 7 +- .../core/receiver/MessageRecvManager.java | 4 +- .../core/receiver/MessageRecvPartition.java | 2 +- .../core/receiver/MessageRecvPartitions.java | 4 +- .../core/compute/ComputeTestSuite.java | 4 +- .../core/compute/input/EdgesInputTest.java | 20 +- .../core/compute/input/EmptyEdgesTest.java | 45 ---- .../compute/input/ResuablePointerTest.java | 55 +++++ .../core/worker/WorkerServiceTest.java | 2 +- 13 files changed, 269 insertions(+), 209 deletions(-) delete mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java create mode 100644 computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ResuablePointerTest.java diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java index 9c8eccbf6..aea8cc6d7 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeManager.java @@ -118,7 +118,8 @@ public WorkerStat compute(ComputationContext context, int superstep) { } this.sendManager.finishSend(MessageType.MSG); // After compute and send finish signal. - Map recvStats = this.recvManager.recvStats(); + Map recvStats = + this.recvManager.recvMessageStats(); for (Map.Entry entry : partitionStats.entrySet()) { PartitionStat partitionStat = entry.getValue(); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java index 4986b2597..53708d11b 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/FileGraphPartition.java @@ -80,8 +80,9 @@ public class FileGraphPartition> { private EdgesInput edgesInput; private MessageInput messageInput; - public FileGraphPartition(ComputerContext context, Managers managers, - int partition) { + public FileGraphPartition(ComputerContext context, + Managers managers, + int partition) { this.context = context; this.fileGenerator = managers.get(FileManager.NAME); this.partition = partition; @@ -118,7 +119,9 @@ public PartitionStat init(PeekableIterator vertices, vertexOut.close(); edgeOut.close(); } catch (IOException e) { - throw new ComputerException("File graph partition init failed", e); + throw new ComputerException( + "Failed to init FileGraphPartition '%s'", + e, this.partition); } return new PartitionStat(this.partition, this.vertexCount, @@ -126,10 +129,14 @@ public PartitionStat init(PeekableIterator vertices, } protected PartitionStat compute0(ComputationContext context, - Computation computation) { + Computation computation) { long activeVertexCount = 0L; - this.beforeCompute(0); - + try { + this.beforeCompute(0); + } catch (IOException e) { + throw new ComputerException( + "Error occurred when beforeCompute at superstep 0", e); + } while (this.vertexInput.hasNext()) { Vertex vertex = this.vertexInput.next(); Edges edges = this.edgesInput.edges(this.vertexInput.idPointer()); @@ -138,19 +145,33 @@ protected PartitionStat compute0(ComputationContext context, if (vertex.active()) { activeVertexCount++; } - this.saveVertex(vertex); + try { + this.saveVertex(vertex); + } catch (IOException e) { + throw new ComputerException( + "Error occurred when saveVertex", e); + } + } + try { + this.afterCompute(0); + } catch (IOException e) { + throw new ComputerException("Error occurred when afterCompute", e); } - this.afterCompute(0); - return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, this.vertexCount - activeVertexCount, 0L, 0L); } protected PartitionStat compute(ComputationContext context, - Computation computation, - int superstep) { - this.beforeCompute(superstep); + Computation computation, + int superstep) { + try { + this.beforeCompute(superstep); + } catch (IOException e) { + throw new ComputerException( + "Error occurred when beforeCompute at superstep %s", + e, superstep); + } long activeVertexCount = 0L; while (this.vertexInput.hasNext()) { Vertex vertex = this.vertexInput.next(); @@ -184,9 +205,20 @@ protected PartitionStat compute(ComputationContext context, if (vertex.active()) { activeVertexCount++; } - this.saveVertex(vertex); + try { + this.saveVertex(vertex); + } catch (IOException e) { + throw new ComputerException( + "Error occurred when saveVertex", e); + } + } + try { + this.afterCompute(superstep); + } catch (IOException e) { + throw new ComputerException( + "Error occurred when afterCompute at superstep %s", + e, superstep); } - this.afterCompute(superstep); return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, this.vertexCount - activeVertexCount, 0L, 0L); @@ -196,7 +228,12 @@ public PartitionStat output() { ComputerOutput output = this.context.config().createObject( ComputerOptions.OUTPUT_CLASS); output.init(this.context.config(), this.partition); - this.beforeOutput(); + try { + this.beforeOutput(); + } catch (IOException e) { + throw new ComputerException("Error occurred when beforeOutput", e); + } + while (this.vertexInput.hasNext()) { Vertex vertex = this.vertexInput.next(); try { @@ -216,7 +253,12 @@ public PartitionStat output() { vertex.edges(edges); output.write(vertex); } - this.afterOutput(); + + try { + this.afterOutput(); + } catch (IOException e) { + throw new ComputerException("Error occurred when afterOutput", e); + } output.close(); return new PartitionStat(this.partition, this.vertexCount, this.edgeCount); @@ -235,16 +277,11 @@ public int partition() { return this.partition; } - private void saveVertex(Vertex vertex) { - try { - this.curStatusOut.writeBoolean(vertex.active()); - Value value = vertex.value(); - E.checkNotNull(value, "Vertex's value can't be null"); - vertex.value().write(this.curValueOut); - } catch (IOException e) { - throw new ComputerException("Save vertex failed, vertex id {}", - vertex.id()); - } + private void saveVertex(Vertex vertex) throws IOException { + this.curStatusOut.writeBoolean(vertex.active()); + Value value = vertex.value(); + E.checkNotNull(value, "Vertex's value can't be null"); + vertex.value().write(this.curValueOut); } private void writeEdges(Pointer id, PeekableIterator edges, @@ -273,95 +310,80 @@ private void writeEdges(Pointer id, PeekableIterator edges, // Not write sub-value length edgeOut.write(subEntry.value().bytes()); } - int valueLength = (int) (edgeOut.position() - valuePosition - - Constants.INT_LEN); - edgeOut.writeFixedInt(valuePosition, valueLength); + long valueLength = edgeOut.position() - valuePosition - + Constants.INT_LEN; + edgeOut.writeFixedInt(valuePosition, (int) valueLength); } else { // status > 0 edges.next(); } } } - private void beforeCompute(int superstep) { + private void beforeCompute(int superstep) throws IOException { this.vertexInput = new VertexInput(this.context, this.vertexFile, this.vertexCount); this.edgesInput = new EdgesInput(this.context, this.edgeFile); - // Inputs - try { - this.vertexInput.init(); - this.edgesInput.init(); - if (superstep != 0) { - this.preStatusFile = this.curStatusFile; - this.preValueFile = this.curValueFile; - this.preStatusIn = new BufferedFileInput(this.preStatusFile); - this.preValueIn = new BufferedFileInput(this.preValueFile); - } - - // Outputs - String statusPath = this.fileGenerator.randomDirectory( - STATUS, Integer.toString(superstep), - Integer.toString(this.partition)); - String valuePath = this.fileGenerator.randomDirectory( - VALUE, Integer.toString(superstep), - Integer.toString(this.partition)); - this.curStatusFile = new File(statusPath); - this.curValueFile = new File(valuePath); - createFile(this.curStatusFile); - createFile(this.curValueFile); - - this.curStatusOut = new BufferedFileOutput(this.curStatusFile); - this.curValueOut = new BufferedFileOutput(this.curValueFile); - } catch (IOException e) { - throw new ComputerException("Before compute call failed", e); + // Inputs of vertex, edges, status, and value. + this.vertexInput.init(); + this.edgesInput.init(); + if (superstep != 0) { + this.preStatusFile = this.curStatusFile; + this.preValueFile = this.curValueFile; + this.preStatusIn = new BufferedFileInput(this.preStatusFile); + this.preValueIn = new BufferedFileInput(this.preValueFile); } + + // Outputs of vertex's status and vertex's value. + String statusPath = this.fileGenerator.randomDirectory( + STATUS, Integer.toString(superstep), + Integer.toString(this.partition)); + String valuePath = this.fileGenerator.randomDirectory( + VALUE, Integer.toString(superstep), + Integer.toString(this.partition)); + this.curStatusFile = new File(statusPath); + this.curValueFile = new File(valuePath); + createFile(this.curStatusFile); + createFile(this.curValueFile); + + this.curStatusOut = new BufferedFileOutput(this.curStatusFile); + this.curValueOut = new BufferedFileOutput(this.curValueFile); } - private void afterCompute(int superstep) { - try { - this.vertexInput.close(); - this.edgesInput.close(); - if (superstep != 0) { - this.preStatusIn.close(); - this.preValueIn.close(); - this.preStatusFile.delete(); - this.preValueFile.delete(); - } - this.curStatusOut.close(); - this.curValueOut.close(); - } catch (IOException e) { - throw new ComputerException("After compute call failed", e); + private void afterCompute(int superstep) throws IOException { + this.vertexInput.close(); + this.edgesInput.close(); + if (superstep != 0) { + this.preStatusIn.close(); + this.preValueIn.close(); + this.preStatusFile.delete(); + this.preValueFile.delete(); } + this.curStatusOut.close(); + this.curValueOut.close(); } - private void beforeOutput() { + private void beforeOutput() throws IOException { this.vertexInput = new VertexInput(this.context, this.vertexFile, this.vertexCount); this.edgesInput = new EdgesInput(this.context, this.edgeFile); - try { - this.vertexInput.init(); - this.edgesInput.init(); - this.preStatusFile = this.curStatusFile; - this.preValueFile = this.curValueFile; - this.preStatusIn = new BufferedFileInput(this.preStatusFile); - this.preValueIn = new BufferedFileInput(this.preValueFile); - } catch (IOException e) { - throw new ComputerException("Before output call failed", e); - } + this.vertexInput.init(); + this.edgesInput.init(); + + this.preStatusFile = this.curStatusFile; + this.preValueFile = this.curValueFile; + this.preStatusIn = new BufferedFileInput(this.preStatusFile); + this.preValueIn = new BufferedFileInput(this.preValueFile); } - private void afterOutput() { - try { - this.vertexInput.close(); - this.edgesInput.close(); - this.preStatusIn.close(); - this.preValueIn.close(); - this.preStatusFile.delete(); - this.preValueFile.delete(); - } catch (IOException e) { - throw new ComputerException("After output call failed", e); - } + private void afterOutput() throws IOException { + this.vertexInput.close(); + this.edgesInput.close(); + this.preStatusIn.close(); + this.preValueIn.close(); + this.preStatusFile.delete(); + this.preValueFile.delete(); } private static void createFile(File file) throws IOException { diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java index 6dd3bb198..1fb69b005 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInput.java @@ -99,24 +99,23 @@ public Edges edges(ReusablePointer vidPointer) { } return EmptyEdges.instance(); } catch (IOException e) { - throw new ComputerException("Can't read from input", e); + throw new ComputerException("Can't read from edges input '%s'", + e, this.edgeFile.getAbsoluteFile()); } } private class SuperEdges implements Edges { - private ReusablePointer vid; + private final ReusablePointer vid; private final long startPosition; - private Iterator currentEdgeIt; + private Iterator currentEdgesIter; private int size; private int iterationTime; - SuperEdges(ReusablePointer vid, - Edges edges, - long startPosition) { + SuperEdges(ReusablePointer vid, Edges edges, long startPosition) { this.vid = vid; this.startPosition = startPosition; - this.currentEdgeIt = edges.iterator(); + this.currentEdgesIter = edges.iterator(); this.size = 0; this.iterationTime = 0; } @@ -153,7 +152,8 @@ private void computeSize() { @Override public void add(Edge edge) { - throw new ComputerException("Can't add edge"); + throw new ComputerException( + "Not support adding edges during computing"); } @Override @@ -168,44 +168,48 @@ public Iterator iterator() { throw new ComputerException("Can't seek to %s", e, this.startPosition); } - return new Iterator() { + return new EdgesIterator(); - @Override - public boolean hasNext() { - if (currentEdgeIt.hasNext()) { - return true; - } else { - long currentPosition = input.position(); - try { - if (input.available() > 0) { - idPointer.read(input); - if (idPointer.compareTo(vid) == 0) { - valuePointer.read(EdgesInput.this.input); - currentEdgeIt = readEdges( - valuePointer.input()) + } + + private class EdgesIterator implements Iterator { + + @Override + public boolean hasNext() { + if (currentEdgesIter.hasNext()) { + return true; + } else { + long currentPosition = input.position(); + try { + if (input.available() > 0) { + idPointer.read(input); + if (idPointer.compareTo(vid) == 0) { + valuePointer.read(EdgesInput.this.input); + currentEdgesIter = readEdges( + valuePointer.input()) .iterator(); - } else { - input.seek(currentPosition); - } + } else { + input.seek(currentPosition); } - } catch (IOException e) { - throw new ComputerException( - "Read Edges from position %s failed", - currentPosition); } + } catch (IOException e) { + throw new ComputerException( + "Error occurred when read edges from edges " + + "input '%s' at position %s", e, + edgeFile.getAbsoluteFile(), currentPosition); } - return currentEdgeIt.hasNext(); } + return currentEdgesIter.hasNext(); + } - @Override - public Edge next() { - return currentEdgeIt.next(); - } - }; + @Override + public Edge next() { + return currentEdgesIter.next(); + } } } - // TODO: use one batch of edges to read batches for each vertex. + // TODO: use one reused Edges instance to read batches for each vertex. private Edges readEdges(RandomAccessInput in) { try { int count = in.readFixedInt(); @@ -248,35 +252,36 @@ private Edges readEdges(RandomAccessInput in) { } return edges; } catch (IOException e) { - throw new ComputerException("Read edges failed", e); + throw new ComputerException("Failed to read edges from input '%s'", + e, this.edgeFile.getAbsoluteFile()); } } -} - -class EmptyEdges implements Edges { + public static class EmptyEdges implements Edges { - private static final EmptyEdges INSTANCE = new EmptyEdges(); + private static final EmptyEdges INSTANCE = new EmptyEdges(); - private EmptyEdges() { - // pass - } + private EmptyEdges() { + // pass + } - public static EmptyEdges instance() { - return INSTANCE; - } + public static EmptyEdges instance() { + return INSTANCE; + } - @Override - public int size() { - return 0; - } + @Override + public int size() { + return 0; + } - @Override - public void add(Edge edge) { - throw new ComputerException("Can't add edge '{}'", edge); - } + @Override + public void add(Edge edge) { + throw new ComputerException( + "Not support adding edges during computing"); + } - @Override - public Iterator iterator() { - return Collections.emptyIterator(); + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java index 517feef19..cdab2fee7 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/MessageInput.java @@ -71,11 +71,12 @@ public Iterator iterator(ReusablePointer vidPointer) { private class MessageIterator implements Iterator { // It indicates whether the value can be returned to client. - boolean valueValid = false; + private boolean valueValid; private ReusablePointer vidPointer; private MessageIterator(ReusablePointer vidPointer) { this.vidPointer = vidPointer; + this.valueValid = false; } @Override diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java index bb6c9a106..923175918 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ReusablePointer.java @@ -63,11 +63,16 @@ public RandomAccessInput input() { try { this.input.seek(0L); } catch (IOException e) { - throw new ComputerException("Seek to position 0 failed", e); + throw new ComputerException( + "ResuablePointer can't seek to position 0", e); } return this.input; } + /** + * Only [0 .. length) of the returned byte array is valid. The extra data + * [length .. bytes.length) is meaningless, may be left by previous pointer. + */ @Override public byte[] bytes() { return this.bytes; diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java index 122798e67..972b22f16 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvManager.java @@ -218,10 +218,10 @@ public Map> messagePartitions() { return partitions.iterators(); } - public Map recvStats() { + public Map recvMessageStats() { this.waitReceivedAllMessages(); E.checkState(this.messagePartitions != null, "The messagePartitions can't be null"); - return this.messagePartitions.recvStats(); + return this.messagePartitions.recvMessageStats(); } } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java index 2991a43c3..4496dd33a 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartition.java @@ -115,7 +115,7 @@ public long totalBytes() { return this.totalBytes; } - public RecvMessageStat recvStat() { + public RecvMessageStat recvMessageStat() { return new RecvMessageStat(0L, this.totalBytes); } diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java index d04f09ff6..2f4c7b3d8 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/receiver/MessageRecvPartitions.java @@ -83,10 +83,10 @@ public Map> iterators() { return entries; } - public Map recvStats() { + public Map recvMessageStats() { Map entries = new HashMap<>(); for (Map.Entry entry : this.partitions.entrySet()) { - entries.put(entry.getKey(), entry.getValue().recvStat()); + entries.put(entry.getKey(), entry.getValue().recvMessageStat()); } return entries; } diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java index 09164a24e..7ed1b0a0b 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/ComputeTestSuite.java @@ -23,13 +23,13 @@ import org.junit.runners.Suite; import com.baidu.hugegraph.computer.core.compute.input.EdgesInputTest; -import com.baidu.hugegraph.computer.core.compute.input.EmptyEdgesTest; import com.baidu.hugegraph.computer.core.compute.input.MessageInputTest; +import com.baidu.hugegraph.computer.core.compute.input.ResuablePointerTest; @RunWith(Suite.class) @Suite.SuiteClasses({ EdgesInputTest.class, - EmptyEdgesTest.class, + ResuablePointerTest.class, MessageInputTest.class, ComputeManagerTest.class }) diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java index 0800730c6..d577b4981 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EdgesInputTest.java @@ -26,7 +26,6 @@ import java.util.function.Consumer; import org.junit.After; -import org.junit.Assert; import org.junit.Test; import com.baidu.hugegraph.computer.core.common.Constants; @@ -61,6 +60,7 @@ import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.EntryOutputImpl; import com.baidu.hugegraph.computer.core.store.hgkvfile.entry.KvEntryWriter; import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; +import com.baidu.hugegraph.testutil.Assert; import com.baidu.hugegraph.testutil.Whitebox; public class EdgesInputTest extends UnitTestBase { @@ -70,7 +70,9 @@ public class EdgesInputTest extends UnitTestBase { @After public void teardown() { - this.managers.closeAll(this.config); + if (this.managers != null) { + this.managers.closeAll(this.config); + } } @Test @@ -88,6 +90,20 @@ public void testMultiple() throws IOException { this.testEdgeFreq(EdgeFrequency.MULTIPLE); } + @Test + public void testEmptyEdges() { + EdgesInput.EmptyEdges edges = EdgesInput.EmptyEdges.instance(); + Iterator it = edges.iterator(); + Assert.assertFalse(it.hasNext()); + Assert.assertEquals(0, edges.size()); + Assert.assertThrows(ComputerException.class, () -> { + edges.add(graphFactory().createEdge()); + }, e -> { + Assert.assertContains("Not support adding edges during computing", + e.getMessage()); + }); + } + private void testEdgeFreq(EdgeFrequency freq) throws IOException { this.config = UnitTestBase.updateWithRequiredOptions( diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java deleted file mode 100644 index 64cf9ce79..000000000 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/EmptyEdgesTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2017 HugeGraph Authors - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.baidu.hugegraph.computer.core.compute.input; - -import java.util.Iterator; - -import org.junit.Test; - -import com.baidu.hugegraph.computer.core.common.exception.ComputerException; -import com.baidu.hugegraph.computer.core.graph.edge.Edge; -import com.baidu.hugegraph.computer.suite.unit.UnitTestBase; -import com.baidu.hugegraph.testutil.Assert; - -public class EmptyEdgesTest extends UnitTestBase { - - @Test - public void testEmptyEdges() { - EmptyEdges edges = EmptyEdges.instance(); - Iterator it = edges.iterator(); - Assert.assertFalse(it.hasNext()); - Assert.assertEquals(0, edges.size()); - Assert.assertThrows(ComputerException.class, () -> { - edges.add(graphFactory().createEdge()); - }, e -> { - Assert.assertContains("Can't add edge", e.getMessage()); - }); - } -} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ResuablePointerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ResuablePointerTest.java new file mode 100644 index 000000000..9ee364847 --- /dev/null +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/compute/input/ResuablePointerTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017 HugeGraph Authors + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.baidu.hugegraph.computer.core.compute.input; + +import java.io.IOException; + +import org.junit.Test; + +import com.baidu.hugegraph.computer.core.common.Constants; +import com.baidu.hugegraph.computer.core.io.BytesOutput; +import com.baidu.hugegraph.computer.core.io.IOFactory; +import com.baidu.hugegraph.testutil.Assert; + +public class ResuablePointerTest { + + @Test + public void testReadWrite() throws IOException { + BytesOutput output1 = IOFactory.createBytesOutput(100); + long position = output1.position(); + output1.writeFixedInt(0); + output1.writeInt(Integer.MAX_VALUE); + int length = (int) (output1.position() - position - Constants.INT_LEN); + output1.writeFixedInt(position, length); + + ReusablePointer p1 = new ReusablePointer(); + p1.read(IOFactory.createBytesInput(output1.buffer())); + + BytesOutput output2 = IOFactory.createBytesOutput(100); + p1.write(output2); + + ReusablePointer p2 = new ReusablePointer(); + p2.read(IOFactory.createBytesInput(output2.buffer())); + Assert.assertEquals(0, p1.compareTo(p2)); + Assert.assertEquals(p1.length(), p2.length()); + Assert.assertEquals(0L, p1.offset()); + Assert.assertEquals(0L, p2.offset()); + } +} diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java index bde7dc6ff..a216ca0ca 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerServiceTest.java @@ -101,7 +101,7 @@ public void testServiceWith1Worker() throws InterruptedException { LOG.error("Failed to start master", e); exceptions[1] = e; } finally { - /** + /* * It must close the service first. The pool will be shutdown * if count down is executed first, and the server thread in * master service will not be closed. From 3d383099651d250c901b130f980e66f11b7dd917 Mon Sep 17 00:00:00 2001 From: houzhizhen Date: Wed, 7 Jul 2021 11:44:29 +0800 Subject: [PATCH 9/9] enable test suite NetworkTestSuite --- .../com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java index 426eab7f9..f16deac8c 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/suite/unit/UnitTestSuite.java @@ -36,6 +36,7 @@ import com.baidu.hugegraph.computer.core.graph.value.LongValue; import com.baidu.hugegraph.computer.core.input.InputTestSuite; import com.baidu.hugegraph.computer.core.io.IOTestSuite; +import com.baidu.hugegraph.computer.core.network.NetworkTestSuite; import com.baidu.hugegraph.computer.core.receiver.ReceiverTestSuite; import com.baidu.hugegraph.computer.core.sender.SenderTestSuite; import com.baidu.hugegraph.computer.core.sort.sorter.SorterTestSuite; @@ -58,7 +59,7 @@ IOTestSuite.class, InputTestSuite.class, WorkerTestSuite.class, - // NetworkTestSuite.class, + NetworkTestSuite.class, StoreTestSuite.class, SorterTestSuite.class, SortingTestSuite.class,