From 0b40172fac5def189fd086931495ae0030462ee1 Mon Sep 17 00:00:00 2001 From: Nakata Date: Tue, 18 Apr 2023 21:35:30 +0900 Subject: [PATCH 01/10] Add filesystem path --- .../client/computation_to_db/client.cpp | 28 ++++++++----------- .../client/computation_to_db/client.hpp | 8 ++++-- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/server/computation_container/client/computation_to_db/client.cpp b/packages/server/computation_container/client/computation_to_db/client.cpp index a08b016fd..1d5cc7f3c 100644 --- a/packages/server/computation_container/client/computation_to_db/client.cpp +++ b/packages/server/computation_container/client/computation_to_db/client.cpp @@ -12,8 +12,6 @@ namespace qmpc::ComputationToDb { -namespace fs = std::experimental::filesystem; - nlohmann::json convertSchemaToJson(const qmpc::ComputationToDb::SchemaType &src) { pb_common_types::Schema col_pb; @@ -70,8 +68,8 @@ void Client::ComputationResultWriter::write() {"meta", {{"piece_id", piece_id}, {"column_number", column_number}}}}; const std::string data = piece_data_json.dump(); - auto ofs = - std::ofstream(resultDbPath + job_uuid + "/" + data_name + "_" + std::to_string(piece_id)); + auto data_file = data_name + "_" + std::to_string(piece_id); + auto ofs = std::ofstream(resultDbPath / job_uuid / data_file); ofs << data; ofs.close(); @@ -200,7 +198,7 @@ std::optional>> Client::readTable( const std::string &data_id, int piece_id ) const { - auto data_path = shareDbPath + data_id + "/" + std::to_string(piece_id); + auto data_path = shareDbPath / data_id / std::to_string(piece_id); if (!fs::exists(data_path)) { return std::nullopt; @@ -223,7 +221,7 @@ std::optional>> Client::readTable( std::vector Client::readSchema(const std::string &data_id) const { // DBから値を取り出す - auto ifs = std::ifstream(shareDbPath + data_id + "/0"); + auto ifs = std::ifstream(shareDbPath / data_id / "0"); std::string data; getline(ifs, data); auto json = nlohmann::json::parse(data); @@ -235,8 +233,8 @@ std::vector Client::readSchema(const std::string &data_id) const // shareDBに対してdataを書き込む void Client::writeShareDB(const std::string &data_id, const std::string &data, int piece_id) { - fs::create_directories(shareDbPath + data_id); - auto ofs = std::ofstream(fs::path(shareDbPath) / data_id / std::to_string(piece_id)); + fs::create_directories(shareDbPath / data_id); + auto ofs = std::ofstream(shareDbPath / data_id / std::to_string(piece_id)); ofs << data; ofs.close(); } @@ -244,7 +242,7 @@ void Client::writeShareDB(const std::string &data_id, const std::string &data, i // Job を DB に新規登録する void Client::registerJob(const std::string &job_uuid, const int &status) const { - fs::create_directories(resultDbPath + job_uuid); + fs::create_directories(resultDbPath / job_uuid); updateJobStatus(job_uuid, status); } @@ -254,9 +252,8 @@ void Client::updateJobStatus(const std::string &job_uuid, const int &status) con const google::protobuf::EnumDescriptor *descriptor = google::protobuf::GetEnumDescriptor(); - std::ofstream ofs( - resultDbPath + job_uuid + "/status_" + descriptor->FindValueByNumber(status)->name() - ); + auto status_file = "status_" + descriptor->FindValueByNumber(status)->name(); + std::ofstream ofs(resultDbPath / job_uuid / status_file); std::chrono::system_clock::time_point tp = std::chrono::system_clock::now(); auto tp_msec = std::chrono::duration_cast(tp.time_since_epoch()); @@ -269,10 +266,9 @@ void Client::saveErrorInfo(const std::string &job_uuid, const pb_common_types::J const google::protobuf::EnumDescriptor *descriptor = google::protobuf::GetEnumDescriptor(); - std::ofstream ofs( - resultDbPath + job_uuid + "/status_" - + descriptor->FindValueByNumber(pb_common_types::JobStatus::ERROR)->name() - ); + auto status_file = + "status_" + descriptor->FindValueByNumber(pb_common_types::JobStatus::ERROR)->name(); + std::ofstream ofs(resultDbPath / job_uuid / status_file); static const auto options = []() { diff --git a/packages/server/computation_container/client/computation_to_db/client.hpp b/packages/server/computation_container/client/computation_to_db/client.hpp index 3175bb17f..1363d8bc1 100644 --- a/packages/server/computation_container/client/computation_to_db/client.hpp +++ b/packages/server/computation_container/client/computation_to_db/client.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -13,6 +14,7 @@ namespace qmpc::ComputationToDb { +namespace fs = std::experimental::filesystem; using SchemaType = std::tuple; std::vector @@ -20,8 +22,8 @@ convertSchemaVectorToJsonVector(const std::vector Date: Thu, 20 Apr 2023 19:22:20 +0900 Subject: [PATCH 02/10] Update computationContainerWriter to use Client --- .../client/computation_to_db/client.cpp | 69 +++++++++++++----- .../client/computation_to_db/client.hpp | 70 ++++++++----------- .../job/jobs/correl_job.hpp | 6 +- .../job/jobs/join_table_job.hpp | 20 ++++-- .../job/jobs/math_job.hpp | 6 +- .../job/jobs/mesh_code_job.hpp | 6 +- 6 files changed, 106 insertions(+), 71 deletions(-) diff --git a/packages/server/computation_container/client/computation_to_db/client.cpp b/packages/server/computation_container/client/computation_to_db/client.cpp index 1d5cc7f3c..96bcdb11a 100644 --- a/packages/server/computation_container/client/computation_to_db/client.cpp +++ b/packages/server/computation_container/client/computation_to_db/client.cpp @@ -42,65 +42,79 @@ std::vector convertSchemaVectorToJsonVector( return dst; } -/************ Client::ComputationResultWriter ************/ -Client::ComputationResultWriter::ComputationResultWriter( - const std::string &job_uuid, int key, int column_number, int piece_size +/************ ComputationResultWriter ************/ +ComputationResultWriter::ComputationResultWriter( + const std::string &job_uuid, int data_type, int column_number, int piece_size ) : current_size(0) , piece_id(0) , job_uuid(job_uuid) - , data_name( - (key == 0) ? "dim1" - : (key == 1) ? "dim2" - : "schema" - ) + , data_type(data_type) // NOTE: Client側で復元する際に0以下だと不都合が生じるため , column_number(std::max(1, column_number)) , piece_size(piece_size) { } -void Client::ComputationResultWriter::write() +void ComputationResultWriter::write(bool fin) { nlohmann::json piece_data_json = { {"job_uuid", job_uuid}, {"result", piece_data}, {"meta", {{"piece_id", piece_id}, {"column_number", column_number}}}}; const std::string data = piece_data_json.dump(); - - auto data_file = data_name + "_" + std::to_string(piece_id); - auto ofs = std::ofstream(resultDbPath / job_uuid / data_file); - ofs << data; - ofs.close(); + Client::getInstance()->writeResultDB(job_uuid, data, data_type, piece_id); ++piece_id; current_size = 0; piece_data.clear(); + + if (fin) + { + Client::getInstance()->updateJobCompleted(job_uuid); + } } -void Client::ComputationResultWriter::emplace(const std::string &s) +void ComputationResultWriter::emplace(const std::string &s) { int size = s.size(); if (current_size + size >= piece_size) { - write(); + write(false); } piece_data.emplace_back(s); current_size += size; } -void Client::ComputationResultWriter::emplace(const std::vector &v) +void ComputationResultWriter::emplace(const std::vector &v) { for (const auto &x : v) { emplace(x); } } -void Client::ComputationResultWriter::emplace(const SchemaType &s) + +void ComputationResultWriter::emplace(const std::vector> &v) +{ + for (const auto &x : v) + { + emplace(x); + } +} + +void ComputationResultWriter::emplace(const SchemaType &s) { auto json = convertSchemaToJson(s); emplace(json.dump()); } +void ComputationResultWriter::emplace(const std::vector &v) +{ + for (const auto &x : v) + { + emplace(x); + } +} + /************ TableWriter ************/ TableWriter::TableWriter(const std::string &data_id, int piece_size) : current_size(0), piece_id(0), data_id(data_id), piece_size(piece_size) @@ -239,6 +253,19 @@ void Client::writeShareDB(const std::string &data_id, const std::string &data, i ofs.close(); } +// resultDBに対してdataを書き込む +void Client::writeResultDB( + const std::string &job_uuid, const std::string &data, int data_type, int piece_id +) +{ + fs::create_directories(resultDbPath / job_uuid); + std::string data_name = (data_type == 0) ? "dim1" : (data_type == 1) ? "dim2" : "schema"; + auto data_file = data_name + "_" + std::to_string(piece_id); + auto ofs = std::ofstream(shareDbPath / job_uuid / data_file); + ofs << data; + ofs.close(); +} + // Job を DB に新規登録する void Client::registerJob(const std::string &job_uuid, const int &status) const { @@ -260,6 +287,12 @@ void Client::updateJobStatus(const std::string &job_uuid, const int &status) con ofs << tp_msec.count(); } +// Job の完了を登録する +void Client::updateJobCompleted(const std::string &job_uuid) const +{ + std::ofstream(resultDbPath / job_uuid / "completed"); +} + void Client::saveErrorInfo(const std::string &job_uuid, const pb_common_types::JobErrorInfo &info) const { diff --git a/packages/server/computation_container/client/computation_to_db/client.hpp b/packages/server/computation_container/client/computation_to_db/client.hpp index 1363d8bc1..9d2995849 100644 --- a/packages/server/computation_container/client/computation_to_db/client.hpp +++ b/packages/server/computation_container/client/computation_to_db/client.hpp @@ -25,26 +25,6 @@ class Client final static inline const auto shareDbPath = fs::path("/db/share/"); static inline const auto resultDbPath = fs::path("/db/result/"); - class ComputationResultWriter - { - int current_size; - int piece_id; - std::vector piece_data; - - const std::string job_uuid; - const std::string data_name; - const int column_number; - const int piece_size; - - public: - ComputationResultWriter(const std::string &, int, int, int); - void write(); - - void emplace(const std::string &); - void emplace(const std::vector &); - void emplace(const SchemaType &); - }; - public: Client(); static std::shared_ptr getInstance(); @@ -55,6 +35,10 @@ class Client final // shareDBに対してdataを書き込む void writeShareDB(const std::string &data_id, const std::string &data, int piece_id = 0); + // resultDBに対してdataを書き込む + void writeResultDB( + const std::string &job_uuid, const std::string &data, int data_type, int piece_id = 0 + ); // Job を DB に新規登録する void registerJob(const std::string &job_uuid, const int &status) const; @@ -62,33 +46,37 @@ class Client final // Job の実行状態を更新する void updateJobStatus(const std::string &job_uuid, const int &status) const; + // Job の完了を登録する + void updateJobCompleted(const std::string &job_uuid) const; + // Job 実行中に発生したエラーに関する情報を保存する void saveErrorInfo(const std::string &job_uuid, const pb_common_types::JobErrorInfo &info) const; +}; - // 保存時にデフォルトで呼ばれる恒等関数 - static inline auto identity = [](const auto &t) { return t; }; +// 計算結果のWriter +class ComputationResultWriter +{ + int current_size; + int piece_id; + std::vector piece_data; + + const std::string job_uuid; + const int data_type; + const int column_number; + const int piece_size; + +public: + ComputationResultWriter(const std::string &, int, int, int piece_size = 1000000); // resultの保存 - // NOTE: result_listにbegin()とend()が実装されている必要がある - template - void writeComputationResult( - const std::string &job_uuid, - const T &result_list, - int data_type, // 0:dim1, 1:dim2, 2:schema - int column_number, - const F &f = identity, // 保存時にitrごとに加工したい場合に指定する - int piece_size = 1000000 - ) const - { - auto writer = ComputationResultWriter(job_uuid, data_type, column_number, piece_size); - for (const auto &x : result_list) - { - writer.emplace(f(x)); - } - writer.write(); - std::ofstream(resultDbPath / job_uuid / "completed"); - } + void emplace(const std::string &); + void emplace(const std::vector &); + void emplace(const std::vector> &); + void emplace(const SchemaType &); + void emplace(const std::vector &); + + void write(bool fin = true); }; // TableWriterとComputationResultWriterは本質的には同じことをしているが, diff --git a/packages/server/computation_container/job/jobs/correl_job.hpp b/packages/server/computation_container/job/jobs/correl_job.hpp index ad68aef66..510096177 100644 --- a/packages/server/computation_container/job/jobs/correl_job.hpp +++ b/packages/server/computation_container/job/jobs/correl_job.hpp @@ -72,8 +72,10 @@ class CorrelJob : public JobBase auto results = toString(ret); auto column_number = (results.empty() ? -1 : results[0].size()); - auto db_client = qmpc::ComputationToDb::Client::getInstance(); - db_client->writeComputationResult(job_uuid, results, 1, column_number); + auto computationResultWriter = + qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, column_number); + computationResultWriter.emplace(results); + computationResultWriter.write(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/join_table_job.hpp b/packages/server/computation_container/job/jobs/join_table_job.hpp index d2b33dd3a..f7b5c9408 100644 --- a/packages/server/computation_container/job/jobs/join_table_job.hpp +++ b/packages/server/computation_container/job/jobs/join_table_job.hpp @@ -43,12 +43,20 @@ class JoinTableJob : public JobBase auto db_client = qmpc::ComputationToDb::Client::getInstance(); auto column_number = new_schemas.size(); - // argを束縛した関数を生成する - auto f = std::bind(removeIdColumn>, arg, std::placeholders::_1); - // tableの保存 - db_client->writeComputationResult(job_uuid, table, 1, column_number, f); - // schemaの保存 - db_client->writeComputationResult(job_uuid, new_schemas, 2, column_number); + // ID列を削除してTableを保存 + auto computationResultWriterTable = + qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, column_number); + for (const auto &row : table) + { + computationResultWriterTable.emplace(removeIdColumn(arg, row)); + } + computationResultWriterTable.write(); + + // Schemaを保存 + auto computationResultWriterSchema = + qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 2, column_number); + computationResultWriterSchema.emplace(new_schemas); + computationResultWriterSchema.write(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/math_job.hpp b/packages/server/computation_container/job/jobs/math_job.hpp index 3c4391c7d..3130689ae 100644 --- a/packages/server/computation_container/job/jobs/math_job.hpp +++ b/packages/server/computation_container/job/jobs/math_job.hpp @@ -52,8 +52,10 @@ class MathJob : public JobBase } auto results = toString(ret); - auto db_client = qmpc::ComputationToDb::Client::getInstance(); - db_client->writeComputationResult(job_uuid, results, 0, results.size()); + auto computationResultWriter = + qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 0, results.size()); + computationResultWriter.emplace(results); + computationResultWriter.write(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/mesh_code_job.hpp b/packages/server/computation_container/job/jobs/mesh_code_job.hpp index 3f72f05cc..1ec7b0275 100644 --- a/packages/server/computation_container/job/jobs/mesh_code_job.hpp +++ b/packages/server/computation_container/job/jobs/mesh_code_job.hpp @@ -105,8 +105,10 @@ class MeshCodeJob : public JobBase auto results = toString(meshcode_list); auto column_number = (results.empty() ? -1 : results[0].size()); - auto db_client = qmpc::ComputationToDb::Client::getInstance(); - db_client->writeComputationResult(job_uuid, results, 1, column_number); + auto computationResultWriter = + qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, column_number); + computationResultWriter.emplace(results); + computationResultWriter.write(); } }; } // namespace qmpc::Job From e8f4aae3dc98ae4938e6ae275d0f00d4be9ea29a Mon Sep 17 00:00:00 2001 From: Nakata Date: Thu, 20 Apr 2023 19:37:58 +0900 Subject: [PATCH 03/10] Clean computationresultWrite --- .../client/computation_to_db/client.cpp | 24 ------------------- .../client/computation_to_db/client.hpp | 11 ++++++--- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/server/computation_container/client/computation_to_db/client.cpp b/packages/server/computation_container/client/computation_to_db/client.cpp index 96bcdb11a..068f0bb23 100644 --- a/packages/server/computation_container/client/computation_to_db/client.cpp +++ b/packages/server/computation_container/client/computation_to_db/client.cpp @@ -85,36 +85,12 @@ void ComputationResultWriter::emplace(const std::string &s) piece_data.emplace_back(s); current_size += size; } -void ComputationResultWriter::emplace(const std::vector &v) -{ - for (const auto &x : v) - { - emplace(x); - } -} - -void ComputationResultWriter::emplace(const std::vector> &v) -{ - for (const auto &x : v) - { - emplace(x); - } -} - void ComputationResultWriter::emplace(const SchemaType &s) { auto json = convertSchemaToJson(s); emplace(json.dump()); } -void ComputationResultWriter::emplace(const std::vector &v) -{ - for (const auto &x : v) - { - emplace(x); - } -} - /************ TableWriter ************/ TableWriter::TableWriter(const std::string &data_id, int piece_size) : current_size(0), piece_id(0), data_id(data_id), piece_size(piece_size) diff --git a/packages/server/computation_container/client/computation_to_db/client.hpp b/packages/server/computation_container/client/computation_to_db/client.hpp index 9d2995849..deeca899c 100644 --- a/packages/server/computation_container/client/computation_to_db/client.hpp +++ b/packages/server/computation_container/client/computation_to_db/client.hpp @@ -70,11 +70,16 @@ class ComputationResultWriter ComputationResultWriter(const std::string &, int, int, int piece_size = 1000000); // resultの保存 + template + void emplace(const Iteratable &v) + { + for (const auto &x : v) + { + emplace(x); + } + } void emplace(const std::string &); - void emplace(const std::vector &); - void emplace(const std::vector> &); void emplace(const SchemaType &); - void emplace(const std::vector &); void write(bool fin = true); }; From 05d0bc0cba462625895b4d6f7269225b19c1dc90 Mon Sep 17 00:00:00 2001 From: Nakata Date: Thu, 20 Apr 2023 19:52:34 +0900 Subject: [PATCH 04/10] Move value_table --- .../client/computation_to_db/BUILD | 36 ------------------- .../server/computation_container/job/BUILD | 2 +- .../computation_container/job/job_base.hpp | 2 +- .../job/table_data/BUILD | 35 ++++++++++++++++++ .../table_data}/join_table.cpp | 0 .../table_data}/join_table.hpp | 2 +- .../table_data}/value_table.cpp | 0 .../table_data}/value_table.hpp | 0 .../test/integration_test/BUILD | 2 +- .../test/integration_test/join_table_test.hpp | 2 +- .../test/unit_test/BUILD | 2 +- 11 files changed, 41 insertions(+), 42 deletions(-) create mode 100644 packages/server/computation_container/job/table_data/BUILD rename packages/server/computation_container/{client/computation_to_db => job/table_data}/join_table.cpp (100%) rename packages/server/computation_container/{client/computation_to_db => job/table_data}/join_table.hpp (89%) rename packages/server/computation_container/{client/computation_to_db => job/table_data}/value_table.cpp (100%) rename packages/server/computation_container/{client/computation_to_db => job/table_data}/value_table.hpp (100%) diff --git a/packages/server/computation_container/client/computation_to_db/BUILD b/packages/server/computation_container/client/computation_to_db/BUILD index b84ed42da..8098dd02a 100644 --- a/packages/server/computation_container/client/computation_to_db/BUILD +++ b/packages/server/computation_container/client/computation_to_db/BUILD @@ -16,39 +16,3 @@ cc_library( ], visibility = ["//visibility:public"], ) - -cc_library( - name = "jointable", - srcs = [ - "join_table.cpp" - ], - hdrs = [ - "join_table.hpp", - ], - deps = [ - "@nlohmann_json//:json", - "@proto//manage_to_computation_container:manage_to_computation_cc_grpc", - "//share:share", - "//share:compare", - "//job:progressManager", - "//logging:log", - "//client/computation_to_db:client", - "//client/computation_to_db:valuetable", - ], - visibility = ["//visibility:public"], -) - -cc_library( - name = "valuetable", - srcs = [ - "value_table.cpp" - ], - hdrs = [ - "value_table.hpp", - ], - deps = [ - "//logging:log", - "//client/computation_to_db:client", - ], - visibility = ["//visibility:public"], -) diff --git a/packages/server/computation_container/job/BUILD b/packages/server/computation_container/job/BUILD index dfc0fb596..1bd622eb2 100644 --- a/packages/server/computation_container/job/BUILD +++ b/packages/server/computation_container/job/BUILD @@ -8,7 +8,7 @@ cc_library( "//computation:computation", "//share:share", "//client/computation_to_db:client", - "//client/computation_to_db:jointable", + "//job/table_data:jointable", "@proto//manage_to_computation_container:manage_to_computation_cc_grpc", "//logging:log" ], diff --git a/packages/server/computation_container/job/job_base.hpp b/packages/server/computation_container/job/job_base.hpp index adce7481b..9bc4fc9c6 100644 --- a/packages/server/computation_container/job/job_base.hpp +++ b/packages/server/computation_container/job/job_base.hpp @@ -12,8 +12,8 @@ #include #include -#include "client/computation_to_db/join_table.hpp" #include "external/proto/manage_to_computation_container/manage_to_computation.grpc.pb.h" +#include "job/table_data/join_table.hpp" #include "job_parameter.hpp" #include "job_status.hpp" #include "logging/logger.hpp" diff --git a/packages/server/computation_container/job/table_data/BUILD b/packages/server/computation_container/job/table_data/BUILD new file mode 100644 index 000000000..3a1c99bb9 --- /dev/null +++ b/packages/server/computation_container/job/table_data/BUILD @@ -0,0 +1,35 @@ +cc_library( + name = "jointable", + srcs = [ + "join_table.cpp" + ], + hdrs = [ + "join_table.hpp", + ], + deps = [ + "@nlohmann_json//:json", + "@proto//manage_to_computation_container:manage_to_computation_cc_grpc", + "//share:share", + "//share:compare", + "//job:progressManager", + "//logging:log", + "//client/computation_to_db:client", + "//job/table_data:valuetable", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "valuetable", + srcs = [ + "value_table.cpp" + ], + hdrs = [ + "value_table.hpp", + ], + deps = [ + "//logging:log", + "//client/computation_to_db:client", + ], + visibility = ["//visibility:public"], +) diff --git a/packages/server/computation_container/client/computation_to_db/join_table.cpp b/packages/server/computation_container/job/table_data/join_table.cpp similarity index 100% rename from packages/server/computation_container/client/computation_to_db/join_table.cpp rename to packages/server/computation_container/job/table_data/join_table.cpp diff --git a/packages/server/computation_container/client/computation_to_db/join_table.hpp b/packages/server/computation_container/job/table_data/join_table.hpp similarity index 89% rename from packages/server/computation_container/client/computation_to_db/join_table.hpp rename to packages/server/computation_container/job/table_data/join_table.hpp index d3d19b957..80ba05d5a 100644 --- a/packages/server/computation_container/client/computation_to_db/join_table.hpp +++ b/packages/server/computation_container/job/table_data/join_table.hpp @@ -1,7 +1,7 @@ #pragma once -#include "client/computation_to_db/value_table.hpp" #include "external/proto/manage_to_computation_container/manage_to_computation.grpc.pb.h" +#include "job/table_data/value_table.hpp" namespace qmpc::ComputationToDb { diff --git a/packages/server/computation_container/client/computation_to_db/value_table.cpp b/packages/server/computation_container/job/table_data/value_table.cpp similarity index 100% rename from packages/server/computation_container/client/computation_to_db/value_table.cpp rename to packages/server/computation_container/job/table_data/value_table.cpp diff --git a/packages/server/computation_container/client/computation_to_db/value_table.hpp b/packages/server/computation_container/job/table_data/value_table.hpp similarity index 100% rename from packages/server/computation_container/client/computation_to_db/value_table.hpp rename to packages/server/computation_container/job/table_data/value_table.hpp diff --git a/packages/server/computation_container/test/integration_test/BUILD b/packages/server/computation_container/test/integration_test/BUILD index 11e09384d..73d509768 100644 --- a/packages/server/computation_container/test/integration_test/BUILD +++ b/packages/server/computation_container/test/integration_test/BUILD @@ -73,7 +73,7 @@ cc_library( ], deps = [ "@gtest//:main", - "//client/computation_to_db:jointable", + "//job/table_data:jointable", ], visibility = ["//visibility:public"], ) diff --git a/packages/server/computation_container/test/integration_test/join_table_test.hpp b/packages/server/computation_container/test/integration_test/join_table_test.hpp index ed2ab923b..2c16e1e8d 100644 --- a/packages/server/computation_container/test/integration_test/join_table_test.hpp +++ b/packages/server/computation_container/test/integration_test/join_table_test.hpp @@ -3,8 +3,8 @@ #include #include "client/computation_to_db/client.hpp" -#include "client/computation_to_db/join_table.hpp" #include "gtest/gtest.h" +#include "job/table_data/join_table.hpp" namespace fs = std::experimental::filesystem; using Schema = std::vector; diff --git a/packages/server/computation_container/test/unit_test/BUILD b/packages/server/computation_container/test/unit_test/BUILD index 65375fa7f..e04e67789 100644 --- a/packages/server/computation_container/test/unit_test/BUILD +++ b/packages/server/computation_container/test/unit_test/BUILD @@ -139,7 +139,7 @@ cc_test( copts = ["-Iexternal/gtest/include"], deps = [ "@gtest//:main", - "//client/computation_to_db:valuetable", + "//job/table_data:valuetable", ], linkopts = [ "-lstdc++fs" From 7d3f4a89fb489a6cc756c2ca189e48bff218ce0a Mon Sep 17 00:00:00 2001 From: Nakata Date: Thu, 20 Apr 2023 20:11:51 +0900 Subject: [PATCH 05/10] Add error message --- .../job/table_data/join_table.cpp | 15 +++++++++------ .../job/table_data/value_table.cpp | 10 ++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/server/computation_container/job/table_data/join_table.cpp b/packages/server/computation_container/job/table_data/join_table.cpp index 3e3b9e87a..dc2d0636a 100644 --- a/packages/server/computation_container/job/table_data/join_table.cpp +++ b/packages/server/computation_container/job/table_data/join_table.cpp @@ -343,8 +343,9 @@ std::string writeVJoinTable( } if (itr1 == table1.end()) { - // TODO: 適切なエラーを返す - throw std::runtime_error(""); + qmpc::Log::throw_with_trace( + std::range_error("The specified row is out of range of the table data.") + ); } // 行を構築 @@ -374,8 +375,9 @@ std::string writeVJoinTable( } if (itr2 == table2.end()) { - // TODO: 適切なエラーを返す - throw std::runtime_error(""); + qmpc::Log::throw_with_trace( + std::range_error("The specified row is out of range of the table data.") + ); } // 行を構築 @@ -446,8 +448,9 @@ std::string writeHJoinTable( } if (itr1 == table1.end() || itr2 == table2.end()) { - // TODO: 適切なエラーを返す - throw std::runtime_error(""); + qmpc::Log::throw_with_trace( + std::range_error("The specified row is out of range of the table data.") + ); } // 行を構築 diff --git a/packages/server/computation_container/job/table_data/value_table.cpp b/packages/server/computation_container/job/table_data/value_table.cpp index 275dc0a87..895debfe6 100644 --- a/packages/server/computation_container/job/table_data/value_table.cpp +++ b/packages/server/computation_container/job/table_data/value_table.cpp @@ -34,8 +34,9 @@ RowType TableIterator::operator*() { if (!itr) { - // TODO: QMPCのerror logを使う - throw std::runtime_error(""); + qmpc::Log::throw_with_trace( + std::range_error("The specified row is out of range of the table data.") + ); } return *(itr.value()); } @@ -43,8 +44,9 @@ TableIterator &TableIterator::operator++() { if (!itr || !piece_table) { - // TODO: QMPCのerror logを使う - throw std::runtime_error(""); + qmpc::Log::throw_with_trace( + std::range_error("The specified row is out of range of the table data.") + ); } ++itr.value(); From 6abdd79e14cb4e4f0648260663c20d68b47bae86 Mon Sep 17 00:00:00 2001 From: Nakata Date: Fri, 21 Apr 2023 14:35:52 +0900 Subject: [PATCH 06/10] Fix ctodb test --- .../test/unit_test/ctodb_test.cpp | 56 ++++++++++--------- .../test/unit_test/value_table_test.cpp | 2 +- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/packages/server/computation_container/test/unit_test/ctodb_test.cpp b/packages/server/computation_container/test/unit_test/ctodb_test.cpp index 61d5b0242..ba4aeb7cd 100644 --- a/packages/server/computation_container/test/unit_test/ctodb_test.cpp +++ b/packages/server/computation_container/test/unit_test/ctodb_test.cpp @@ -242,45 +242,44 @@ TEST(ComputationToDbTest, SuccessupdateJobStatusTest) initialize(job_uuid); } -// resultの保存 -// void writeComputationResult(const std::string &job_uuid, const T &result_list, int data_type, int -// column_number, int piece_size) const -TEST(ComputationToDbTest, SuccessWriteComputationResultArrayTest) +TEST(ComputationToDbTest, SuccessComputationResultWriterArrayTest) { - const std::string job_uuid = "WriteComputaSuccessWriteComputationResultArrayTest"; + const std::string job_uuid = "SuccessComputationResultWriterArrayTest"; initialize(job_uuid); const std::vector data = {"12", "15", "21"}; fs::create_directories("/db/result/" + job_uuid); - auto cc_to_db = qmpc::ComputationToDb::Client::getInstance(); - cc_to_db->writeComputationResult(job_uuid, data, 0, 3); + auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 0, 3); + writer.emplace(data); + writer.write(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/dim1_0"); std::string read_data; getline(ifs, read_data); - const auto true_data = R"({"job_uuid":"WriteComputaSuccessWriteComputationResultArrayTest")" + const auto true_data = R"({"job_uuid":"SuccessComputationResultWriterArrayTest")" R"(,"meta":{"column_number":3,"piece_id":0})" R"(,"result":["12","15","21"]})"; EXPECT_EQ(read_data, true_data); initialize(job_uuid); } -TEST(ComputationToDbTest, SuccessWriteComputationResultArray2dimTest) +TEST(ComputationToDbTest, SuccessComputationResultWriter2dimTest) { - const std::string job_uuid = "SuccessWriteComputationResultArray2dimTest"; + const std::string job_uuid = "SuccessComputationResultWriter2dimTest"; initialize(job_uuid); const std::vector> data = {{"12", "15"}, {"21", "51"}}; fs::create_directories("/db/result/" + job_uuid); - auto cc_to_db = qmpc::ComputationToDb::Client::getInstance(); - cc_to_db->writeComputationResult(job_uuid, data, 1, 2); + auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 2); + writer.emplace(data); + writer.write(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/dim2_0"); std::string read_data; getline(ifs, read_data); - const auto true_data = R"({"job_uuid":"SuccessWriteComputationResultArray2dimTest")" + const auto true_data = R"({"job_uuid":"SuccessComputationResultWriter2dimTest")" R"(,"meta":{"column_number":2,"piece_id":0})" R"(,"result":["12","15","21","51"]})"; EXPECT_EQ(read_data, true_data); @@ -288,9 +287,9 @@ TEST(ComputationToDbTest, SuccessWriteComputationResultArray2dimTest) initialize(job_uuid); } -TEST(ComputationToDbTest, SuccessWriteComputationResultSchemaTest) +TEST(ComputationToDbTest, SuccessComputationResultWriterSchemaTest) { - const std::string job_uuid = "SuccessWriteComputationResultSchemaTest"; + const std::string job_uuid = "SuccessComputationResultWriterSchemaTest"; initialize(job_uuid); using SchemaType = qmpc::ComputationToDb::SchemaType; @@ -300,14 +299,15 @@ TEST(ComputationToDbTest, SuccessWriteComputationResultSchemaTest) SchemaType("s3", pb_common_types::ShareValueTypeEnum::SHARE_VALUE_TYPE_FIXED_POINT)}; fs::create_directories("/db/result/" + job_uuid); - auto cc_to_db = qmpc::ComputationToDb::Client::getInstance(); - cc_to_db->writeComputationResult(job_uuid, schema, 2, 3); + auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 2, 3); + writer.emplace(data); + writer.write(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/schema_0"); std::string read_data; getline(ifs, read_data); const auto true_data = - R"({"job_uuid":"SuccessWriteComputationResultSchemaTest")" + R"({"job_uuid":"SuccessComputationResultWriterSchemaTest")" R"(,"meta":{"column_number":3,"piece_id":0})" R"(,"result":["{\"name\":\"s1\",\"type\":\"SHARE_VALUE_TYPE_FIXED_POINT\"}",)" R"("{\"name\":\"s2\",\"type\":\"SHARE_VALUE_TYPE_FIXED_POINT\"}","{\"name\":\"s3\",\"type\":\"SHARE_VALUE_TYPE_FIXED_POINT\"}"]})"; @@ -316,16 +316,17 @@ TEST(ComputationToDbTest, SuccessWriteComputationResultSchemaTest) initialize(job_uuid); } -TEST(ComputationToDbTest, SuccessWriteComputationResultArrayPieceTest) +TEST(ComputationToDbTest, SuccessComputationResultWriterArrayPieceTest) { - const std::string job_uuid = "SuccessWriteComputationResultArrayPieceTest"; + const std::string job_uuid = "SuccessComputationResultWriterArrayPieceTest"; initialize(job_uuid); const std::vector> data = {{"12", "15", "21"}}; fs::create_directories("/db/result/" + job_uuid); - auto cc_to_db = qmpc::ComputationToDb::Client::getInstance(); - cc_to_db->writeComputationResult(job_uuid, data, 1, 3, cc_to_db->identity, 4); + auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 3, 4); + writer.emplace(data); + writer.write(); std::vector true_result = {R"(["12"])", R"(["15"])", R"(["21"])"}; for (size_t piece_id = 0; piece_id < true_result.size(); ++piece_id) @@ -333,7 +334,7 @@ TEST(ComputationToDbTest, SuccessWriteComputationResultArrayPieceTest) auto ifs = std::ifstream("/db/result/" + job_uuid + "/dim2_" + std::to_string(piece_id)); std::string read_data; getline(ifs, read_data); - const auto true_data = R"({"job_uuid":"SuccessWriteComputationResultArrayPieceTest")" + const auto true_data = R"({"job_uuid":"SuccessComputationResultWriterArrayPieceTest")" R"(,"meta":{"column_number":3,"piece_id":)" + std::to_string(piece_id) + R"(},"result":)" + true_result[piece_id] + R"(})"; @@ -343,16 +344,17 @@ TEST(ComputationToDbTest, SuccessWriteComputationResultArrayPieceTest) initialize(job_uuid); } -TEST(ComputationToDbTest, SuccessWriteComputationResultCompletedTest) +TEST(ComputationToDbTest, SuccessComputationResultWriterCompletedTest) { - const std::string job_uuid = "WriteComputationResultTestId"; + const std::string job_uuid = "SuccessComputationResultWriterCompletedTest"; initialize(job_uuid); const std::vector> data = {{"12", "15", "21"}}; fs::create_directories("/db/result/" + job_uuid); - auto cc_to_db = qmpc::ComputationToDb::Client::getInstance(); - cc_to_db->writeComputationResult(job_uuid, data, 1, 3); + auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 3); + writer.emplace(data); + writer.write(); auto exist = fs::exists("/db/result/" + job_uuid + "/completed"); EXPECT_TRUE(exist); diff --git a/packages/server/computation_container/test/unit_test/value_table_test.cpp b/packages/server/computation_container/test/unit_test/value_table_test.cpp index 63789c4df..a6ceb8b8b 100644 --- a/packages/server/computation_container/test/unit_test/value_table_test.cpp +++ b/packages/server/computation_container/test/unit_test/value_table_test.cpp @@ -1,8 +1,8 @@ #include #include -#include "client/computation_to_db/value_table.hpp" #include "gtest/gtest.h" +#include "job/table_data/value_table.hpp" namespace fs = std::experimental::filesystem; From e538c2c08518e290d2a501e18e8d11b2b70e3b64 Mon Sep 17 00:00:00 2001 From: Nakata Date: Fri, 21 Apr 2023 16:13:39 +0900 Subject: [PATCH 07/10] Fix writeResultDB --- .../computation_container/client/computation_to_db/client.cpp | 2 +- .../server/computation_container/test/unit_test/ctodb_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/computation_container/client/computation_to_db/client.cpp b/packages/server/computation_container/client/computation_to_db/client.cpp index 068f0bb23..f9d2468f2 100644 --- a/packages/server/computation_container/client/computation_to_db/client.cpp +++ b/packages/server/computation_container/client/computation_to_db/client.cpp @@ -237,7 +237,7 @@ void Client::writeResultDB( fs::create_directories(resultDbPath / job_uuid); std::string data_name = (data_type == 0) ? "dim1" : (data_type == 1) ? "dim2" : "schema"; auto data_file = data_name + "_" + std::to_string(piece_id); - auto ofs = std::ofstream(shareDbPath / job_uuid / data_file); + auto ofs = std::ofstream(resultDbPath / job_uuid / data_file); ofs << data; ofs.close(); } diff --git a/packages/server/computation_container/test/unit_test/ctodb_test.cpp b/packages/server/computation_container/test/unit_test/ctodb_test.cpp index ba4aeb7cd..a5d33bb2b 100644 --- a/packages/server/computation_container/test/unit_test/ctodb_test.cpp +++ b/packages/server/computation_container/test/unit_test/ctodb_test.cpp @@ -300,7 +300,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriterSchemaTest) fs::create_directories("/db/result/" + job_uuid); auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 2, 3); - writer.emplace(data); + writer.emplace(schema); writer.write(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/schema_0"); From b969f890b8721c380b425c917a01db6577c9faf1 Mon Sep 17 00:00:00 2001 From: Nakata Date: Mon, 24 Apr 2023 15:25:25 +0900 Subject: [PATCH 08/10] Move value_table --- packages/server/computation_container/job/BUILD | 2 +- packages/server/computation_container/job/job_base.hpp | 2 +- .../server/computation_container/test/integration_test/BUILD | 2 +- .../test/integration_test/join_table_test.hpp | 2 +- packages/server/computation_container/test/unit_test/BUILD | 2 +- .../computation_container/test/unit_test/value_table_test.cpp | 2 +- .../computation_container/{job/table_data => value_table}/BUILD | 2 +- .../{job/table_data => value_table}/join_table.cpp | 0 .../{job/table_data => value_table}/join_table.hpp | 2 +- .../{job/table_data => value_table}/value_table.cpp | 0 .../{job/table_data => value_table}/value_table.hpp | 0 11 files changed, 8 insertions(+), 8 deletions(-) rename packages/server/computation_container/{job/table_data => value_table}/BUILD (94%) rename packages/server/computation_container/{job/table_data => value_table}/join_table.cpp (100%) rename packages/server/computation_container/{job/table_data => value_table}/join_table.hpp (91%) rename packages/server/computation_container/{job/table_data => value_table}/value_table.cpp (100%) rename packages/server/computation_container/{job/table_data => value_table}/value_table.hpp (100%) diff --git a/packages/server/computation_container/job/BUILD b/packages/server/computation_container/job/BUILD index 1bd622eb2..7fe62d2ba 100644 --- a/packages/server/computation_container/job/BUILD +++ b/packages/server/computation_container/job/BUILD @@ -8,7 +8,7 @@ cc_library( "//computation:computation", "//share:share", "//client/computation_to_db:client", - "//job/table_data:jointable", + "//value_table:jointable", "@proto//manage_to_computation_container:manage_to_computation_cc_grpc", "//logging:log" ], diff --git a/packages/server/computation_container/job/job_base.hpp b/packages/server/computation_container/job/job_base.hpp index 9bc4fc9c6..e1e608eff 100644 --- a/packages/server/computation_container/job/job_base.hpp +++ b/packages/server/computation_container/job/job_base.hpp @@ -13,11 +13,11 @@ #include #include "external/proto/manage_to_computation_container/manage_to_computation.grpc.pb.h" -#include "job/table_data/join_table.hpp" #include "job_parameter.hpp" #include "job_status.hpp" #include "logging/logger.hpp" #include "share/share.hpp" +#include "value_table/join_table.hpp" namespace qmpc::Job { diff --git a/packages/server/computation_container/test/integration_test/BUILD b/packages/server/computation_container/test/integration_test/BUILD index 73d509768..ec4a9eb22 100644 --- a/packages/server/computation_container/test/integration_test/BUILD +++ b/packages/server/computation_container/test/integration_test/BUILD @@ -73,7 +73,7 @@ cc_library( ], deps = [ "@gtest//:main", - "//job/table_data:jointable", + "//value_table:jointable", ], visibility = ["//visibility:public"], ) diff --git a/packages/server/computation_container/test/integration_test/join_table_test.hpp b/packages/server/computation_container/test/integration_test/join_table_test.hpp index 2c16e1e8d..60652a9e8 100644 --- a/packages/server/computation_container/test/integration_test/join_table_test.hpp +++ b/packages/server/computation_container/test/integration_test/join_table_test.hpp @@ -4,7 +4,7 @@ #include "client/computation_to_db/client.hpp" #include "gtest/gtest.h" -#include "job/table_data/join_table.hpp" +#include "value_table/join_table.hpp" namespace fs = std::experimental::filesystem; using Schema = std::vector; diff --git a/packages/server/computation_container/test/unit_test/BUILD b/packages/server/computation_container/test/unit_test/BUILD index e04e67789..4da152fca 100644 --- a/packages/server/computation_container/test/unit_test/BUILD +++ b/packages/server/computation_container/test/unit_test/BUILD @@ -139,7 +139,7 @@ cc_test( copts = ["-Iexternal/gtest/include"], deps = [ "@gtest//:main", - "//job/table_data:valuetable", + "//value_table:valuetable", ], linkopts = [ "-lstdc++fs" diff --git a/packages/server/computation_container/test/unit_test/value_table_test.cpp b/packages/server/computation_container/test/unit_test/value_table_test.cpp index a6ceb8b8b..53a9393b3 100644 --- a/packages/server/computation_container/test/unit_test/value_table_test.cpp +++ b/packages/server/computation_container/test/unit_test/value_table_test.cpp @@ -2,7 +2,7 @@ #include #include "gtest/gtest.h" -#include "job/table_data/value_table.hpp" +#include "value_table/value_table.hpp" namespace fs = std::experimental::filesystem; diff --git a/packages/server/computation_container/job/table_data/BUILD b/packages/server/computation_container/value_table/BUILD similarity index 94% rename from packages/server/computation_container/job/table_data/BUILD rename to packages/server/computation_container/value_table/BUILD index 3a1c99bb9..57dfb36b5 100644 --- a/packages/server/computation_container/job/table_data/BUILD +++ b/packages/server/computation_container/value_table/BUILD @@ -14,7 +14,7 @@ cc_library( "//job:progressManager", "//logging:log", "//client/computation_to_db:client", - "//job/table_data:valuetable", + "//value_table:valuetable", ], visibility = ["//visibility:public"], ) diff --git a/packages/server/computation_container/job/table_data/join_table.cpp b/packages/server/computation_container/value_table/join_table.cpp similarity index 100% rename from packages/server/computation_container/job/table_data/join_table.cpp rename to packages/server/computation_container/value_table/join_table.cpp diff --git a/packages/server/computation_container/job/table_data/join_table.hpp b/packages/server/computation_container/value_table/join_table.hpp similarity index 91% rename from packages/server/computation_container/job/table_data/join_table.hpp rename to packages/server/computation_container/value_table/join_table.hpp index 80ba05d5a..10c91c745 100644 --- a/packages/server/computation_container/job/table_data/join_table.hpp +++ b/packages/server/computation_container/value_table/join_table.hpp @@ -1,7 +1,7 @@ #pragma once #include "external/proto/manage_to_computation_container/manage_to_computation.grpc.pb.h" -#include "job/table_data/value_table.hpp" +#include "value_table/value_table.hpp" namespace qmpc::ComputationToDb { diff --git a/packages/server/computation_container/job/table_data/value_table.cpp b/packages/server/computation_container/value_table/value_table.cpp similarity index 100% rename from packages/server/computation_container/job/table_data/value_table.cpp rename to packages/server/computation_container/value_table/value_table.cpp diff --git a/packages/server/computation_container/job/table_data/value_table.hpp b/packages/server/computation_container/value_table/value_table.hpp similarity index 100% rename from packages/server/computation_container/job/table_data/value_table.hpp rename to packages/server/computation_container/value_table/value_table.hpp From 8ebeb2dcafe0fa6f94dae4f0eb7d8d82efd4648d Mon Sep 17 00:00:00 2001 From: Nakata Date: Mon, 24 Apr 2023 15:44:11 +0900 Subject: [PATCH 09/10] Update write to split completed --- .../client/computation_to_db/client.cpp | 14 +++++++------- .../client/computation_to_db/client.hpp | 5 +++-- .../computation_container/job/jobs/correl_job.hpp | 2 +- .../job/jobs/join_table_job.hpp | 4 ++-- .../computation_container/job/jobs/math_job.hpp | 2 +- .../job/jobs/mesh_code_job.hpp | 2 +- .../test/unit_test/ctodb_test.cpp | 14 +++++++------- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/server/computation_container/client/computation_to_db/client.cpp b/packages/server/computation_container/client/computation_to_db/client.cpp index f9d2468f2..fc8f1ebd2 100644 --- a/packages/server/computation_container/client/computation_to_db/client.cpp +++ b/packages/server/computation_container/client/computation_to_db/client.cpp @@ -56,7 +56,7 @@ ComputationResultWriter::ComputationResultWriter( { } -void ComputationResultWriter::write(bool fin) +void ComputationResultWriter::write() { nlohmann::json piece_data_json = { {"job_uuid", job_uuid}, @@ -68,11 +68,6 @@ void ComputationResultWriter::write(bool fin) ++piece_id; current_size = 0; piece_data.clear(); - - if (fin) - { - Client::getInstance()->updateJobCompleted(job_uuid); - } } void ComputationResultWriter::emplace(const std::string &s) @@ -80,7 +75,7 @@ void ComputationResultWriter::emplace(const std::string &s) int size = s.size(); if (current_size + size >= piece_size) { - write(false); + write(); } piece_data.emplace_back(s); current_size += size; @@ -90,6 +85,11 @@ void ComputationResultWriter::emplace(const SchemaType &s) auto json = convertSchemaToJson(s); emplace(json.dump()); } +void ComputationResultWriter::completed() +{ + write(); + Client::getInstance()->updateJobCompleted(job_uuid); +} /************ TableWriter ************/ TableWriter::TableWriter(const std::string &data_id, int piece_size) diff --git a/packages/server/computation_container/client/computation_to_db/client.hpp b/packages/server/computation_container/client/computation_to_db/client.hpp index deeca899c..7583d8a45 100644 --- a/packages/server/computation_container/client/computation_to_db/client.hpp +++ b/packages/server/computation_container/client/computation_to_db/client.hpp @@ -66,6 +66,8 @@ class ComputationResultWriter const int column_number; const int piece_size; + void write(); + public: ComputationResultWriter(const std::string &, int, int, int piece_size = 1000000); @@ -80,8 +82,7 @@ class ComputationResultWriter } void emplace(const std::string &); void emplace(const SchemaType &); - - void write(bool fin = true); + void completed(); }; // TableWriterとComputationResultWriterは本質的には同じことをしているが, diff --git a/packages/server/computation_container/job/jobs/correl_job.hpp b/packages/server/computation_container/job/jobs/correl_job.hpp index 510096177..4196f162c 100644 --- a/packages/server/computation_container/job/jobs/correl_job.hpp +++ b/packages/server/computation_container/job/jobs/correl_job.hpp @@ -75,7 +75,7 @@ class CorrelJob : public JobBase auto computationResultWriter = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, column_number); computationResultWriter.emplace(results); - computationResultWriter.write(); + computationResultWriter.completed(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/join_table_job.hpp b/packages/server/computation_container/job/jobs/join_table_job.hpp index f7b5c9408..5d4e82776 100644 --- a/packages/server/computation_container/job/jobs/join_table_job.hpp +++ b/packages/server/computation_container/job/jobs/join_table_job.hpp @@ -50,13 +50,13 @@ class JoinTableJob : public JobBase { computationResultWriterTable.emplace(removeIdColumn(arg, row)); } - computationResultWriterTable.write(); + computationResultWriterTable.completed(); // Schemaを保存 auto computationResultWriterSchema = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 2, column_number); computationResultWriterSchema.emplace(new_schemas); - computationResultWriterSchema.write(); + computationResultWriterSchema.completed(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/math_job.hpp b/packages/server/computation_container/job/jobs/math_job.hpp index 3130689ae..674bdd092 100644 --- a/packages/server/computation_container/job/jobs/math_job.hpp +++ b/packages/server/computation_container/job/jobs/math_job.hpp @@ -55,7 +55,7 @@ class MathJob : public JobBase auto computationResultWriter = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 0, results.size()); computationResultWriter.emplace(results); - computationResultWriter.write(); + computationResultWriter.completed(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/job/jobs/mesh_code_job.hpp b/packages/server/computation_container/job/jobs/mesh_code_job.hpp index 1ec7b0275..a8858efee 100644 --- a/packages/server/computation_container/job/jobs/mesh_code_job.hpp +++ b/packages/server/computation_container/job/jobs/mesh_code_job.hpp @@ -108,7 +108,7 @@ class MeshCodeJob : public JobBase auto computationResultWriter = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, column_number); computationResultWriter.emplace(results); - computationResultWriter.write(); + computationResultWriter.completed(); } }; } // namespace qmpc::Job diff --git a/packages/server/computation_container/test/unit_test/ctodb_test.cpp b/packages/server/computation_container/test/unit_test/ctodb_test.cpp index a5d33bb2b..09f1e407d 100644 --- a/packages/server/computation_container/test/unit_test/ctodb_test.cpp +++ b/packages/server/computation_container/test/unit_test/ctodb_test.cpp @@ -252,7 +252,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriterArrayTest) auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 0, 3); writer.emplace(data); - writer.write(); + writer.completed(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/dim1_0"); std::string read_data; @@ -274,7 +274,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriter2dimTest) auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 2); writer.emplace(data); - writer.write(); + writer.completed(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/dim2_0"); std::string read_data; @@ -301,7 +301,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriterSchemaTest) auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 2, 3); writer.emplace(schema); - writer.write(); + writer.completed(); auto ifs = std::ifstream("/db/result/" + job_uuid + "/schema_0"); std::string read_data; @@ -326,7 +326,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriterArrayPieceTest) auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 3, 4); writer.emplace(data); - writer.write(); + writer.completed(); std::vector true_result = {R"(["12"])", R"(["15"])", R"(["21"])"}; for (size_t piece_id = 0; piece_id < true_result.size(); ++piece_id) @@ -354,7 +354,7 @@ TEST(ComputationToDbTest, SuccessComputationResultWriterCompletedTest) auto writer = qmpc::ComputationToDb::ComputationResultWriter(job_uuid, 1, 3); writer.emplace(data); - writer.write(); + writer.completed(); auto exist = fs::exists("/db/result/" + job_uuid + "/completed"); EXPECT_TRUE(exist); @@ -380,7 +380,7 @@ TEST(ComputationToDbTest, SuccessTableWriteTest) { writer.emplace(row); } - writer.write(); + writer.completed(); auto ifs = std::ifstream("/db/share/" + data_id + "/0"); std::string data; @@ -410,7 +410,7 @@ TEST(ComputationToDbTest, SuccessTableWritePieceTest) { writer.emplace(row); } - writer.write(); + writer.completed(); std::vector true_data = { R"({"meta":{"piece_id":0,"schema":[{"name":"attr1","type":"SHARE_VALUE_TYPE_FIXED_POINT"},)" From 61133babbe63120b42ba4dc9c403a85d79f44b9f Mon Sep 17 00:00:00 2001 From: Nakata Date: Mon, 24 Apr 2023 16:12:37 +0900 Subject: [PATCH 10/10] Fix ctodb test --- .../computation_container/test/unit_test/ctodb_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/computation_container/test/unit_test/ctodb_test.cpp b/packages/server/computation_container/test/unit_test/ctodb_test.cpp index 09f1e407d..bd4b84922 100644 --- a/packages/server/computation_container/test/unit_test/ctodb_test.cpp +++ b/packages/server/computation_container/test/unit_test/ctodb_test.cpp @@ -380,7 +380,7 @@ TEST(ComputationToDbTest, SuccessTableWriteTest) { writer.emplace(row); } - writer.completed(); + writer.write(); auto ifs = std::ifstream("/db/share/" + data_id + "/0"); std::string data; @@ -410,7 +410,7 @@ TEST(ComputationToDbTest, SuccessTableWritePieceTest) { writer.emplace(row); } - writer.completed(); + writer.write(); std::vector true_data = { R"({"meta":{"piece_id":0,"schema":[{"name":"attr1","type":"SHARE_VALUE_TYPE_FIXED_POINT"},)"