From aba4790b184bca691e12f001300f440a023b3140 Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Wed, 25 May 2022 17:24:29 +0200 Subject: [PATCH] math_opt: sync from google3 --- ortools/math_opt/constraints/sos/BUILD.bazel | 28 +++ ortools/math_opt/constraints/sos/validator.cc | 45 ++++ ortools/math_opt/constraints/sos/validator.h | 28 +++ ortools/math_opt/core/BUILD.bazel | 43 ---- ortools/math_opt/core/model_summary.cc | 44 +++- ortools/math_opt/core/model_summary.h | 44 +++- ortools/math_opt/cpp/BUILD.bazel | 32 +-- ortools/math_opt/cpp/callback.cc | 2 +- ortools/math_opt/cpp/callback.h | 2 +- ortools/math_opt/cpp/id_map.h | 2 +- ortools/math_opt/cpp/id_set.h | 2 +- ortools/math_opt/cpp/key_types.h | 2 +- ortools/math_opt/cpp/linear_constraint.h | 2 +- ortools/math_opt/cpp/map_filter.h | 2 +- ortools/math_opt/cpp/model.cc | 10 +- ortools/math_opt/cpp/model.h | 2 +- .../math_opt/cpp/model_solve_parameters.cc | 2 +- ortools/math_opt/cpp/model_solve_parameters.h | 2 +- ortools/math_opt/cpp/solution.cc | 2 +- ortools/math_opt/cpp/solution.h | 2 +- ortools/math_opt/cpp/solve.cc | 2 +- ortools/math_opt/cpp/solve.h | 2 +- ortools/math_opt/cpp/solve_result.cc | 2 +- ortools/math_opt/cpp/solve_result.h | 2 +- ortools/math_opt/cpp/sparse_containers.h | 2 +- ortools/math_opt/cpp/statistics.cc | 2 +- ortools/math_opt/cpp/update_tracker.cc | 2 +- ortools/math_opt/cpp/update_tracker.h | 2 +- .../math_opt/cpp/variable_and_expressions.h | 2 +- ortools/math_opt/model.proto | 68 ++++-- ortools/math_opt/model_update.proto | 24 ++ ortools/math_opt/solvers/gurobi_solver.cc | 2 +- ortools/math_opt/storage/BUILD.bazel | 84 +++++++ .../{core => storage}/model_storage.cc | 110 ++------- .../{core => storage}/model_storage.h | 99 +++----- .../math_opt/storage/model_storage_types.h | 29 +++ .../{core => storage}/model_update_merge.cc | 221 +++--------------- ortools/math_opt/storage/model_update_merge.h | 40 ++++ .../math_opt/storage/proto_merging_utils.cc | 182 +++++++++++++++ .../proto_merging_utils.h} | 38 +-- ortools/math_opt/storage/sparse_matrix.cc | 145 ++++++++++++ ortools/math_opt/storage/sparse_matrix.h | 217 +++++++++++++++++ ortools/math_opt/validators/BUILD.bazel | 15 ++ .../validators/linear_expression_validator.cc | 45 ++++ .../validators/linear_expression_validator.h | 28 +++ .../math_opt/validators/model_validator.cc | 39 +++- ortools/math_opt/validators/model_validator.h | 1 + 47 files changed, 1203 insertions(+), 500 deletions(-) create mode 100644 ortools/math_opt/constraints/sos/BUILD.bazel create mode 100644 ortools/math_opt/constraints/sos/validator.cc create mode 100644 ortools/math_opt/constraints/sos/validator.h create mode 100644 ortools/math_opt/storage/BUILD.bazel rename ortools/math_opt/{core => storage}/model_storage.cc (87%) rename ortools/math_opt/{core => storage}/model_storage.h (93%) create mode 100644 ortools/math_opt/storage/model_storage_types.h rename ortools/math_opt/{core => storage}/model_update_merge.cc (50%) create mode 100644 ortools/math_opt/storage/model_update_merge.h create mode 100644 ortools/math_opt/storage/proto_merging_utils.cc rename ortools/math_opt/{core/model_update_merge.h => storage/proto_merging_utils.h} (85%) create mode 100644 ortools/math_opt/storage/sparse_matrix.cc create mode 100644 ortools/math_opt/storage/sparse_matrix.h create mode 100644 ortools/math_opt/validators/linear_expression_validator.cc create mode 100644 ortools/math_opt/validators/linear_expression_validator.h diff --git a/ortools/math_opt/constraints/sos/BUILD.bazel b/ortools/math_opt/constraints/sos/BUILD.bazel new file mode 100644 index 0000000000..0ccfe7e7be --- /dev/null +++ b/ortools/math_opt/constraints/sos/BUILD.bazel @@ -0,0 +1,28 @@ +package(default_visibility = ["//ortools/math_opt:__subpackages__"]) + +exports_files( + [ + "BUILD.bazel", + ] + glob([ + "*.cc", + "*.h", + "*.proto", + ]), + visibility = [ + "//ortools/open_source:__subpackages__", + ], +) + +cc_library( + name = "validator", + srcs = ["validator.cc"], + hdrs = ["validator.h"], + deps = [ + "@com_google_absl//absl/status", + "//ortools/math_opt:model_cc_proto", + "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/core:model_summary", + "//ortools/math_opt/validators:linear_expression_validator", + "//ortools/math_opt/validators:scalar_validator", + ], +) diff --git a/ortools/math_opt/constraints/sos/validator.cc b/ortools/math_opt/constraints/sos/validator.cc new file mode 100644 index 0000000000..bbbc464cc4 --- /dev/null +++ b/ortools/math_opt/constraints/sos/validator.cc @@ -0,0 +1,45 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ortools/math_opt/constraints/sos/validator.h" + +#include "absl/status/status.h" +#include "ortools/math_opt/core/model_summary.h" +#include "ortools/math_opt/model.pb.h" +#include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/validators/linear_expression_validator.h" +#include "ortools/math_opt/validators/scalar_validator.h" +#include "ortools/base/status_macros.h" + +namespace operations_research::math_opt { + +absl::Status ValidateConstraint(const SosConstraintProto& constraint, + const IdNameBiMap& variable_universe) { + if (!constraint.weights().empty() && + constraint.weights_size() != constraint.expressions_size()) { + return util::InvalidArgumentErrorBuilder() + << "Length mismatch between weights and expressions: " + << constraint.weights_size() << " vs. " + << constraint.expressions_size(); + } + for (const LinearExpressionProto& expression : constraint.expressions()) { + RETURN_IF_ERROR(ValidateLinearExpression(expression, variable_universe)) + << "Invalid SOS expression"; + } + for (const double weight : constraint.weights()) { + RETURN_IF_ERROR(CheckScalarNoNanNoInf(weight)) << "Invalid SOS weight"; + } + return absl::OkStatus(); +} + +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/constraints/sos/validator.h b/ortools/math_opt/constraints/sos/validator.h new file mode 100644 index 0000000000..b85e2e3e64 --- /dev/null +++ b/ortools/math_opt/constraints/sos/validator.h @@ -0,0 +1,28 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_MATH_OPT_CONSTRAINTS_SOS_VALIDATOR_H_ +#define OR_TOOLS_MATH_OPT_CONSTRAINTS_SOS_VALIDATOR_H_ + +#include "absl/status/status.h" +#include "ortools/math_opt/core/model_summary.h" +#include "ortools/math_opt/model.pb.h" + +namespace operations_research::math_opt { + +absl::Status ValidateConstraint(const SosConstraintProto& constraint, + const IdNameBiMap& variable_universe); + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_CONSTRAINTS_SOS_VALIDATOR_H_ diff --git a/ortools/math_opt/core/BUILD.bazel b/ortools/math_opt/core/BUILD.bazel index 566a1b158a..c0189c2c38 100644 --- a/ortools/math_opt/core/BUILD.bazel +++ b/ortools/math_opt/core/BUILD.bazel @@ -49,34 +49,6 @@ cc_library( ], ) -cc_library( - name = "model_storage", - srcs = ["model_storage.cc"], - hdrs = ["model_storage.h"], - deps = [ - ":model_update_merge", - ":sparse_vector_view", - "//ortools/base", - "//ortools/base:intops", - "//ortools/base:map_util", - "//ortools/math_opt:model_cc_proto", - "//ortools/math_opt:model_update_cc_proto", - "//ortools/math_opt:result_cc_proto", - "//ortools/math_opt:solution_cc_proto", - "//ortools/math_opt:sparse_containers_cc_proto", - "//ortools/math_opt/validators:model_validator", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/meta:type_traits", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/types:span", - ], -) - cc_library( name = "solver_interface", srcs = ["solver_interface.cc"], @@ -133,21 +105,6 @@ cc_library( ], ) -cc_library( - name = "model_update_merge", - srcs = ["model_update_merge.cc"], - hdrs = ["model_update_merge.h"], - deps = [ - ":sparse_vector_view", - "//ortools/base", - "//ortools/base:protobuf_util", - "//ortools/math_opt:model_cc_proto", - "//ortools/math_opt:model_update_cc_proto", - "//ortools/math_opt:sparse_containers_cc_proto", - "@com_google_absl//absl/container:flat_hash_set", - ], -) - cc_library( name = "solve_interrupter", srcs = ["solve_interrupter.cc"], diff --git a/ortools/math_opt/core/model_summary.cc b/ortools/math_opt/core/model_summary.cc index 5e254614b2..79fd8e558f 100644 --- a/ortools/math_opt/core/model_summary.cc +++ b/ortools/math_opt/core/model_summary.cc @@ -15,18 +15,27 @@ #include #include +#include +#include #include #include +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "ortools/base/linked_hash_map.h" +#include "ortools/base/logging.h" +#include "ortools/base/status_macros.h" +#include "ortools/math_opt/model.pb.h" +#include "ortools/math_opt/model_update.pb.h" namespace operations_research { namespace math_opt { -namespace { -// TODO(b/232526223): this is an exact copy of -// CheckIdsRangeAndStrictlyIncreasing from ids_validator.h, find a way to share -// the code. +namespace internal { + absl::Status CheckIdsRangeAndStrictlyIncreasing2( absl::Span ids) { int64_t previous{-1}; @@ -47,7 +56,7 @@ absl::Status CheckIdsRangeAndStrictlyIncreasing2( return absl::OkStatus(); } -} // namespace +} // namespace internal IdNameBiMap::IdNameBiMap( std::initializer_list> ids) @@ -85,9 +94,9 @@ IdNameBiMap& IdNameBiMap::operator=(const IdNameBiMap& other) { absl::Status IdNameBiMap::BulkUpdate( absl::Span deleted_ids, absl::Span new_ids, const absl::Span names) { - RETURN_IF_ERROR(CheckIdsRangeAndStrictlyIncreasing2(deleted_ids)) + RETURN_IF_ERROR(internal::CheckIdsRangeAndStrictlyIncreasing2(deleted_ids)) << "invalid deleted ids"; - RETURN_IF_ERROR(CheckIdsRangeAndStrictlyIncreasing2(new_ids)) + RETURN_IF_ERROR(internal::CheckIdsRangeAndStrictlyIncreasing2(new_ids)) << "invalid new ids"; if (!names.empty() && names.size() != new_ids.size()) { return util::InvalidArgumentErrorBuilder() @@ -107,7 +116,10 @@ absl::Status IdNameBiMap::BulkUpdate( } ModelSummary::ModelSummary(const bool check_names) - : variables(check_names), linear_constraints(check_names) {} + : variables(check_names), + linear_constraints(check_names), + sos1_constraints(check_names), + sos2_constraints(check_names) {} absl::StatusOr ModelSummary::Create(const ModelProto& model, const bool check_names) { @@ -118,6 +130,12 @@ absl::StatusOr ModelSummary::Create(const ModelProto& model, RETURN_IF_ERROR(summary.linear_constraints.BulkUpdate( {}, model.linear_constraints().ids(), model.linear_constraints().names())) << "Model.linear_constraints are invalid"; + RETURN_IF_ERROR(internal::UpdateBiMapFromMappedConstraints( + {}, model.sos1_constraints(), summary.sos1_constraints)) + << "Model.sos1_constraints are invalid"; + RETURN_IF_ERROR(internal::UpdateBiMapFromMappedConstraints( + {}, model.sos2_constraints(), summary.sos2_constraints)) + << "Model.sos2_constraints are invalid"; return summary; } @@ -131,6 +149,16 @@ absl::Status ModelSummary::Update(const ModelUpdateProto& model_update) { model_update.new_linear_constraints().ids(), model_update.new_linear_constraints().names())) << "invalid linear constraints"; + RETURN_IF_ERROR(internal::UpdateBiMapFromMappedConstraints( + model_update.sos1_constraint_updates().deleted_constraint_ids(), + model_update.sos1_constraint_updates().new_constraints(), + sos1_constraints)) + << "invalid sos1 constraints"; + RETURN_IF_ERROR(internal::UpdateBiMapFromMappedConstraints( + model_update.sos2_constraint_updates().deleted_constraint_ids(), + model_update.sos2_constraint_updates().new_constraints(), + sos2_constraints)) + << "invalid sos2 constraints"; return absl::OkStatus(); } diff --git a/ortools/math_opt/core/model_summary.h b/ortools/math_opt/core/model_summary.h index 3d9b95a776..c6dd97bf0a 100644 --- a/ortools/math_opt/core/model_summary.h +++ b/ortools/math_opt/core/model_summary.h @@ -16,17 +16,21 @@ #include #include +#include #include +#include #include #include +#include +#include "absl/algorithm/container.h" #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "ortools/base/linked_hash_map.h" #include "ortools/base/logging.h" -#include "ortools/base/map_util.h" -#include "ortools/base/status_builder.h" #include "ortools/base/status_macros.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h" @@ -131,6 +135,8 @@ struct ModelSummary { IdNameBiMap variables; IdNameBiMap linear_constraints; + IdNameBiMap sos1_constraints; + IdNameBiMap sos2_constraints; }; //////////////////////////////////////////////////////////////////////////////// @@ -219,6 +225,40 @@ absl::Status IdNameBiMap::SetNextFreeId(const int64_t new_next_free_id) { return absl::OkStatus(); } +namespace internal { + +// TODO(b/232526223): this is an exact copy of +// CheckIdsRangeAndStrictlyIncreasing from ids_validator.h, find a way to share +// the code. +// NOTE: This function is only exposed in the header because we need +// UpdateBiMapFromMappedConstraints here for testing purposes. +absl::Status CheckIdsRangeAndStrictlyIncreasing2(absl::Span ids); + +// NOTE: This is only exposed in the header for testing purposes. +template +absl::Status UpdateBiMapFromMappedConstraints( + const absl::Span deleted_ids, + const google::protobuf::Map& proto_map, + IdNameBiMap& bimap) { + RETURN_IF_ERROR(CheckIdsRangeAndStrictlyIncreasing2(deleted_ids)) + << "invalid deleted ids"; + for (const int64_t id : deleted_ids) { + RETURN_IF_ERROR(bimap.Erase(id)); + } + std::vector new_ids; + new_ids.reserve(proto_map.size()); + for (const auto& [id, value] : proto_map) { + new_ids.push_back(id); + } + absl::c_sort(new_ids); + for (const int64_t id : new_ids) { + RETURN_IF_ERROR(bimap.Insert(id, proto_map.at(id).name())); + } + return absl::OkStatus(); +} + +} // namespace internal + } // namespace math_opt } // namespace operations_research diff --git a/ortools/math_opt/cpp/BUILD.bazel b/ortools/math_opt/cpp/BUILD.bazel index cef3cda279..0c21f2b57e 100644 --- a/ortools/math_opt/cpp/BUILD.bazel +++ b/ortools/math_opt/cpp/BUILD.bazel @@ -36,8 +36,8 @@ cc_library( "//ortools/base:status_macros", "//ortools/math_opt:solution_cc_proto", "//ortools/math_opt:sparse_containers_cc_proto", - "//ortools/math_opt/core:model_storage", "//ortools/math_opt/core:sparse_vector_view", + "//ortools/math_opt/storage:model_storage", "//ortools/math_opt/validators:ids_validator", "//ortools/math_opt/validators:sparse_vector_validator", "//ortools/util:status_macros", @@ -62,7 +62,7 @@ cc_library( "//ortools/base:status_macros", "//ortools/math_opt:model_cc_proto", "//ortools/math_opt:model_update_cc_proto", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", @@ -79,7 +79,7 @@ cc_library( "//ortools/base", "//ortools/base:intops", "//ortools/math_opt/core:arrow_operator_proxy", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/types:span", @@ -96,7 +96,7 @@ cc_library( "//ortools/base", "//ortools/base:intops", "//ortools/base:map_util", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", ], @@ -111,7 +111,7 @@ cc_library( ":variable_and_expressions", "//ortools/base", "//ortools/base:intops", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", ], ) @@ -129,8 +129,8 @@ cc_library( "//ortools/base:intops", "//ortools/math_opt:result_cc_proto", "//ortools/math_opt:solution_cc_proto", - "//ortools/math_opt/core:model_storage", "//ortools/math_opt/core:sparse_vector_view", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:optional", @@ -150,11 +150,11 @@ cc_library( "//ortools/base", "//ortools/base:protoutil", "//ortools/base:status_macros", - "//ortools/util:status_macros", "//ortools/math_opt:result_cc_proto", "//ortools/math_opt:solution_cc_proto", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "//ortools/port:proto_utils", + "//ortools/util:status_macros", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -171,7 +171,7 @@ cc_library( ":id_set", "//ortools/base:intops", "//ortools/math_opt:sparse_containers_cc_proto", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", ], ) @@ -192,8 +192,8 @@ cc_library( "//ortools/math_opt:callback_cc_proto", "//ortools/math_opt:solution_cc_proto", "//ortools/math_opt:sparse_containers_cc_proto", - "//ortools/math_opt/core:model_storage", "//ortools/math_opt/core:sparse_vector_view", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -207,7 +207,7 @@ cc_library( hdrs = ["key_types.h"], deps = [ "//ortools/base", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/strings", ], ) @@ -219,7 +219,7 @@ cc_library( ":key_types", "//ortools/base", "//ortools/math_opt/core:arrow_operator_proxy", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_set", ], ) @@ -237,7 +237,7 @@ cc_library( "//ortools/math_opt:model_parameters_cc_proto", "//ortools/math_opt:solution_cc_proto", "//ortools/math_opt:sparse_containers_cc_proto", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_protobuf//:protobuf", ], ) @@ -250,7 +250,7 @@ cc_library( "//ortools/base", "//ortools/math_opt:model_cc_proto", "//ortools/math_opt:model_update_cc_proto", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/strings", ], ) @@ -315,8 +315,8 @@ cc_library( "//ortools/base:status_macros", "//ortools/math_opt:callback_cc_proto", "//ortools/math_opt:parameters_cc_proto", - "//ortools/math_opt/core:model_storage", "//ortools/math_opt/core:solver", + "//ortools/math_opt/storage:model_storage", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/status:statusor", @@ -374,6 +374,6 @@ cc_library( hdrs = ["statistics.h"], deps = [ ":model", - "//ortools/math_opt/core:model_storage", + "//ortools/math_opt/storage:model_storage", ], ) diff --git a/ortools/math_opt/cpp/callback.cc b/ortools/math_opt/cpp/callback.cc index 95f80b3374..cee4bf5ded 100644 --- a/ortools/math_opt/cpp/callback.cc +++ b/ortools/math_opt/cpp/callback.cc @@ -28,12 +28,12 @@ #include "ortools/base/protoutil.h" #include "ortools/base/strong_int.h" #include "ortools/math_opt/callback.pb.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/cpp/map_filter.h" #include "ortools/math_opt/cpp/sparse_containers.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/callback.h b/ortools/math_opt/cpp/callback.h index 629983f051..764451fc70 100644 --- a/ortools/math_opt/cpp/callback.h +++ b/ortools/math_opt/cpp/callback.h @@ -75,10 +75,10 @@ #include "absl/time/time.h" #include "absl/types/span.h" #include "ortools/math_opt/callback.pb.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/enums.h" // IWYU pragma: export #include "ortools/math_opt/cpp/map_filter.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/id_map.h b/ortools/math_opt/cpp/id_map.h index 3cddc33cc5..7243c15218 100644 --- a/ortools/math_opt/cpp/id_map.h +++ b/ortools/math_opt/cpp/id_map.h @@ -26,8 +26,8 @@ #include "ortools/base/logging.h" #include "ortools/base/strong_int.h" #include "ortools/math_opt/core/arrow_operator_proxy.h" // IWYU pragma: export -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/key_types.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/id_set.h b/ortools/math_opt/cpp/id_set.h index 16cdf16ea7..f269b703e8 100644 --- a/ortools/math_opt/cpp/id_set.h +++ b/ortools/math_opt/cpp/id_set.h @@ -21,8 +21,8 @@ #include "absl/container/flat_hash_set.h" #include "ortools/base/logging.h" #include "ortools/math_opt/core/arrow_operator_proxy.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/key_types.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/key_types.h b/ortools/math_opt/cpp/key_types.h index af1bb300fb..f078752811 100644 --- a/ortools/math_opt/cpp/key_types.h +++ b/ortools/math_opt/cpp/key_types.h @@ -41,7 +41,7 @@ #include "absl/strings/string_view.h" #include "ortools/base/logging.h" -#include "ortools/math_opt/core/model_storage.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/linear_constraint.h b/ortools/math_opt/cpp/linear_constraint.h index 342d0e1031..723b1ba5ca 100644 --- a/ortools/math_opt/cpp/linear_constraint.h +++ b/ortools/math_opt/cpp/linear_constraint.h @@ -21,9 +21,9 @@ #include "ortools/base/logging.h" #include "ortools/base/strong_int.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/id_map.h" // IWYU pragma: export #include "ortools/math_opt/cpp/variable_and_expressions.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/map_filter.h b/ortools/math_opt/cpp/map_filter.h index faa26f3998..2a8cf8d1a8 100644 --- a/ortools/math_opt/cpp/map_filter.h +++ b/ortools/math_opt/cpp/map_filter.h @@ -19,9 +19,9 @@ #include #include "ortools/base/strong_int.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/id_set.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/model.cc b/ortools/math_opt/cpp/model.cc index 36fb12fff7..68d61e28b6 100644 --- a/ortools/math_opt/cpp/model.cc +++ b/ortools/math_opt/cpp/model.cc @@ -26,7 +26,7 @@ #include "ortools/base/logging.h" #include "ortools/base/status_macros.h" #include "ortools/base/strong_int.h" -#include "ortools/math_opt/core/model_storage.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { @@ -186,7 +186,7 @@ void Model::AddToObjective(const QuadraticExpression& objective_terms) { } LinearExpression Model::ObjectiveAsLinearExpression() const { - CHECK(storage()->quadratic_objective().empty()) + CHECK_EQ(storage()->num_quadratic_objective_terms(), 0) << "The objective function contains quadratic terms and cannot be " "represented as a LinearExpression"; LinearExpression result = storage()->objective_offset(); @@ -201,9 +201,9 @@ QuadraticExpression Model::ObjectiveAsQuadraticExpression() const { for (const auto& [v, coef] : storage()->linear_objective()) { result += Variable(storage(), v) * coef; } - for (const auto& [vars, coef] : storage()->quadratic_objective()) { - result += QuadraticTerm(Variable(storage(), vars.first), - Variable(storage(), vars.second), coef); + for (const auto& [v1, v2, coef] : storage()->quadratic_objective_terms()) { + result += + QuadraticTerm(Variable(storage(), v1), Variable(storage(), v2), coef); } return result; } diff --git a/ortools/math_opt/cpp/model.h b/ortools/math_opt/cpp/model.h index 587e130b83..a563fbbc2f 100644 --- a/ortools/math_opt/cpp/model.h +++ b/ortools/math_opt/cpp/model.h @@ -22,13 +22,13 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "ortools/base/logging.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/key_types.h" #include "ortools/math_opt/cpp/linear_constraint.h" // IWYU pragma: export #include "ortools/math_opt/cpp/update_tracker.h" // IWYU pragma: export #include "ortools/math_opt/cpp/variable_and_expressions.h" // IWYU pragma: export #include "ortools/math_opt/model.pb.h" // IWYU pragma: export #include "ortools/math_opt/model_update.pb.h" // IWYU pragma: export +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/model_solve_parameters.cc b/ortools/math_opt/cpp/model_solve_parameters.cc index 006a822c42..f735f541b4 100644 --- a/ortools/math_opt/cpp/model_solve_parameters.cc +++ b/ortools/math_opt/cpp/model_solve_parameters.cc @@ -20,13 +20,13 @@ #include #include "google/protobuf/message.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/solution.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/model_parameters.pb.h" #include "ortools/math_opt/solution.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/model_solve_parameters.h b/ortools/math_opt/cpp/model_solve_parameters.h index 8aab2de1ae..09e0d94094 100644 --- a/ortools/math_opt/cpp/model_solve_parameters.h +++ b/ortools/math_opt/cpp/model_solve_parameters.h @@ -20,12 +20,12 @@ #include #include -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/map_filter.h" // IWYU pragma: export #include "ortools/math_opt/cpp/solution.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/model_parameters.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/solution.cc b/ortools/math_opt/cpp/solution.cc index b5e348b12c..e992568690 100644 --- a/ortools/math_opt/cpp/solution.cc +++ b/ortools/math_opt/cpp/solution.cc @@ -22,13 +22,13 @@ #include "ortools/base/logging.h" #include "ortools/base/status_builder.h" #include "ortools/base/status_macros.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/sparse_containers.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/solution.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage.h" #include "ortools/math_opt/validators/ids_validator.h" #include "ortools/math_opt/validators/sparse_vector_validator.h" #include "ortools/util/status_macros.h" diff --git a/ortools/math_opt/cpp/solution.h b/ortools/math_opt/cpp/solution.h index 6554f7ceea..8d5a20ac25 100644 --- a/ortools/math_opt/cpp/solution.h +++ b/ortools/math_opt/cpp/solution.h @@ -19,13 +19,13 @@ #include "absl/types/optional.h" #include "absl/types/span.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/basis_status.h" #include "ortools/math_opt/cpp/enums.h" // IWYU pragma: export #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/result.pb.h" // IWYU pragma: export #include "ortools/math_opt/solution.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/solve.cc b/ortools/math_opt/cpp/solve.cc index 40825f37dd..18cbc25354 100644 --- a/ortools/math_opt/cpp/solve.cc +++ b/ortools/math_opt/cpp/solve.cc @@ -24,9 +24,9 @@ #include "ortools/base/logging.h" #include "ortools/base/status_macros.h" #include "ortools/math_opt/callback.pb.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/core/solver.h" #include "ortools/math_opt/cpp/model.h" +#include "ortools/math_opt/storage/model_storage.h" #include "ortools/util/status_macros.h" namespace operations_research { diff --git a/ortools/math_opt/cpp/solve.h b/ortools/math_opt/cpp/solve.h index b5dbea1270..3933701488 100644 --- a/ortools/math_opt/cpp/solve.h +++ b/ortools/math_opt/cpp/solve.h @@ -25,7 +25,6 @@ #include #include "absl/status/statusor.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/core/solver.h" #include "ortools/math_opt/cpp/model.h" #include "ortools/math_opt/cpp/parameters.h" // IWYU pragma: export @@ -33,6 +32,7 @@ #include "ortools/math_opt/cpp/solve_result.h" // IWYU pragma: export #include "ortools/math_opt/cpp/solver_init_arguments.h" // IWYU pragma: export #include "ortools/math_opt/parameters.pb.h" // IWYU pragma: export +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/solve_result.cc b/ortools/math_opt/cpp/solve_result.cc index a13a9e0ea7..6bc0077e93 100644 --- a/ortools/math_opt/cpp/solve_result.cc +++ b/ortools/math_opt/cpp/solve_result.cc @@ -27,10 +27,10 @@ #include "ortools/base/logging.h" #include "ortools/base/protoutil.h" #include "ortools/base/status_macros.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/solution.pb.h" +#include "ortools/math_opt/storage/model_storage.h" #include "ortools/port/proto_utils.h" #include "ortools/util/status_macros.h" diff --git a/ortools/math_opt/cpp/solve_result.h b/ortools/math_opt/cpp/solve_result.h index 751133fd73..d04e6ed9f8 100644 --- a/ortools/math_opt/cpp/solve_result.h +++ b/ortools/math_opt/cpp/solve_result.h @@ -24,12 +24,12 @@ #include "absl/types/span.h" #include "ortools/base/logging.h" #include "ortools/gscip/gscip.pb.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/enums.h" // IWYU pragma: export #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/solution.h" // IWYU pragma: export #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/result.pb.h" // IWYU pragma: export +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/sparse_containers.h b/ortools/math_opt/cpp/sparse_containers.h index d9c5757111..774dc56b25 100644 --- a/ortools/math_opt/cpp/sparse_containers.h +++ b/ortools/math_opt/cpp/sparse_containers.h @@ -20,13 +20,13 @@ #include "ortools/base/logging.h" #include "ortools/base/status_builder.h" #include "ortools/base/status_macros.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/cpp/basis_status.h" #include "ortools/math_opt/cpp/linear_constraint.h" #include "ortools/math_opt/cpp/variable_and_expressions.h" #include "ortools/math_opt/solution.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage.h" #include "ortools/math_opt/validators/ids_validator.h" #include "ortools/math_opt/validators/sparse_vector_validator.h" #include "ortools/util/status_macros.h" diff --git a/ortools/math_opt/cpp/statistics.cc b/ortools/math_opt/cpp/statistics.cc index 2ffca08ec5..4198da31ee 100644 --- a/ortools/math_opt/cpp/statistics.cc +++ b/ortools/math_opt/cpp/statistics.cc @@ -20,8 +20,8 @@ #include #include -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/model.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research::math_opt { namespace { diff --git a/ortools/math_opt/cpp/update_tracker.cc b/ortools/math_opt/cpp/update_tracker.cc index ec7f6275b2..ef6f6afda3 100644 --- a/ortools/math_opt/cpp/update_tracker.cc +++ b/ortools/math_opt/cpp/update_tracker.cc @@ -19,8 +19,8 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "ortools/base/logging.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/model.pb.h" +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/update_tracker.h b/ortools/math_opt/cpp/update_tracker.h index 90dcace59c..62d6f3fc60 100644 --- a/ortools/math_opt/cpp/update_tracker.h +++ b/ortools/math_opt/cpp/update_tracker.h @@ -20,9 +20,9 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h" // IWYU pragma: export +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/cpp/variable_and_expressions.h b/ortools/math_opt/cpp/variable_and_expressions.h index 1b388bad7a..2dc9b8170c 100644 --- a/ortools/math_opt/cpp/variable_and_expressions.h +++ b/ortools/math_opt/cpp/variable_and_expressions.h @@ -101,9 +101,9 @@ #include "absl/container/flat_hash_map.h" #include "ortools/base/logging.h" #include "ortools/base/strong_int.h" -#include "ortools/math_opt/core/model_storage.h" #include "ortools/math_opt/cpp/id_map.h" // IWYU pragma: export #include "ortools/math_opt/cpp/key_types.h" // IWYU pragma: export +#include "ortools/math_opt/storage/model_storage.h" namespace operations_research { namespace math_opt { diff --git a/ortools/math_opt/model.proto b/ortools/math_opt/model.proto index 2b17f703b6..58aa23e99c 100644 --- a/ortools/math_opt/model.proto +++ b/ortools/math_opt/model.proto @@ -88,23 +88,35 @@ message LinearConstraintsProto { repeated string names = 4; } -// An optimization problem of the form +// Data for representing a single SOS1 or SOS2 constraint. +message SosConstraintProto { + // The expressions over which to apply the SOS constraint: + // * SOS1: At most one element takes a nonzero value. + // * SOS2: At most two elements take nonzero values, and they must be + // adjacent in the repeated ordering. + repeated LinearExpressionProto expressions = 1; + + // Either empty or of equal length to expressions. If empty, default weights + // are 1, 2, ... + repeated double weights = 2; + + // Parent messages may have uniqueness requirements on this field; e.g., see + // ModelProto.sos1_constraints and SosConstraintUpdatesProto.new_constraints. + string name = 3; +} + +// An optimization problem. For full details, see go/mathopt-model. // -// min(/max) c * x -// s.t. -// cons_lb <= A * x <= cons_ub -// var_lb <= x <= var_ub -// x_i integer for i in I +// MathOpt supports: +// - Continuous and integer decision variables with optional finite bounds. +// - Linear and quadratic objectives, either minimized or maximized. +// - A number of constraints types, including: +// * Linear constraints +// * Logical constraints +// > SOS1 and SOS2 constraints // -// where: -// * x is a vector of decision variables in R^n -// * c, var_lb, var_ub are vectors in R^n -// * cons_lb, cons_ub are vectors in R^m -// * A is a sparse matrix in R^{m by n} -// * potentially var_lb, cons_lb are -inf -// * potentially var_ub, cons_ub are +inf -// -// For more details see go/mathopt-model +// By default, constraints are represented in "id-to-data" maps. However, we +// represent linear constraints in a more efficient "struct-of-arrays" format. message ModelProto { string name = 1; VariablesProto variables = 2; @@ -119,4 +131,30 @@ message ModelProto { // * Matrix entries not specified are zero. // * linear_constraint_matrix.values must all be finite. SparseDoubleMatrixProto linear_constraint_matrix = 5; + + // Mapped constraints (i.e., stored in "constraint ID"-to-"constraint data" + // map). For each subsequent submessage, we require that: + // * Each key is in [0, max(int64)). + // * Each key is unique in its respective map (but not necessarily across + // constraint types) + // * Each value contains a name field, and each nonempty name must be + // distinct across all map entries (but not necessarily across constraint + // types). + + // Reserved for quadratic constraints. + reserved 6; + + // SOS1 constraints in the model, which constrain that at most one + // `expression` can be nonzero. The optional `weights` entries are an + // implementation detail used by the solver to (hopefully) converge more + // quickly. In more detail, solvers may (or may not) use these weights to + // select branching decisions that produce "balanced" children nodes. + map sos1_constraints = 7; + + // SOS2 constraints in the model, which constrain that at most two entries of + // `expression` can be nonzero, and they must be adjacent in their ordering. + // If no `weights` are provided, this ordering is their linear ordering in the + // `expressions` list; if `weights` are presented, the ordering is taken with + // respect to these values in increasing order. + map sos2_constraints = 8; } diff --git a/ortools/math_opt/model_update.proto b/ortools/math_opt/model_update.proto index 6ae4f41067..56acb7b5c1 100644 --- a/ortools/math_opt/model_update.proto +++ b/ortools/math_opt/model_update.proto @@ -107,6 +107,23 @@ message LinearConstraintUpdatesProto { SparseDoubleVectorProto upper_bounds = 2; } +// Data for updates to SOS1 and SOS2 constraints; only addition and deletion, no +// support for in-place constraint updates. +message SosConstraintUpdatesProto { + // Removes SOS constraints from the model. + // + // Each value must be in [0, max(int64)). Values must be in strictly + // increasing order. Applies only to existing SOS constraint ids that have not + // yet been deleted. + repeated int64 deleted_constraint_ids = 1; + + // Add new SOS constraints to the model. All keys must be in [0, max(int64)), + // and must be greater than any ids used in the initial model and previous + // updates. All nonempty names should be distinct from existing names and each + // other. + map new_constraints = 2; +} + // Updates to a ModelProto. message ModelUpdateProto { // Removes variables from the model. @@ -161,4 +178,11 @@ message ModelUpdateProto { // * Zero values delete existing entries, and have no effect for new entries. // * linear_constraint_matrix.values must all be finite. SparseDoubleMatrixProto linear_constraint_matrix_updates = 8; + + // Reserved for quadratic constraint updates. + reserved 9; + + // Updates the general constraints (addition or deletion only). + SosConstraintUpdatesProto sos1_constraint_updates = 10; + SosConstraintUpdatesProto sos2_constraint_updates = 11; } diff --git a/ortools/math_opt/solvers/gurobi_solver.cc b/ortools/math_opt/solvers/gurobi_solver.cc index 5a3b04d31a..fc337fbe67 100644 --- a/ortools/math_opt/solvers/gurobi_solver.cc +++ b/ortools/math_opt/solvers/gurobi_solver.cc @@ -1860,7 +1860,7 @@ GurobiSolver::RegisterCallback(const CallbackRegistrationProto& registration, // can not be performed safely. RETURN_IF_ERROR(gurobi_->SetIntParam(GRB_INT_PAR_LAZYCONSTRAINTS, 1)); } - return absl::make_unique( + return std::make_unique( GurobiCallbackInput{ .user_cb = cb, .message_cb = message_cb, diff --git a/ortools/math_opt/storage/BUILD.bazel b/ortools/math_opt/storage/BUILD.bazel new file mode 100644 index 0000000000..da6eb8fe89 --- /dev/null +++ b/ortools/math_opt/storage/BUILD.bazel @@ -0,0 +1,84 @@ +package(default_visibility = ["//ortools/math_opt:__subpackages__"]) + +cc_library( + name = "model_storage_types", + hdrs = ["model_storage_types.h"], + deps = ["//ortools/base:intops",], +) + +cc_library( + name = "sparse_matrix", + srcs = ["sparse_matrix.cc"], + hdrs = ["sparse_matrix.h"], + deps = [ + ":model_storage_types", + "//ortools/base:intops", + "//ortools/base:map_util", + "//ortools/base:strong_vector", + "//ortools/math_opt:sparse_containers_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +cc_library( + name = "model_storage", + srcs = ["model_storage.cc"], + hdrs = ["model_storage.h"], + deps = [ + ":model_storage_types", + ":model_update_merge", + ":sparse_matrix", + "//ortools/base", + #"//ortools/base:logging", + "//ortools/base:map_util", + "//ortools/base:intops", + "//ortools/math_opt:model_cc_proto", + "//ortools/math_opt:model_update_cc_proto", + "//ortools/math_opt:solution_cc_proto", + "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/core:sparse_vector_view", + "//ortools/math_opt/validators:model_validator", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/meta:type_traits", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:span", + ], +) + +cc_library( + name = "model_update_merge", + srcs = ["model_update_merge.cc"], + hdrs = ["model_update_merge.h"], + deps = [ + ":proto_merging_utils", + "//ortools/base", + #"//ortools/base:logging", + #"@com_google_absl//absl/log:check", + "//ortools/math_opt:model_cc_proto", + "//ortools/math_opt:model_update_cc_proto", + "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/core:sparse_vector_view", + ], +) + +cc_library( + name = "proto_merging_utils", + srcs = ["proto_merging_utils.cc"], + hdrs = ["proto_merging_utils.h"], + deps = [ + "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/core:sparse_vector_view", + "//ortools/base:protobuf_util", + "//ortools/base", + #"//ortools/base:logging", + #"@com_google_absl//absl/log:check", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/ortools/math_opt/core/model_storage.cc b/ortools/math_opt/storage/model_storage.cc similarity index 87% rename from ortools/math_opt/core/model_storage.cc rename to ortools/math_opt/storage/model_storage.cc index 604a9763d5..0bf2abb132 100644 --- a/ortools/math_opt/core/model_storage.cc +++ b/ortools/math_opt/storage/model_storage.cc @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ortools/math_opt/core/model_storage.h" +#include "ortools/math_opt/storage/model_storage.h" #include #include @@ -33,12 +33,14 @@ #include "ortools/base/map_util.h" #include "ortools/base/status_macros.h" #include "ortools/base/strong_int.h" -#include "ortools/math_opt/core/model_update_merge.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h" #include "ortools/math_opt/solution.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage_types.h" +#include "ortools/math_opt/storage/model_update_merge.h" +#include "ortools/math_opt/storage/sparse_matrix.h" #include "ortools/math_opt/validators/model_validator.h" namespace operations_research { @@ -110,18 +112,6 @@ void AppendFromMap(const absl::flat_hash_set& dirty_keys, } } -template -absl::flat_hash_map SparseBasisVectorToMap( - const SparseBasisStatusVector& sparse_vector) { - absl::flat_hash_map result; - CHECK_EQ(sparse_vector.ids_size(), sparse_vector.values_size()); - result.reserve(sparse_vector.ids_size()); - for (const auto [id, value] : MakeView(sparse_vector)) { - gtl::InsertOrDie(&result, T(id), static_cast(value)); - } - return result; -} - // If an element in keys is not found in coefficients, it is set to 0.0 in // matrix. Keys must be in lexicographic ordering (i.e. sorted). // NOTE: This signature can be updated to take a Span instead of a vector if @@ -256,9 +246,6 @@ void ModelStorage::AddVariableInternal(const VariableId id, if (!lazy_matrix_columns_.empty()) { gtl::InsertOrDie(&lazy_matrix_columns_, id, {}); } - if (!lazy_quadratic_objective_by_variable_.empty()) { - gtl::InsertOrDie(&lazy_quadratic_objective_by_variable_, id, {}); - } } void ModelStorage::AddVariables(const VariablesProto& variables) { @@ -284,33 +271,16 @@ void ModelStorage::DeleteVariable(const VariableId id) { dirty_variable_upper_bounds_.erase(id); dirty_variable_is_integer_.erase(id); dirty_linear_objective_coefficients_.erase(id); - } - // If we do not have any quadratic updates to delete, we would like to avoid - // initializing the lazy data structures. The updates might tracked in: - // 1. dirty_quadratic_objective_coefficients_ (both variables old) - // 2. quadratic_objective_ (at least one new variable) - // If both maps are empty, we can skip the update and initializiation. Note - // that we could be a bit more clever here based on whether the deleted - // variable is new or old, but that makes the logic more complex. - if (!quadratic_objective_.empty() || - !dirty_quadratic_objective_coefficients_.empty()) { - EnsureLazyQuadraticObjective(); - const auto related_variables = - lazy_quadratic_objective_by_variable_.extract(id); - for (const VariableId other_id : related_variables.mapped()) { - // Due to the extract above, the at lookup will fail if other_id == id. - if (id != other_id) { - CHECK_GT(lazy_quadratic_objective_by_variable_.at(other_id).erase(id), - 0); - } - const auto ordered_pair = internal::MakeOrderedPair(id, other_id); - quadratic_objective_.erase(ordered_pair); + for (const VariableId related : quadratic_objective_.RelatedVariables(id)) { // We can only have a dirty update to wipe clean if both variables are old - if (id < variables_checkpoint_ && other_id < variables_checkpoint_) { - dirty_quadratic_objective_coefficients_.erase(ordered_pair); + if (related < variables_checkpoint_) { + dirty_quadratic_objective_coefficients_.erase( + internal::MakeOrderedPair(id, related)); } } } + + quadratic_objective_.Delete(id); for (const LinearConstraintId related_constraint : lazy_matrix_columns_.at(id)) { CHECK_GT(lazy_matrix_rows_.at(related_constraint).erase(id), 0); @@ -440,7 +410,7 @@ ModelProto ModelStorage::ExportModel() const { SortedMapKeys(linear_objective_), linear_objective_, *result.mutable_objective()->mutable_linear_coefficients()); *result.mutable_objective()->mutable_quadratic_coefficients() = - ExportMatrix(quadratic_objective_, SortedMapKeys(quadratic_objective_)); + quadratic_objective_.Proto(); // Pull out the linear constraints. for (const LinearConstraintId con : SortedMapKeys(linear_constraints_)) { @@ -535,42 +505,11 @@ std::optional ModelStorage::ExportSharedModelUpdate() { obj_updates->mutable_linear_coefficients()->add_values(*double_value); } } - // If we do not have any quadratic updates to push, we would like to avoid - // initializing the lazy data structures. The updates might tracked in: - // 1. dirty_quadratic_objective_coefficients_ (both variables old) - // 2. quadratic_objective_ (at least one new variable) - // If both maps are empty, we can skip the update and initializiation. - if (!quadratic_objective_.empty() || - !dirty_quadratic_objective_coefficients_.empty()) { - EnsureLazyQuadraticObjective(); - // NOTE: dirty_quadratic_objective_coefficients_ only tracks terms where - // both variables are "old". - std::vector> quadratic_objective_updates( - dirty_quadratic_objective_coefficients_.begin(), - dirty_quadratic_objective_coefficients_.end()); - // Now, we loop through the "new" variables and track updates involving - // them. We need to look out for two things: - // * The "other" variable in the term can either be new or old. - // * We cannot doubly insert terms when both variables are new. - // Note that this traversal is doing at most twice as much work as - // necessary. - for (VariableId new_var = variables_checkpoint_; - new_var < next_variable_id_; ++new_var) { - if (variables_.contains(new_var)) { - for (const VariableId other_var : - lazy_quadratic_objective_by_variable_.at(new_var)) { - if (other_var <= new_var) { - quadratic_objective_updates.push_back( - internal::MakeOrderedPair(new_var, other_var)); - } - } - } - } - std::sort(quadratic_objective_updates.begin(), - quadratic_objective_updates.end()); - *result.mutable_objective_updates()->mutable_quadratic_coefficients() = - ExportMatrix(quadratic_objective_, quadratic_objective_updates); - } + + *result.mutable_objective_updates()->mutable_quadratic_coefficients() = + quadratic_objective_.Update(variables_, variables_checkpoint_, + next_variable_id_, + dirty_quadratic_objective_coefficients_); // Update the linear constraints auto lin_con_updates = result.mutable_linear_constraint_updates(); @@ -646,22 +585,6 @@ void ModelStorage::EnsureLazyMatrixRows() { } } -void ModelStorage::EnsureLazyQuadraticObjective() { - if (lazy_quadratic_objective_by_variable_.empty()) { - for (const auto& [var, data] : variables_) { - lazy_quadratic_objective_by_variable_.insert({var, {}}); - } - for (const auto& [vars, coeff] : quadratic_objective_) { - lazy_quadratic_objective_by_variable_.at(vars.first).insert(vars.second); - lazy_quadratic_objective_by_variable_.at(vars.second).insert(vars.first); - } - for (const auto& vars : dirty_quadratic_objective_coefficients_) { - lazy_quadratic_objective_by_variable_.at(vars.first).insert(vars.second); - lazy_quadratic_objective_by_variable_.at(vars.second).insert(vars.first); - } - } -} - void ModelStorage::SharedCheckpoint() { variables_checkpoint_ = next_variable_id_; linear_constraints_checkpoint_ = next_linear_constraint_id_; @@ -803,6 +726,7 @@ void ModelStorage::CheckpointLocked(const UpdateTrackerId update_tracker) { } } SharedCheckpoint(); + data->updates.clear(); } diff --git a/ortools/math_opt/core/model_storage.h b/ortools/math_opt/storage/model_storage.h similarity index 93% rename from ortools/math_opt/core/model_storage.h rename to ortools/math_opt/storage/model_storage.h index f7b2ffca16..75da53fc38 100644 --- a/ortools/math_opt/core/model_storage.h +++ b/ortools/math_opt/storage/model_storage.h @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef OR_TOOLS_MATH_OPT_CORE_MODEL_STORAGE_H_ -#define OR_TOOLS_MATH_OPT_CORE_MODEL_STORAGE_H_ +#ifndef OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_H_ +#define OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_H_ #include #include @@ -33,14 +33,12 @@ #include "ortools/base/strong_int.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h" +#include "ortools/math_opt/storage/model_storage_types.h" +#include "ortools/math_opt/storage/sparse_matrix.h" namespace operations_research { namespace math_opt { -DEFINE_STRONG_INT_TYPE(VariableId, int64_t); -DEFINE_STRONG_INT_TYPE(LinearConstraintId, int64_t); -DEFINE_STRONG_INT_TYPE(UpdateTrackerId, int64_t); - // An index based C++ API for building & storing optimization problems. // // Note that this API should usually not be used by C++ users that should prefer @@ -376,10 +374,14 @@ class ModelStorage { inline const absl::flat_hash_map& linear_objective() const; + inline int64_t num_quadratic_objective_terms() const; + // The variable pairs with nonzero quadratic objective coefficients. The keys - // are ordered such that .first <= .second. - inline const absl::flat_hash_map, double>& - quadratic_objective() const; + // are ordered such that .first <= .second. All values are nonempty. + // + // TODO(b/233630053) do no allocate the result, expose an iterator API. + inline std::vector> + quadratic_objective_terms() const; // Returns a sorted vector of all variables in the model with nonzero linear // objective coefficients. @@ -577,10 +579,6 @@ class ModelStorage { // the model. void EnsureLazyMatrixRows(); - // Initializes lazy_quadratic_objective_by_variable_ if it is still empty and - // there is at least one variable in the model. - void EnsureLazyQuadraticObjective(); - // Export a single variable to proto. void AppendVariable(VariableId id, VariablesProto& variables_proto) const; @@ -616,28 +614,20 @@ class ModelStorage { absl::flat_hash_map variables_; absl::flat_hash_map linear_constraints_; + // The values of the map must never include zero. absl::flat_hash_map linear_objective_; - // The values of the map must never include zero. The keys must be upper - // triangular, i.e. .first <= .second. - absl::flat_hash_map, double> - quadratic_objective_; + + SparseSymmetricMatrix quadratic_objective_; + // The values of the map must never include zero. absl::flat_hash_map, double> linear_constraint_matrix_; + absl::flat_hash_map> lazy_matrix_columns_; absl::flat_hash_map> lazy_matrix_rows_; - // To handle deletions we need to have an efficient way to look up which - // quadratic objective terms involve a given variable. This map stores this - // information where the key corresponds to a variable and the value is the - // set of all variables appearing in a quadratic objective term with the key. - // This data structure is only initialized after a call to - // EnsureLazyQuadraticObjective. As of 11/17/2021, this will have occurred if - // a nonzero quadratic objective term has ever been added to the model. - absl::flat_hash_map> - lazy_quadratic_objective_by_variable_; // Update information // @@ -918,9 +908,7 @@ double ModelStorage::linear_objective_coefficient(VariableId variable) const { double ModelStorage::quadratic_objective_coefficient( const VariableId first_variable, const VariableId second_variable) const { - return gtl::FindWithDefault( - quadratic_objective_, - internal::MakeOrderedPair(first_variable, second_variable)); + return quadratic_objective_.get(first_variable, second_variable); } bool ModelStorage::is_linear_objective_coefficient_nonzero( @@ -930,8 +918,7 @@ bool ModelStorage::is_linear_objective_coefficient_nonzero( bool ModelStorage::is_quadratic_objective_coefficient_nonzero( const VariableId first_variable, const VariableId second_variable) const { - return quadratic_objective_.contains( - internal::MakeOrderedPair(first_variable, second_variable)); + return quadratic_objective_.get(first_variable, second_variable) != 0.0; } void ModelStorage::set_is_maximize(bool is_maximize) { @@ -977,33 +964,12 @@ void ModelStorage::set_linear_objective_coefficient(VariableId variable, void ModelStorage::set_quadratic_objective_coefficient( const VariableId first_variable, const VariableId second_variable, double value) { + const bool updated = + quadratic_objective_.set(first_variable, second_variable, value); const std::pair key = internal::MakeOrderedPair(first_variable, second_variable); - bool was_updated = false; - if (value == 0.0) { - if (quadratic_objective_.erase(key) > 0) { - was_updated = true; - } - } else { - const auto [iterator, inserted] = - quadratic_objective_.try_emplace(key, value); - if (inserted) { - was_updated = true; - } else if (iterator->second != value) { - iterator->second = value; - was_updated = true; - } - } - if (was_updated) { - if (!lazy_quadratic_objective_by_variable_.empty()) { - lazy_quadratic_objective_by_variable_.at(first_variable) - .insert(second_variable); - lazy_quadratic_objective_by_variable_.at(second_variable) - .insert(first_variable); - } - if (key.second < variables_checkpoint_) { - dirty_quadratic_objective_coefficients_.insert(key); - } + if (updated && key.second < variables_checkpoint_) { + dirty_quadratic_objective_coefficients_.insert(key); } } @@ -1012,11 +978,12 @@ void ModelStorage::clear_objective() { while (!linear_objective_.empty()) { set_linear_objective_coefficient(linear_objective_.begin()->first, 0.0); } - while (!quadratic_objective_.empty()) { - set_quadratic_objective_coefficient( - quadratic_objective_.begin()->first.first, - quadratic_objective_.begin()->first.second, 0.0); + for (const auto [var_pair, value] : quadratic_objective_.values()) { + if (var_pair.second < variables_checkpoint_ && value != 0.0) { + dirty_quadratic_objective_coefficients_.insert(var_pair); + } } + quadratic_objective_.Clear(); } const absl::flat_hash_map& ModelStorage::linear_objective() @@ -1024,12 +991,16 @@ const absl::flat_hash_map& ModelStorage::linear_objective() return linear_objective_; } -const absl::flat_hash_map, double>& -ModelStorage::quadratic_objective() const { - return quadratic_objective_; +int64_t ModelStorage::num_quadratic_objective_terms() const { + return quadratic_objective_.nonzeros(); +} + +std::vector> +ModelStorage::quadratic_objective_terms() const { + return quadratic_objective_.Terms(); } } // namespace math_opt } // namespace operations_research -#endif // OR_TOOLS_MATH_OPT_CORE_MODEL_STORAGE_H_ +#endif // OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_H_ diff --git a/ortools/math_opt/storage/model_storage_types.h b/ortools/math_opt/storage/model_storage_types.h new file mode 100644 index 0000000000..e89426cc33 --- /dev/null +++ b/ortools/math_opt/storage/model_storage_types.h @@ -0,0 +1,29 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_TYPES_H_ +#define OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_TYPES_H_ + +#include + +#include "ortools/base/strong_int.h" + +namespace operations_research::math_opt { + +DEFINE_STRONG_INT_TYPE(VariableId, int64_t); +DEFINE_STRONG_INT_TYPE(LinearConstraintId, int64_t); +DEFINE_STRONG_INT_TYPE(UpdateTrackerId, int64_t); + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_STORAGE_MODEL_STORAGE_TYPES_H_ diff --git a/ortools/math_opt/core/model_update_merge.cc b/ortools/math_opt/storage/model_update_merge.cc similarity index 50% rename from ortools/math_opt/core/model_update_merge.cc rename to ortools/math_opt/storage/model_update_merge.cc index d0d24dd52a..685081d281 100644 --- a/ortools/math_opt/core/model_update_merge.cc +++ b/ortools/math_opt/storage/model_update_merge.cc @@ -11,24 +11,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ortools/math_opt/core/model_update_merge.h" +#include "ortools/math_opt/storage/model_update_merge.h" -#include #include #include #include -#include -#include "absl/container/flat_hash_set.h" -#include "ortools/base/integral_types.h" #include "ortools/base/logging.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/proto_merging_utils.h" -namespace operations_research { -namespace math_opt { +namespace operations_research::math_opt { void MergeIntoUpdate(const ModelUpdateProto& from_new, ModelUpdateProto& into_old) { @@ -36,13 +32,12 @@ void MergeIntoUpdate(const ModelUpdateProto& from_new, // variables that were created in `into_old`. Below we will simply remove // those variables from the list of new variables in the merge; thus making // the update as if those variables never existed. - internal::MergeIntoSortedIds(from_new.deleted_variable_ids(), - *into_old.mutable_deleted_variable_ids(), - /*deleted=*/into_old.new_variables().ids()); - internal::MergeIntoSortedIds( - from_new.deleted_linear_constraint_ids(), - *into_old.mutable_deleted_linear_constraint_ids(), - /*deleted=*/into_old.new_linear_constraints().ids()); + MergeIntoSortedIds(from_new.deleted_variable_ids(), + *into_old.mutable_deleted_variable_ids(), + /*deleted=*/into_old.new_variables().ids()); + MergeIntoSortedIds(from_new.deleted_linear_constraint_ids(), + *into_old.mutable_deleted_linear_constraint_ids(), + /*deleted=*/into_old.new_linear_constraints().ids()); // For variables and linear constraints updates, we want to ignore updates of: // @@ -67,25 +62,25 @@ void MergeIntoUpdate(const ModelUpdateProto& from_new, into_old.new_linear_constraints().ids()); // Merge updates of variable properties. - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.variable_updates().lower_bounds(), *into_old.mutable_variable_updates()->mutable_lower_bounds(), from_deleted_and_into_new_variable_ids); - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.variable_updates().upper_bounds(), *into_old.mutable_variable_updates()->mutable_upper_bounds(), from_deleted_and_into_new_variable_ids); - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.variable_updates().integers(), *into_old.mutable_variable_updates()->mutable_integers(), from_deleted_and_into_new_variable_ids); // Merge updates of linear constraints properties. - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.linear_constraint_updates().lower_bounds(), *into_old.mutable_linear_constraint_updates()->mutable_lower_bounds(), from_deleted_and_into_new_linear_constraint_ids); - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.linear_constraint_updates().upper_bounds(), *into_old.mutable_linear_constraint_updates()->mutable_upper_bounds(), from_deleted_and_into_new_linear_constraint_ids); @@ -110,28 +105,28 @@ void MergeIntoUpdate(const ModelUpdateProto& from_new, CHECK_GT(*from_new.new_variables().ids().begin(), *into_old.new_variables().ids().rbegin()); } - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_variables().ids(), /*values=*/*into_old.mutable_new_variables()->mutable_lower_bounds(), /*deleted=*/from_new.deleted_variable_ids(), /*updates=*/from_new.variable_updates().lower_bounds()); - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_variables().ids(), /*values=*/*into_old.mutable_new_variables()->mutable_upper_bounds(), /*deleted=*/from_new.deleted_variable_ids(), /*updates=*/from_new.variable_updates().upper_bounds()); - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_variables().ids(), /*values=*/*into_old.mutable_new_variables()->mutable_integers(), /*deleted=*/from_new.deleted_variable_ids(), /*updates=*/from_new.variable_updates().integers()); - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_variables().ids(), /*values=*/*into_old.mutable_new_variables()->mutable_names(), /*deleted=*/from_new.deleted_variable_ids(), // We use an empty view here since names can't be updated. /*updates=*/SparseVectorView()); - internal::RemoveDeletedIds( + RemoveDeletedIds( /*ids=*/*into_old.mutable_new_variables()->mutable_ids(), /*deleted=*/from_new.deleted_variable_ids()); into_old.mutable_new_variables()->MergeFrom(from_new.new_variables()); @@ -143,25 +138,25 @@ void MergeIntoUpdate(const ModelUpdateProto& from_new, CHECK_GT(*from_new.new_linear_constraints().ids().begin(), *into_old.new_linear_constraints().ids().rbegin()); } - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_linear_constraints().ids(), /*values=*/ *into_old.mutable_new_linear_constraints()->mutable_lower_bounds(), /*deleted=*/from_new.deleted_linear_constraint_ids(), /*updates=*/from_new.linear_constraint_updates().lower_bounds()); - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_linear_constraints().ids(), /*values=*/ *into_old.mutable_new_linear_constraints()->mutable_upper_bounds(), /*deleted=*/from_new.deleted_linear_constraint_ids(), /*updates=*/from_new.linear_constraint_updates().upper_bounds()); - internal::UpdateNewElementProperty( + UpdateNewElementProperty( /*ids=*/into_old.new_linear_constraints().ids(), /*values=*/*into_old.mutable_new_linear_constraints()->mutable_names(), /*deleted=*/from_new.deleted_linear_constraint_ids(), // We use an empty view here since names can't be updated. /*updates=*/SparseVectorView()); - internal::RemoveDeletedIds( + RemoveDeletedIds( /*ids=*/*into_old.mutable_new_linear_constraints()->mutable_ids(), /*deleted=*/from_new.deleted_linear_constraint_ids()); into_old.mutable_new_linear_constraints()->MergeFrom( @@ -176,184 +171,22 @@ void MergeIntoUpdate(const ModelUpdateProto& from_new, into_old.mutable_objective_updates()->set_offset_update( from_new.objective_updates().offset_update()); } - internal::MergeIntoSparseVector( + MergeIntoSparseVector( from_new.objective_updates().linear_coefficients(), *into_old.mutable_objective_updates()->mutable_linear_coefficients(), from_new.deleted_variable_ids()); - internal::MergeIntoSparseDoubleMatrix( + MergeIntoSparseDoubleMatrix( from_new.objective_updates().quadratic_coefficients(), *into_old.mutable_objective_updates()->mutable_quadratic_coefficients(), /*deleted_rows=*/from_new.deleted_variable_ids(), /*deleted_columns=*/from_new.deleted_variable_ids()); // Merge the linear constraints coefficients. - internal::MergeIntoSparseDoubleMatrix( + MergeIntoSparseDoubleMatrix( from_new.linear_constraint_matrix_updates(), *into_old.mutable_linear_constraint_matrix_updates(), /*deleted_rows=*/from_new.deleted_linear_constraint_ids(), /*deleted_columns=*/from_new.deleted_variable_ids()); } -namespace internal { - -void RemoveDeletedIds(google::protobuf::RepeatedField& ids, - const google::protobuf::RepeatedField& deleted) { - int next_insertion_point = 0; - int deleted_i = 0; - for (const int64_t id : ids) { - while (deleted_i < deleted.size() && deleted[deleted_i] < id) { - ++deleted_i; - } - if (deleted_i < deleted.size() && deleted[deleted_i] == id) { - continue; - } - ids[next_insertion_point] = id; - ++next_insertion_point; - } - ids.Truncate(next_insertion_point); -} - -void MergeIntoSortedIds( - const google::protobuf::RepeatedField& from_new, - google::protobuf::RepeatedField& into_old, - const google::protobuf::RepeatedField& deleted) { - google::protobuf::RepeatedField result; - - int from_new_i = 0; - int into_old_i = 0; - int deleted_i = 0; - - // Functions that adds the input id to the result if it is not in deleted. It - // updates deleted_i as a side effect too. - const auto add_if_not_deleted = [&](const int64_t id) { - while (deleted_i < deleted.size() && deleted[deleted_i] < id) { - ++deleted_i; - } - if (deleted_i == deleted.size() || deleted[deleted_i] != id) { - result.Add(id); - } - }; - - while (from_new_i < from_new.size() && into_old_i < into_old.size()) { - if (from_new[from_new_i] < into_old[into_old_i]) { - add_if_not_deleted(from_new[from_new_i]); - ++from_new_i; - } else if (from_new[from_new_i] > into_old[into_old_i]) { - add_if_not_deleted(into_old[into_old_i]); - ++into_old_i; - } else { // from_new[from_new_i] == into_old[into_old_i] - add_if_not_deleted(from_new[from_new_i]); - ++from_new_i; - ++into_old_i; - } - } - - // At this point either from_new_i == from_new.size() or to_i == to.size() or - // both. And the one that is not empty, if it exists, has elements greater - // than all other elements already inserted. - for (; from_new_i < from_new.size(); ++from_new_i) { - add_if_not_deleted(from_new[from_new_i]); - } - for (; into_old_i < into_old.size(); ++into_old_i) { - add_if_not_deleted(into_old[into_old_i]); - } - - into_old.Swap(&result); -} - -void MergeIntoSparseDoubleMatrix( - const SparseDoubleMatrixProto& from_new, SparseDoubleMatrixProto& into_old, - const google::protobuf::RepeatedField& deleted_rows, - const google::protobuf::RepeatedField& deleted_columns) { - SparseDoubleMatrixProto result; - auto& result_row_ids = *result.mutable_row_ids(); - auto& result_column_ids = *result.mutable_column_ids(); - auto& result_coefficients = *result.mutable_coefficients(); - - // Contrary to rows that are traversed in order (the matrix is using row-major - // order), columns are not. Thus we would have to start the iteration on - // deleted_columns for each new row of the matrix if we wanted to use the same - // approach as with rows. This would be O(num_rows * num_deleted_columns). - // - // Here we use a hash-set to be O(num_matrix_elements + - // num_deleted_columns). The downside is that we consumed - // O(num_deleted_columns) additional memory. - // - // We could have used binary search that would be O(num_matrix_elements * - // lg(num_deleted_columns)) but without additional memory. - const absl::flat_hash_set deleted_columns_set( - deleted_columns.begin(), deleted_columns.end()); - - int from_new_i = 0; - int into_old_i = 0; - int deleted_rows_i = 0; - - // Functions that adds the input tuple (row_id, col_id, coefficient) to the - // result if the input row_id and col_id are not in deleted_rows or - // deleted_columns. It updates deleted_rows_i and deleted_columns_i as a side - // effect too. - const auto add_if_not_deleted = [&](const int64_t row_id, - const int64_t col_id, - const double coefficient) { - while (deleted_rows_i < deleted_rows.size() && - deleted_rows[deleted_rows_i] < row_id) { - ++deleted_rows_i; - } - if ((deleted_rows_i != deleted_rows.size() && - deleted_rows[deleted_rows_i] == row_id) || - deleted_columns_set.contains(col_id)) { - return; - } - result_row_ids.Add(row_id); - result_column_ids.Add(col_id); - result_coefficients.Add(coefficient); - }; - - while (from_new_i < from_new.row_ids_size() && - into_old_i < into_old.row_ids_size()) { - // Matrices are in row-major order and std::pair comparison is - // lexicographical, thus matrices are sorted in the natural order of pairs - // of coordinates (row, col). - const auto from_new_coordinates = std::make_pair( - from_new.row_ids(from_new_i), from_new.column_ids(from_new_i)); - const auto into_old_coordinates = std::make_pair( - into_old.row_ids(into_old_i), into_old.column_ids(into_old_i)); - if (from_new_coordinates < into_old_coordinates) { - add_if_not_deleted(from_new_coordinates.first, - from_new_coordinates.second, - from_new.coefficients(from_new_i)); - ++from_new_i; - } else if (from_new_coordinates > into_old_coordinates) { - add_if_not_deleted(into_old_coordinates.first, - into_old_coordinates.second, - into_old.coefficients(into_old_i)); - ++into_old_i; - } else { // from_new_coordinates == into_old_coordinates - add_if_not_deleted(from_new_coordinates.first, - from_new_coordinates.second, - from_new.coefficients(from_new_i)); - ++from_new_i; - ++into_old_i; - } - } - - // At this point either from_new_i == from_new.row_ids_size() or - // to_i == to.row_ids_size() (or both). And the one that is not empty, if it - // exists, has elements greater than all other elements already inserted. - for (; from_new_i < from_new.row_ids_size(); ++from_new_i) { - add_if_not_deleted(from_new.row_ids(from_new_i), - from_new.column_ids(from_new_i), - from_new.coefficients(from_new_i)); - } - for (; into_old_i < into_old.row_ids_size(); ++into_old_i) { - add_if_not_deleted(into_old.row_ids(into_old_i), - into_old.column_ids(into_old_i), - into_old.coefficients(into_old_i)); - } - - into_old.Swap(&result); -} - -} // namespace internal -} // namespace math_opt -} // namespace operations_research +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/storage/model_update_merge.h b/ortools/math_opt/storage/model_update_merge.h new file mode 100644 index 0000000000..d655c153c2 --- /dev/null +++ b/ortools/math_opt/storage/model_update_merge.h @@ -0,0 +1,40 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_MATH_OPT_STORAGE_MODEL_UPDATE_MERGE_H_ +#define OR_TOOLS_MATH_OPT_STORAGE_MODEL_UPDATE_MERGE_H_ + +#include "ortools/math_opt/model_update.pb.h" + +namespace operations_research::math_opt { + +// Merges the `from_new` update into the `into_old` one. +// +// The `from_new` update must represent an update that happens after the +// `into_old` one is applied. Thus when the two updates have overlaps, the +// `from_new` one overrides the value of the `into_old` one (i.e. the `from_new` +// update is expected to be more recent). +// +// This function also CHECKs that the ids of new variables and constraints in +// `from_new` are greater than the ones in `into_old` (as expected if `from_new` +// happens after `into_old`). +// +// Note that the complexity is O(size(from_new) + size(into_old)) thus if you +// need to merge a long list of updates this may be not efficient enough. In +// that case an n-way merge would be needed to be implemented here. +void MergeIntoUpdate(const ModelUpdateProto& from_new, + ModelUpdateProto& into_old); + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_STORAGE_MODEL_UPDATE_MERGE_H_ diff --git a/ortools/math_opt/storage/proto_merging_utils.cc b/ortools/math_opt/storage/proto_merging_utils.cc new file mode 100644 index 0000000000..b9d8c335e6 --- /dev/null +++ b/ortools/math_opt/storage/proto_merging_utils.cc @@ -0,0 +1,182 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ortools/math_opt/storage/proto_merging_utils.h" + +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "ortools/math_opt/sparse_containers.pb.h" + +namespace operations_research::math_opt { + +void RemoveDeletedIds(google::protobuf::RepeatedField& ids, + const google::protobuf::RepeatedField& deleted) { + int next_insertion_point = 0; + int deleted_i = 0; + for (const int64_t id : ids) { + while (deleted_i < deleted.size() && deleted[deleted_i] < id) { + ++deleted_i; + } + if (deleted_i < deleted.size() && deleted[deleted_i] == id) { + continue; + } + ids[next_insertion_point] = id; + ++next_insertion_point; + } + ids.Truncate(next_insertion_point); +} + +void MergeIntoSortedIds( + const google::protobuf::RepeatedField& from_new, + google::protobuf::RepeatedField& into_old, + const google::protobuf::RepeatedField& deleted) { + google::protobuf::RepeatedField result; + + int from_new_i = 0; + int into_old_i = 0; + int deleted_i = 0; + + // Functions that adds the input id to the result if it is not in deleted. It + // updates deleted_i as a side effect too. + const auto add_if_not_deleted = [&](const int64_t id) { + while (deleted_i < deleted.size() && deleted[deleted_i] < id) { + ++deleted_i; + } + if (deleted_i == deleted.size() || deleted[deleted_i] != id) { + result.Add(id); + } + }; + + while (from_new_i < from_new.size() && into_old_i < into_old.size()) { + if (from_new[from_new_i] < into_old[into_old_i]) { + add_if_not_deleted(from_new[from_new_i]); + ++from_new_i; + } else if (from_new[from_new_i] > into_old[into_old_i]) { + add_if_not_deleted(into_old[into_old_i]); + ++into_old_i; + } else { // from_new[from_new_i] == into_old[into_old_i] + add_if_not_deleted(from_new[from_new_i]); + ++from_new_i; + ++into_old_i; + } + } + + // At this point either from_new_i == from_new.size() or to_i == to.size() or + // both. And the one that is not empty, if it exists, has elements greater + // than all other elements already inserted. + for (; from_new_i < from_new.size(); ++from_new_i) { + add_if_not_deleted(from_new[from_new_i]); + } + for (; into_old_i < into_old.size(); ++into_old_i) { + add_if_not_deleted(into_old[into_old_i]); + } + + into_old.Swap(&result); +} + +void MergeIntoSparseDoubleMatrix( + const SparseDoubleMatrixProto& from_new, SparseDoubleMatrixProto& into_old, + const google::protobuf::RepeatedField& deleted_rows, + const google::protobuf::RepeatedField& deleted_columns) { + SparseDoubleMatrixProto result; + auto& result_row_ids = *result.mutable_row_ids(); + auto& result_column_ids = *result.mutable_column_ids(); + auto& result_coefficients = *result.mutable_coefficients(); + + // Contrary to rows that are traversed in order (the matrix is using row-major + // order), columns are not. Thus we would have to start the iteration on + // deleted_columns for each new row of the matrix if we wanted to use the same + // approach as with rows. This would be O(num_rows * num_deleted_columns). + // + // Here we use a hash-set to be O(num_matrix_elements + + // num_deleted_columns). The downside is that we consumed + // O(num_deleted_columns) additional memory. + // + // We could have used binary search that would be O(num_matrix_elements * + // lg(num_deleted_columns)) but without additional memory. + const absl::flat_hash_set deleted_columns_set( + deleted_columns.begin(), deleted_columns.end()); + + int from_new_i = 0; + int into_old_i = 0; + int deleted_rows_i = 0; + + // Functions that adds the input tuple (row_id, col_id, coefficient) to the + // result if the input row_id and col_id are not in deleted_rows or + // deleted_columns. It updates deleted_rows_i and deleted_columns_i as a side + // effect too. + const auto add_if_not_deleted = [&](const int64_t row_id, + const int64_t col_id, + const double coefficient) { + while (deleted_rows_i < deleted_rows.size() && + deleted_rows[deleted_rows_i] < row_id) { + ++deleted_rows_i; + } + if ((deleted_rows_i != deleted_rows.size() && + deleted_rows[deleted_rows_i] == row_id) || + deleted_columns_set.contains(col_id)) { + return; + } + result_row_ids.Add(row_id); + result_column_ids.Add(col_id); + result_coefficients.Add(coefficient); + }; + + while (from_new_i < from_new.row_ids_size() && + into_old_i < into_old.row_ids_size()) { + // Matrices are in row-major order and std::pair comparison is + // lexicographical, thus matrices are sorted in the natural order of pairs + // of coordinates (row, col). + const auto from_new_coordinates = std::make_pair( + from_new.row_ids(from_new_i), from_new.column_ids(from_new_i)); + const auto into_old_coordinates = std::make_pair( + into_old.row_ids(into_old_i), into_old.column_ids(into_old_i)); + if (from_new_coordinates < into_old_coordinates) { + add_if_not_deleted(from_new_coordinates.first, + from_new_coordinates.second, + from_new.coefficients(from_new_i)); + ++from_new_i; + } else if (from_new_coordinates > into_old_coordinates) { + add_if_not_deleted(into_old_coordinates.first, + into_old_coordinates.second, + into_old.coefficients(into_old_i)); + ++into_old_i; + } else { // from_new_coordinates == into_old_coordinates + add_if_not_deleted(from_new_coordinates.first, + from_new_coordinates.second, + from_new.coefficients(from_new_i)); + ++from_new_i; + ++into_old_i; + } + } + + // At this point either from_new_i == from_new.row_ids_size() or + // to_i == to.row_ids_size() (or both). And the one that is not empty, if it + // exists, has elements greater than all other elements already inserted. + for (; from_new_i < from_new.row_ids_size(); ++from_new_i) { + add_if_not_deleted(from_new.row_ids(from_new_i), + from_new.column_ids(from_new_i), + from_new.coefficients(from_new_i)); + } + for (; into_old_i < into_old.row_ids_size(); ++into_old_i) { + add_if_not_deleted(into_old.row_ids(into_old_i), + into_old.column_ids(into_old_i), + into_old.coefficients(into_old_i)); + } + + into_old.Swap(&result); +} + +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/core/model_update_merge.h b/ortools/math_opt/storage/proto_merging_utils.h similarity index 85% rename from ortools/math_opt/core/model_update_merge.h rename to ortools/math_opt/storage/proto_merging_utils.h index c5385d079d..ddd4674c18 100644 --- a/ortools/math_opt/core/model_update_merge.h +++ b/ortools/math_opt/storage/proto_merging_utils.h @@ -11,39 +11,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef OR_TOOLS_MATH_OPT_CORE_MODEL_UPDATE_MERGE_H_ -#define OR_TOOLS_MATH_OPT_CORE_MODEL_UPDATE_MERGE_H_ +#ifndef OR_TOOLS_MATH_OPT_STORAGE_PROTO_MERGING_UTILS_H_ +#define OR_TOOLS_MATH_OPT_STORAGE_PROTO_MERGING_UTILS_H_ -#include #include #include "ortools/base/logging.h" #include "ortools/base/protobuf_util.h" #include "ortools/math_opt/core/sparse_vector_view.h" -#include "ortools/math_opt/model_update.pb.h" #include "ortools/math_opt/sparse_containers.pb.h" -namespace operations_research { -namespace math_opt { - -// Merges the `from_new` update into the `into_old` one. -// -// The `from_new` update must represent an update that happens after the -// `into_old` one is applied. Thus when the two updates have overlaps, the -// `from_new` one overrides the value of the `into_old` one (i.e. the `from_new` -// update is expected to be more recent). -// -// This function also CHECKs that the ids of new variables and constraints in -// `from_new` are greater than the ones in `into_old` (as expected if `from_new` -// happens after `into_old`). -// -// Note that the complexity is O(size(from_new) + size(into_old)) thus if you -// need to merge a long list of updates this may be not efficient enough. In -// that case an n-way merge would be needed to be implemented here. -void MergeIntoUpdate(const ModelUpdateProto& from_new, - ModelUpdateProto& into_old); - -namespace internal { +namespace operations_research::math_opt { // Removes from the sorted list `ids` all elements found in the sorted list // `deleted`. The elements should be unique in each sorted list. @@ -107,14 +85,10 @@ void UpdateNewElementProperty( const google::protobuf::RepeatedField& deleted, const SparseVector& updates); -} // namespace internal - //////////////////////////////////////////////////////////////////////////////// // Inline functions implementations. //////////////////////////////////////////////////////////////////////////////// -namespace internal { - template void MergeIntoSparseVector( const SparseVector& from_new, SparseVector& into_old, @@ -207,8 +181,6 @@ void UpdateNewElementProperty( google::protobuf::util::Truncate(&values, next_insertion_point); } -} // namespace internal -} // namespace math_opt -} // namespace operations_research +} // namespace operations_research::math_opt -#endif // OR_TOOLS_MATH_OPT_CORE_MODEL_UPDATE_MERGE_H_ +#endif // OR_TOOLS_MATH_OPT_STORAGE_PROTO_MERGING_UTILS_H_ diff --git a/ortools/math_opt/storage/sparse_matrix.cc b/ortools/math_opt/storage/sparse_matrix.cc new file mode 100644 index 0000000000..30dfec6b46 --- /dev/null +++ b/ortools/math_opt/storage/sparse_matrix.cc @@ -0,0 +1,145 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ortools/math_opt/storage/sparse_matrix.h" + +#include "ortools/base/map_util.h" + +namespace operations_research::math_opt { +namespace { + +// When the fraction of entries in values_ with value 0.0 is larger than +// kZerosCleanup, we compact the data structure and remove all zero entries. +constexpr double kZerosCleanup = 1.0 / 3.0; + +} // namespace + +void SparseSymmetricMatrix::Delete(const VariableId variable) { + auto related_vars = related_variables_.find(variable); + if (related_vars == related_variables_.end()) { + return; + } + for (const VariableId related : related_vars->second) { + auto mat_value = values_.find(make_key(variable, related)); + if (mat_value != values_.end() && mat_value->second != 0.0) { + nonzeros_--; + mat_value->second = 0.0; + } + } + CompactIfNeeded(); +} + +std::vector SparseSymmetricMatrix::RelatedVariables( + const VariableId variable) const { + std::vector result; + if (!related_variables_.contains(variable)) { + return result; + } + for (const VariableId second : related_variables_.at(variable)) { + if (get(variable, second) != 0) { + result.push_back(second); + } + } + return result; +} + +std::vector> SparseSymmetricMatrix::Terms( + const VariableId variable) const { + std::vector> result; + if (!related_variables_.contains(variable)) { + return result; + } + for (const VariableId second : related_variables_.at(variable)) { + double val = get(variable, second); + if (val != 0) { + result.push_back({second, val}); + } + } + return result; +} + +std::vector> +SparseSymmetricMatrix::Terms() const { + std::vector> result; + result.reserve(nonzeros_); + for (const auto& [var_pair, value] : values_) { + if (value != 0.0) { + result.push_back({var_pair.first, var_pair.second, value}); + } + } + return result; +} + +void SparseSymmetricMatrix::CompactIfNeeded() { + const int64_t zeros = values_.size() - nonzeros_; + if (static_cast(zeros) / values_.size() <= kZerosCleanup) { + return; + } + ++compactions_; + for (auto related_var_it = related_variables_.begin(); + related_var_it != related_variables_.end();) { + const VariableId v = related_var_it->first; + std::vector& related = related_var_it->second; + int64_t write = 0; + for (int read = 0; read < related.size(); ++read) { + auto val = values_.find(make_key(v, related[read])); + if (val != values_.end()) { + if (val->second != 0) { + related[write] = related[read]; + ++write; + } else { + values_.erase(val); + } + } + } + if (write == 0) { + related_variables_.erase(related_var_it++); + } else { + related.resize(write); + ++related_var_it; + } + } +} + +void SparseSymmetricMatrix::Clear() { + related_variables_.clear(); + values_.clear(); + nonzeros_ = 0; +} + +SparseDoubleMatrixProto SparseSymmetricMatrix::Proto() const { + SparseDoubleMatrixProto result; + + std::vector vars_in_order; + for (const auto& [v, _] : related_variables_) { + vars_in_order.push_back(v); + } + absl::c_sort(vars_in_order); + + for (const VariableId v : vars_in_order) { + // TODO(b/233630053): reuse the allocation once an iterator API is + // supported. + std::vector> related = Terms(v); + absl::c_sort(related); + for (const auto [other, coef] : related) { + if (v <= other) { + result.add_row_ids(v.value()); + result.add_column_ids(other.value()); + result.add_coefficients(coef); + } + } + } + return result; +} + +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/storage/sparse_matrix.h b/ortools/math_opt/storage/sparse_matrix.h new file mode 100644 index 0000000000..22edf20c54 --- /dev/null +++ b/ortools/math_opt/storage/sparse_matrix.h @@ -0,0 +1,217 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Classes for modeling sparse matrices. +#ifndef OR_TOOLS_MATH_OPT_STORAGE_SPARSE_MATRIX_H_ +#define OR_TOOLS_MATH_OPT_STORAGE_SPARSE_MATRIX_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "ortools/base/map_util.h" +#include "ortools/base/strong_int.h" +#include "ortools/base/strong_vector.h" +#include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/storage/model_storage_types.h" + +namespace operations_research::math_opt { + +// A sparse symmetric double valued matrix over VariableIds. +// +// Note that the matrix is sparse in both: +// * The IDs of the rows/columns (both VariableIds), stored as flat_hash_map. +// * The entries with nonzero value. +// +// Getting/setting/clearing entries are O(1) operations. Getting a row of the +// matrix runs in O(size of the row) if nothing has been deleted, and getting +// all the rows runs in O(number of nonzero entries), even with deletions +// (with deletions, accessing a particular row with many deletions may be slow). +// +// Implementation: The entries are stored in a +// flat_hash_map, double> `values_` where for each +// key, key.first <= key.second. Additionally, we maintain a +// flat_hash_map> `related_variables_` that says +// for each variable, which variables they have a nonzero term with. When a +// coefficient is set to zero or a variable is deleted, we do not immediately +// delete the data from values_ or related_variables_, we simply set the +// coefficient to zero in values_. We track how many zeros are in values_, and +// when more than some constant fraction of all entries are zero (see +// kZerosCleanup in cc file), we clean up related_variables_ and values_ to +// remove all the zeros. Iteration over the rows or total entries of the matrix +// must check for zeros in values_ and skip these terms. +// +// Memory use: +// * 3*8 bytes per nonzero plus hash capacity overhead for values_. +// * 2*8 bytes per nonzero plus vector capacity overhead for +// related_variables_. +// * ~5*8 bytes per variable participating in any quadratic term; one heap +// allocation per such variable. +class SparseSymmetricMatrix { + public: + // Setting `value` to zero removes the value from the matrix. + // Returns true if `value` is different from the existing value in the matrix. + inline bool set(VariableId first, VariableId second, double value); + + // Zero is returned if the value is not present. + inline double get(VariableId first, VariableId second) const; + + // Zeros out all coefficients for this variable. + void Delete(VariableId variable); + + // Returns the variables that have nonzero entries with `variable`. + // + // The return order is deterministic but not defined. + // TODO(b/233630053): expose an iterator based API to avoid making a copy. + std::vector RelatedVariables(VariableId variable) const; + + // Returns the variable value pairs (x, c) where `variable` and x have nonzero + // coefficient c. + // + // The return order is deterministic but not defined. + // TODO(b/233630053): expose an iterator based API to avoid making a copy. + std::vector> Terms(VariableId variable) const; + + // Returns (x, y, c) tuples where variables x and y have nonzero coefficient + // c, and x <= y. + // + // The return order is non-deterministic and not defined. + // TODO(b/233630053): expose an iterator based API to avoid making a copy. + std::vector> Terms() const; + + // Removes all terms from the matrix. + void Clear(); + + // The number of (var, var) keys with nonzero value. Note that (x, y) and + // (y, x) are the same key. + int64_t nonzeros() const { return nonzeros_; } + + // Visible for testing, do not depend on this. + int64_t compactions() const { return compactions_; } + + // TODO(b/233630053): do not expose values_ directly, instead offer a way to + // iterate over all the nonzero entries. + // Warning: this map will contain zeros. + const absl::flat_hash_map, double>& values() + const { + return values_; + } + + SparseDoubleMatrixProto Proto() const; + + template + inline SparseDoubleMatrixProto Update( + const VarContainer& variables, VariableId checkpoint, VariableId next_var, + const absl::flat_hash_set>& dirty) + const; + + private: + inline std::pair make_key(VariableId first, + VariableId second) const; + void CompactIfNeeded(); + + // The keys of values_ have key.first <= key.second. + absl::flat_hash_map, double> values_; + absl::flat_hash_map> related_variables_; + + // The number of nonzero elements in values_. + int64_t nonzeros_ = 0; + int64_t compactions_ = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// +// Inlined functions +//////////////////////////////////////////////////////////////////////////////// + +std::pair SparseSymmetricMatrix::make_key( + const VariableId first, const VariableId second) const { + return {std::min(first, second), std::max(first, second)}; +} + +bool SparseSymmetricMatrix::set(const VariableId first, const VariableId second, + const double value) { + const std::pair key = make_key(first, second); + auto map_iter = values_.find(key); + + if (map_iter == values_.end()) { + if (value == 0.0) { + return false; + } + related_variables_[first].push_back(second); + if (first != second) { + related_variables_[second].push_back(first); + } + values_[key] = value; + nonzeros_++; + return true; + } else { + if (map_iter->second == value) { + return false; + } + const double old_value = map_iter->second; + map_iter->second = value; + if (value == 0.0) { + nonzeros_--; + CompactIfNeeded(); + } else if (old_value == 0.0) { + nonzeros_++; + } + return true; + } +} + +double SparseSymmetricMatrix::get(const VariableId first, + const VariableId second) const { + return gtl::FindWithDefault(values_, make_key(first, second)); +} + +template +SparseDoubleMatrixProto SparseSymmetricMatrix::Update( + const VarContainer& variables, const VariableId checkpoint, + const VariableId next_var, + const absl::flat_hash_set>& dirty) const { + std::vector> updates; + for (const std::pair pair : dirty) { + // If either variable has been deleted, don't add it. + if (variables.contains(pair.first) && variables.contains(pair.second)) { + updates.push_back(pair); + } + } + + for (const VariableId v : MakeStrongIntRange(checkpoint, next_var)) { + if (related_variables_.contains(v)) { + // TODO(b/233630053): do not allocate here. + for (const VariableId other : RelatedVariables(v)) { + if (v <= other) { + updates.push_back({v, other}); + } else if (other < checkpoint) { + updates.push_back({other, v}); + } + } + } + } + absl::c_sort(updates); + SparseDoubleMatrixProto result; + for (const auto [row, col] : updates) { + result.add_row_ids(row.value()); + result.add_column_ids(col.value()); + result.add_coefficients(get(row, col)); + } + return result; +} + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_STORAGE_SPARSE_MATRIX_H_ diff --git a/ortools/math_opt/validators/BUILD.bazel b/ortools/math_opt/validators/BUILD.bazel index bff122f746..d7fda25a50 100644 --- a/ortools/math_opt/validators/BUILD.bazel +++ b/ortools/math_opt/validators/BUILD.bazel @@ -67,6 +67,7 @@ cc_library( "//ortools/math_opt:model_cc_proto", "//ortools/math_opt:model_update_cc_proto", "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/constraints/sos:validator", "//ortools/math_opt/core:model_summary", "//ortools/math_opt/core:sparse_vector_view", "@com_google_absl//absl/status", @@ -189,3 +190,17 @@ cc_library( "@com_google_absl//absl/strings", ], ) + +cc_library( + name = "linear_expression_validator", + srcs = ["linear_expression_validator.cc"], + hdrs = ["linear_expression_validator.h"], + deps = [ + ":scalar_validator", + ":sparse_vector_validator", + "//ortools/math_opt:sparse_containers_cc_proto", + "//ortools/math_opt/core:model_summary", + "//ortools/math_opt/core:sparse_vector_view", + "@com_google_absl//absl/status", + ], +) diff --git a/ortools/math_opt/validators/linear_expression_validator.cc b/ortools/math_opt/validators/linear_expression_validator.cc new file mode 100644 index 0000000000..dbf660903c --- /dev/null +++ b/ortools/math_opt/validators/linear_expression_validator.cc @@ -0,0 +1,45 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ortools/math_opt/validators/linear_expression_validator.h" + +#include + +#include "absl/status/status.h" +#include "ortools/base/status_macros.h" +#include "ortools/math_opt/core/model_summary.h" +#include "ortools/math_opt/core/sparse_vector_view.h" +#include "ortools/math_opt/sparse_containers.pb.h" +#include "ortools/math_opt/validators/scalar_validator.h" +#include "ortools/math_opt/validators/sparse_vector_validator.h" + +namespace operations_research::math_opt { + +absl::Status ValidateLinearExpression(const LinearExpressionProto& expression, + const IdNameBiMap& variable_universe) { + RETURN_IF_ERROR(CheckIdsAndValues( + MakeView(expression.ids(), expression.coefficients()), + {.allow_positive_infinity = false, .allow_negative_infinity = false})) + << "invalid linear expression terms"; + for (const int64_t var_id : expression.ids()) { + if (!variable_universe.HasId(var_id)) { + return util::InvalidArgumentErrorBuilder() + << "invalid variable id: " << var_id; + } + } + RETURN_IF_ERROR(CheckScalarNoNanNoInf(expression.offset())) + << "invalid linear expression offset"; + return absl::OkStatus(); +} + +} // namespace operations_research::math_opt diff --git a/ortools/math_opt/validators/linear_expression_validator.h b/ortools/math_opt/validators/linear_expression_validator.h new file mode 100644 index 0000000000..837a6e1276 --- /dev/null +++ b/ortools/math_opt/validators/linear_expression_validator.h @@ -0,0 +1,28 @@ +// Copyright 2010-2021 Google LLC +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OR_TOOLS_MATH_OPT_VALIDATORS_LINEAR_EXPRESSION_VALIDATOR_H_ +#define OR_TOOLS_MATH_OPT_VALIDATORS_LINEAR_EXPRESSION_VALIDATOR_H_ + +#include "absl/status/status.h" +#include "ortools/math_opt/core/model_summary.h" +#include "ortools/math_opt/sparse_containers.pb.h" + +namespace operations_research::math_opt { + +absl::Status ValidateLinearExpression(const LinearExpressionProto& expression, + const IdNameBiMap& variable_universe); + +} // namespace operations_research::math_opt + +#endif // OR_TOOLS_MATH_OPT_VALIDATORS_LINEAR_EXPRESSION_VALIDATOR_H_ diff --git a/ortools/math_opt/validators/model_validator.cc b/ortools/math_opt/validators/model_validator.cc index d7541c2efc..7edf2116c8 100644 --- a/ortools/math_opt/validators/model_validator.cc +++ b/ortools/math_opt/validators/model_validator.cc @@ -13,15 +13,13 @@ #include "ortools/math_opt/validators/model_validator.h" -#include #include -#include -#include +#include #include "absl/status/status.h" -#include "absl/strings/str_cat.h" -#include "absl/types/span.h" +#include "absl/status/statusor.h" #include "ortools/base/status_macros.h" +#include "ortools/math_opt/constraints/sos/validator.h" #include "ortools/math_opt/core/model_summary.h" #include "ortools/math_opt/core/sparse_vector_view.h" #include "ortools/math_opt/model.pb.h" @@ -168,6 +166,20 @@ absl::Status LinearConstraintMatrixIdsValidForUpdate( return absl::OkStatus(); } +// To use this helper, you must implement an overload for: +// ValidateConstraint(const MyConstraintProto& constraint, +// const IdNameBiMap& variable_universe); +template +absl::Status ValidateConstraintMap( + const google::protobuf::Map& constraints, + const IdNameBiMap& variable_universe) { + for (const auto& [id, constraint] : constraints) { + RETURN_IF_ERROR(ValidateConstraint(constraint, variable_universe)) + << "invalid constraint with id: " << id; + } + return absl::OkStatus(); +} + } // namespace // ///////////////////////////////////////////////////////////////////////////// @@ -190,6 +202,14 @@ absl::StatusOr ValidateModel(const ModelProto& model, model_summary.linear_constraints, model_summary.variables)) << "Model.linear_constraint_matrix ids are inconsistent"; + + RETURN_IF_ERROR( + ValidateConstraintMap(model.sos1_constraints(), model_summary.variables)) + << "Model.sos1_constraints invalid"; + RETURN_IF_ERROR( + ValidateConstraintMap(model.sos2_constraints(), model_summary.variables)) + << "Model.sos2_constraints invalid"; + return model_summary; } @@ -230,6 +250,15 @@ absl::Status ValidateModelUpdate(const ModelUpdateProto& model_update, model_summary.linear_constraints, model_summary.variables)) << "invalid linear constraint matrix update"; + RETURN_IF_ERROR(ValidateConstraintMap( + model_update.sos1_constraint_updates().new_constraints(), + model_summary.variables)) + << "ModelUpdateProto.sos1_constraint_updates.new_constraints invalid"; + RETURN_IF_ERROR(ValidateConstraintMap( + model_update.sos2_constraint_updates().new_constraints(), + model_summary.variables)) + << "ModelUpdateProto.sos2_constraint_updates.new_constraints invalid"; + return absl::OkStatus(); } diff --git a/ortools/math_opt/validators/model_validator.h b/ortools/math_opt/validators/model_validator.h index eb481d0eb3..e48df8d5ec 100644 --- a/ortools/math_opt/validators/model_validator.h +++ b/ortools/math_opt/validators/model_validator.h @@ -15,6 +15,7 @@ #define OR_TOOLS_MATH_OPT_VALIDATORS_MODEL_VALIDATOR_H_ #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "ortools/math_opt/core/model_summary.h" #include "ortools/math_opt/model.pb.h" #include "ortools/math_opt/model_update.pb.h"