diff --git a/.bazelrc b/.bazelrc index 76afa1ad32..fa0ca7a2f0 100644 --- a/.bazelrc +++ b/.bazelrc @@ -81,11 +81,11 @@ build:ci --keep_going # Show test errors. build:ci --test_output=errors -# Override timeout for tests -build:ci --test_timeout_filters=-eternal +# Skip tests that are too large to run on CI. +build:ci --test_size_filters=small,-medium,-large,-enormous -# Only show failing tests to reduce output -build:ci --test_summary=terse +# Print information only about tests executed +build:ci --test_summary=short # Attempt to work around intermittent issue while trying to fetch remote blob. # See e.g. https://github.com/bazelbuild/bazel/issues/18694. diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f30547c7d..7f34130af3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,6 +200,7 @@ if(BUILD_TESTING) "NOT BUILD_DEPS" ON) CMAKE_DEPENDENT_OPTION(BUILD_benchmark "Build benchmark" OFF "NOT BUILD_DEPS" ON) + set(BUILD_protobuf_matchers ON) # Fuzztest do not support MSVC or toolchain if(APPLE OR MSVC OR CMAKE_CROSSCOMPILING) set(USE_fuzztest OFF) @@ -215,11 +216,13 @@ if(BUILD_TESTING) endif() else() set(BUILD_googletest OFF) + set(BUILD_protobuf_matchers OFF) set(BUILD_benchmark OFF) set(USE_fuzztest OFF) set(BUILD_fuzztest OFF) endif() message(STATUS "Build googletest: ${BUILD_googletest}") +message(STATUS "Build protobuf_matchers: ${BUILD_protobuf_matchers}") message(STATUS "Build benchmark: ${BUILD_benchmark}") message(STATUS "Enable fuzztest: ${USE_fuzztest}") message(STATUS "Build fuzztest: ${BUILD_fuzztest}") diff --git a/MODULE.bazel b/MODULE.bazel index 6899e7cb2a..5f37bfb477 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -46,6 +46,7 @@ bazel_dep(name = "fuzztest", version = "20250805.0") bazel_dep(name = "glpk", version = "5.0.bcr.4") bazel_dep(name = "google_benchmark", version = "1.9.2") bazel_dep(name = "googletest", version = "1.17.0") +bazel_dep(name = "protobuf-matchers", version = "0.1.1") bazel_dep(name = "highs", version = "1.11.0") bazel_dep(name = "protobuf", version = "32.0") bazel_dep(name = "re2", version = "2025-08-12") diff --git a/cmake/cpp.cmake b/cmake/cpp.cmake index 00055926f8..96268ae019 100644 --- a/cmake/cpp.cmake +++ b/cmake/cpp.cmake @@ -231,7 +231,7 @@ endfunction() # Parameters: # NAME: CMake target name # SOURCES: List of source files -# [TYPE]: SHARED or STATIC +# [TYPE]: SHARED, STATIC or INTERFACE # [COMPILE_DEFINITIONS]: List of private compile definitions # [COMPILE_OPTIONS]: List of private compile options # [LINK_LIBRARIES]: List of **public** libraries to use when linking @@ -275,16 +275,18 @@ function(ortools_cxx_library) message(STATUS "Configuring library ${LIBRARY_NAME} ...") add_library(${LIBRARY_NAME} ${LIBRARY_TYPE} "") - target_sources(${LIBRARY_NAME} PRIVATE ${LIBRARY_SOURCES}) - target_include_directories(${LIBRARY_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) - target_compile_definitions(${LIBRARY_NAME} PRIVATE ${LIBRARY_COMPILE_DEFINITIONS}) - target_compile_features(${LIBRARY_NAME} PRIVATE cxx_std_17) - target_compile_options(${LIBRARY_NAME} PRIVATE ${LIBRARY_COMPILE_OPTIONS}) - target_link_libraries(${LIBRARY_NAME} PUBLIC - ${PROJECT_NAMESPACE}::ortools - ${LIBRARY_LINK_LIBRARIES} - ) - target_link_options(${LIBRARY_NAME} PRIVATE ${LIBRARY_LINK_OPTIONS}) + if(LIBRARY_TYPE STREQUAL "INTERFACE") + target_include_directories(${LIBRARY_NAME} INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${LIBRARY_NAME} INTERFACE ${PROJECT_NAMESPACE}::ortools ${LIBRARY_LINK_LIBRARIES}) + else() + target_sources(${LIBRARY_NAME} PRIVATE ${LIBRARY_SOURCES}) + target_include_directories(${LIBRARY_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_definitions(${LIBRARY_NAME} PRIVATE ${LIBRARY_COMPILE_DEFINITIONS}) + target_compile_features(${LIBRARY_NAME} PRIVATE cxx_std_17) + target_compile_options(${LIBRARY_NAME} PRIVATE ${LIBRARY_COMPILE_OPTIONS}) + target_link_libraries(${LIBRARY_NAME} PUBLIC ${PROJECT_NAMESPACE}::ortools ${LIBRARY_LINK_LIBRARIES}) + target_link_options(${LIBRARY_NAME} PRIVATE ${LIBRARY_LINK_OPTIONS}) + endif() include(GNUInstallDirs) if(APPLE) diff --git a/cmake/dependencies/CMakeLists.txt b/cmake/dependencies/CMakeLists.txt index d14cd5342a..e40906beb7 100644 --- a/cmake/dependencies/CMakeLists.txt +++ b/cmake/dependencies/CMakeLists.txt @@ -152,6 +152,7 @@ if(BUILD_Protobuf) UPDATE_COMMAND git reset --hard PATCH_COMMAND git apply --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/../../patches/protobuf-v32.0.patch" + OVERRIDE_FIND_PACKAGE # Make package visible for "protobuf-matchers" below ) FetchContent_MakeAvailable(Protobuf) list(POP_BACK CMAKE_MESSAGE_INDENT) @@ -537,6 +538,23 @@ if(BUILD_googletest) message(CHECK_PASS "fetched") endif() +if(BUILD_protobuf_matchers) + message(CHECK_START "Fetching protobuf-matchers") + list(APPEND CMAKE_MESSAGE_INDENT " ") + FetchContent_Declare( + protobuf-matchers + GIT_REPOSITORY https://github.com/inazarenko/protobuf-matchers.git + GIT_TAG v0.1.1 + GIT_SHALLOW TRUE + UPDATE_COMMAND git reset --hard + PATCH_COMMAND git apply --ignore-whitespace + "${CMAKE_CURRENT_LIST_DIR}/../../patches/protobuf-matchers-v0.1.1.patch" + ) + FetchContent_MakeAvailable(protobuf-matchers) + list(POP_BACK CMAKE_MESSAGE_INDENT) + message(CHECK_PASS "fetched") +endif() + if(BUILD_benchmark) message(CHECK_START "Fetching benchmark") list(APPEND CMAKE_MESSAGE_INDENT " ") diff --git a/ortools/algorithms/BUILD.bazel b/ortools/algorithms/BUILD.bazel index 2f2d2cb9c7..ff0f1ffd1f 100644 --- a/ortools/algorithms/BUILD.bazel +++ b/ortools/algorithms/BUILD.bazel @@ -93,6 +93,7 @@ cc_library( cc_test( name = "radix_sort_test", + size = "medium", srcs = ["radix_sort_test.cc"], copts = select({ "@platforms//os:windows": ["/Zc:preprocessor"], diff --git a/ortools/algorithms/CMakeLists.txt b/ortools/algorithms/CMakeLists.txt index 4d94166a58..da84d96cc8 100644 --- a/ortools/algorithms/CMakeLists.txt +++ b/ortools/algorithms/CMakeLists.txt @@ -65,4 +65,9 @@ if(BUILD_TESTING) GTest::gmock ) endif() + + # These tests are too long so we disable them. + set_tests_properties( + cxx_algorithms_radix_sort_test + PROPERTIES DISABLED TRUE) endif() diff --git a/ortools/algorithms/java/CMakeLists.txt b/ortools/algorithms/java/CMakeLists.txt index 82c9e26691..29d7865e8b 100644 --- a/ortools/algorithms/java/CMakeLists.txt +++ b/ortools/algorithms/java/CMakeLists.txt @@ -34,4 +34,9 @@ if(BUILD_TESTING) foreach(FILE_NAME IN LISTS JAVA_SRCS) add_java_test(FILE_NAME ${FILE_NAME}) endforeach() + + # These tests are too long so we disable them. + set_tests_properties( + java_algorithms_KnapsackSolverTest + PROPERTIES DISABLED TRUE) endif() diff --git a/ortools/base/BUILD.bazel b/ortools/base/BUILD.bazel index fa1d98a29d..1e1b632e87 100644 --- a/ortools/base/BUILD.bazel +++ b/ortools/base/BUILD.bazel @@ -182,9 +182,9 @@ cc_library( testonly = True, hdrs = ["gmock.h"], deps = [ - ":protocol-buffer-matchers", "@abseil-cpp//absl/status:status_matchers", "@googletest//:gtest", + "@protobuf-matchers//protobuf-matchers", ], ) @@ -373,19 +373,6 @@ cc_library( ], ) -cc_library( - name = "protocol-buffer-matchers", - testonly = True, - srcs = ["protocol-buffer-matchers.cc"], - hdrs = ["protocol-buffer-matchers.h"], - deps = [ - "@abseil-cpp//absl/log:check", - "@abseil-cpp//absl/strings", - "@googletest//:gtest", - "@protobuf", - ], -) - cc_library( name = "protoutil", hdrs = ["protoutil.h"], diff --git a/ortools/base/CMakeLists.txt b/ortools/base/CMakeLists.txt index c58c6abfff..b554d18b43 100644 --- a/ortools/base/CMakeLists.txt +++ b/ortools/base/CMakeLists.txt @@ -14,7 +14,6 @@ file(GLOB _SRCS "*.h" "*.cc") list(FILTER _SRCS EXCLUDE REGEX "/.*_test.cc") list(FILTER _SRCS EXCLUDE REGEX "/gmock\.h") -list(FILTER _SRCS EXCLUDE REGEX "/protocol-buffer-matchers\..*") set(NAME ${PROJECT_NAME}_base) @@ -51,10 +50,8 @@ ortools_cxx_library( base_gmock SOURCES "gmock.h" - "protocol-buffer-matchers.cc" - "protocol-buffer-matchers.h" TYPE - STATIC + INTERFACE LINK_LIBRARIES absl::log absl::strings @@ -62,6 +59,7 @@ ortools_cxx_library( GTest::gtest GTest::gmock protobuf::libprotobuf + protobuf-matchers TESTING ) diff --git a/ortools/base/gmock.h b/ortools/base/gmock.h index 0720afc6df..0005f17ed8 100644 --- a/ortools/base/gmock.h +++ b/ortools/base/gmock.h @@ -16,7 +16,23 @@ #include "absl/status/status_matchers.h" #include "ortools/base/gmock.h" -#include "ortools/base/protocol-buffer-matchers.h" // IWYU pragma: export +#include "protobuf-matchers/protocol-buffer-matchers.h" + +namespace testing { +using ::protobuf_matchers::EqualsProto; +using ::protobuf_matchers::EquivToProto; +namespace proto { +using ::protobuf_matchers::proto::Approximately; +using ::protobuf_matchers::proto::IgnoringFieldPaths; +using ::protobuf_matchers::proto::IgnoringFields; +using ::protobuf_matchers::proto::IgnoringRepeatedFieldOrdering; +using ::protobuf_matchers::proto::Partially; +using ::protobuf_matchers::proto::TreatingNaNsAsEqual; +using ::protobuf_matchers::proto::WhenDeserialized; +using ::protobuf_matchers::proto::WhenDeserializedAs; +using ::protobuf_matchers::proto::WithDifferencerConfig; +} // namespace proto +} // namespace testing namespace testing::status { using ::absl_testing::IsOk; diff --git a/ortools/base/protocol-buffer-matchers.cc b/ortools/base/protocol-buffer-matchers.cc deleted file mode 100644 index bba51ab653..0000000000 --- a/ortools/base/protocol-buffer-matchers.cc +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2010-2025 Google LLC -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// emulates g3/testing/base/public/gmock_utils/protocol-buffer-matchers.cc -#include "ortools/base/protocol-buffer-matchers.h" - -#include - -#include "absl/log/check.h" -#include "absl/log/log.h" -#include "absl/strings/string_view.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/io/tokenizer.h" -#include "google/protobuf/message.h" -#include "google/protobuf/text_format.h" -#include "google/protobuf/util/message_differencer.h" - -namespace testing { -namespace internal { -// Utilities. -class StringErrorCollector : public ::google::protobuf::io::ErrorCollector { - public: - explicit StringErrorCollector(std::string* error_text) - : error_text_(error_text) {} - - void RecordError(int line, int column, absl::string_view message) override { - std::ostringstream stream; - stream << line << '(' << column << "): " << message << std::endl; - *error_text_ += stream.str(); - } - - void RecordWarning(int line, int column, absl::string_view message) override { - std::ostringstream stream; - stream << line << '(' << column << "): " << message << std::endl; - *error_text_ += stream.str(); - } - - private: - std::string* error_text_; - StringErrorCollector(const StringErrorCollector&) = delete; - StringErrorCollector& operator=(const StringErrorCollector&) = delete; -}; - -bool ParsePartialFromAscii(const std::string& pb_ascii, - ::google::protobuf::Message* proto, - std::string* error_text) { - ::google::protobuf::TextFormat::Parser parser; - StringErrorCollector collector(error_text); - parser.RecordErrorsTo(&collector); - parser.AllowPartialMessage(true); - return parser.ParseFromString(pb_ascii, proto); -} - -// Returns true iff p and q can be compared (i.e. have the same descriptor). -bool ProtoComparable(const ::google::protobuf::Message& p, - const ::google::protobuf::Message& q) { - return p.GetDescriptor() == q.GetDescriptor(); -} - -template -std::string JoinStringPieces(const Container& strings, - absl::string_view separator) { - std::stringstream stream; - absl::string_view sep = ""; - for (const absl::string_view str : strings) { - stream << sep << str; - sep = separator; - } - return stream.str(); -} - -// Find all the descriptors for the ingore_fields. -std::vector GetFieldDescriptors( - const ::google::protobuf::Descriptor* proto_descriptor, - const std::vector& ignore_fields) { - std::vector ignore_descriptors; - std::vector remaining_descriptors; - - const ::google::protobuf::DescriptorPool* pool = - proto_descriptor->file()->pool(); - for (const std::string& name : ignore_fields) { - if (const ::google::protobuf::FieldDescriptor* field = - pool->FindFieldByName(name)) { - ignore_descriptors.push_back(field); - } else { - remaining_descriptors.push_back(name); - } - } - - CHECK(remaining_descriptors.empty()) - << "Could not find fields for proto " << proto_descriptor->full_name() - << " with fully qualified names: " - << JoinStringPieces(remaining_descriptors, ","); - return ignore_descriptors; -} - -// Sets the ignored fields corresponding to ignore_fields in differencer. Dies -// if any is invalid. -void SetIgnoredFieldsOrDie( - const ::google::protobuf::Descriptor& root_descriptor, - const std::vector& ignore_fields, - ::google::protobuf::util::MessageDifferencer* differencer) { - if (!ignore_fields.empty()) { - std::vector ignore_descriptors = - GetFieldDescriptors(&root_descriptor, ignore_fields); - for (std::vector::iterator it = - ignore_descriptors.begin(); - it != ignore_descriptors.end(); ++it) { - differencer->IgnoreField(*it); - } - } -} - -// A criterion that ignores a field path. -class IgnoreFieldPathCriteria - : public ::google::protobuf::util::MessageDifferencer::IgnoreCriteria { - public: - explicit IgnoreFieldPathCriteria( - const std::vector< - ::google::protobuf::util::MessageDifferencer::SpecificField>& - field_path) - : ignored_field_path_(field_path) {} - - bool IsIgnored( - const ::google::protobuf::Message& message1, - const ::google::protobuf::Message& message2, - const ::google::protobuf::FieldDescriptor* field, - const std::vector< - ::google::protobuf::util::MessageDifferencer::SpecificField>& - parent_fields) override { - // The off by one is for the current field. - if (parent_fields.size() + 1 != ignored_field_path_.size()) { - return false; - } - for (size_t i = 0; i < parent_fields.size(); ++i) { - const auto& cur_field = parent_fields[i]; - const auto& ignored_field = ignored_field_path_[i]; - // We could compare pointers but it's not guaranteed that descriptors come - // from the same pool. - if (cur_field.field->full_name() != ignored_field.field->full_name()) { - return false; - } - - // repeated_field[i] is ignored if repeated_field is ignored. To put it - // differently: if ignored_field specifies an index, we ignore only a - // field with the same index. - if (ignored_field.index != -1 && ignored_field.index != cur_field.index) { - return false; - } - } - return field->full_name() == ignored_field_path_.back().field->full_name(); - } - - private: - const std::vector - ignored_field_path_; -}; - -// Parses a field path and returns individual components. -std::vector -ParseFieldPathOrDie(const std::string& relative_field_path, - const ::google::protobuf::Descriptor& root_descriptor) { - std::vector - field_path; - - // We're parsing a dot-separated list of elements that can be either: - // - field names - // - extension names - // - indexed field names - // The parser is very permissive as to what is a field name, then we check - // the field name against the descriptor. - - // Regular parsers. Consume() does not handle optional captures so we split it - // in two regexps. - const std::regex field_regex(R"(([^.()[\]]+))"); - const std::regex field_subscript_regex(R"(([^.()[\]]+)\[(\d+)\])"); - const std::regex extension_regex(R"(\(([^)]+)\))"); - - const auto begin = std::begin(relative_field_path); - auto it = begin; - const auto end = std::end(relative_field_path); - while (it != end) { - // Consume a dot, except on the first iteration. - if (it != std::begin(relative_field_path) && *(it++) != '.') { - LOG(FATAL) << "Cannot parse field path '" << relative_field_path - << "' at offset " << std::distance(begin, it) - << ": expected '.'"; - } - // Try to consume a field name. If that fails, consume an extension name. - ::google::protobuf::util::MessageDifferencer::SpecificField field; - std::smatch match_results; - if (std::regex_search(it, end, match_results, field_subscript_regex) || - std::regex_search(it, end, match_results, field_regex)) { - std::string name = match_results[1].str(); - if (field_path.empty()) { - field.field = root_descriptor.FindFieldByName(name); - CHECK(field.field) << "No such field '" << name << "' in message '" - << root_descriptor.full_name() << "'"; - } else { - const ::google::protobuf::util::MessageDifferencer::SpecificField& - parent = field_path.back(); - field.field = parent.field->message_type()->FindFieldByName(name); - CHECK(field.field) << "No such field '" << name << "' in '" - << parent.field->full_name() << "'"; - } - if (match_results.size() > 2 && match_results[2].matched) { - std::string number = match_results[2].str(); - field.index = std::stoi(number); - } - - } else if (std::regex_search(it, end, match_results, extension_regex)) { - std::string name = match_results[1].str(); - field.field = ::google::protobuf::DescriptorPool::generated_pool() - ->FindExtensionByName(name); - CHECK(field.field) << "No such extension '" << name << "'"; - if (field_path.empty()) { - CHECK(root_descriptor.IsExtensionNumber(field.field->number())) - << "Extension '" << name << "' does not extend message '" - << root_descriptor.full_name() << "'"; - } else { - const ::google::protobuf::util::MessageDifferencer::SpecificField& - parent = field_path.back(); - CHECK(parent.field->message_type()->IsExtensionNumber( - field.field->number())) - << "Extension '" << name << "' does not extend '" - << parent.field->full_name() << "'"; - } - } else { - LOG(FATAL) << "Cannot parse field path '" << relative_field_path - << "' at offset " << std::distance(begin, it) - << ": expected field or extension"; - } - auto consume = match_results[0].length(); - it += consume; - field_path.push_back(field); - } - - CHECK(!field_path.empty()); - CHECK(field_path.back().index == -1) - << "Terminally ignoring fields by index is currently not supported ('" - << relative_field_path << "')"; - - return field_path; -} - -// Sets the ignored field paths corresponding to field_paths in differencer. -// Dies if any path is invalid. -void SetIgnoredFieldPathsOrDie( - const ::google::protobuf::Descriptor& root_descriptor, - const std::vector& field_paths, - ::google::protobuf::util::MessageDifferencer* differencer) { - for (const std::string& field_path : field_paths) { - differencer->AddIgnoreCriteria(new IgnoreFieldPathCriteria( - ParseFieldPathOrDie(field_path, root_descriptor))); - } -} - -// Configures a MessageDifferencer and DefaultFieldComparator to use the logic -// described in comp. The configured differencer is the output of this function, -// but a FieldComparator must be provided to keep ownership clear. -void ConfigureDifferencer( - const ProtoComparison& comp, - ::google::protobuf::util::DefaultFieldComparator* comparator, - ::google::protobuf::util::MessageDifferencer* differencer, - const ::google::protobuf::Descriptor* descriptor) { - differencer->set_message_field_comparison(comp.field_comp); - differencer->set_scope(comp.scope); - comparator->set_float_comparison(comp.float_comp); - comparator->set_treat_nan_as_equal(comp.treating_nan_as_equal); - differencer->set_repeated_field_comparison(comp.repeated_field_comp); - SetIgnoredFieldsOrDie(*descriptor, comp.ignore_fields, differencer); - SetIgnoredFieldPathsOrDie(*descriptor, comp.ignore_field_paths, differencer); - if (comp.float_comp == kProtoApproximate && - (comp.has_custom_margin || comp.has_custom_fraction)) { - // Two fields will be considered equal if they're within the fraction _or_ - // within the margin. So setting the fraction to 0.0 makes this effectively - // a "SetMargin". Similarly, setting the margin to 0.0 makes this - // effectively a "SetFraction". - comparator->SetDefaultFractionAndMargin(comp.float_fraction, - comp.float_margin); - } - differencer->set_field_comparator(comparator); - if (comp.differencer_config_function) { - comp.differencer_config_function(comparator, differencer); - } -} - -// Returns true iff actual and expected are comparable and match. The -// comp argument specifies how the two are compared. -bool ProtoCompare(const ProtoComparison& comp, - const ::google::protobuf::Message& actual, - const ::google::protobuf::Message& expected) { - if (!ProtoComparable(actual, expected)) return false; - - ::google::protobuf::util::MessageDifferencer differencer; - ::google::protobuf::util::DefaultFieldComparator field_comparator; - ConfigureDifferencer(comp, &field_comparator, &differencer, - actual.GetDescriptor()); - - // It's important for 'expected' to be the first argument here, as - // Compare() is not symmetric. When we do a partial comparison, - // only fields present in the first argument of Compare() are - // considered. - return differencer.Compare(expected, actual); -} - -// Describes the types of the expected and the actual protocol buffer. -std::string DescribeTypes(const ::google::protobuf::Message& expected, - const ::google::protobuf::Message& actual) { - std::ostringstream s; - s << "whose type should be " << expected.GetDescriptor()->full_name() - << " but actually is " << actual.GetDescriptor()->full_name(); - return s.str(); -} - -// Prints the protocol buffer pointed to by proto. -std::string PrintProtoPointee(const ::google::protobuf::Message* proto) { - if (proto == nullptr) return ""; - return "which points to " + ::testing::PrintToString(*proto); -} - -// Describes the differences between the two protocol buffers. -std::string DescribeDiff(const ProtoComparison& comp, - const ::google::protobuf::Message& actual, - const ::google::protobuf::Message& expected) { - ::google::protobuf::util::MessageDifferencer differencer; - ::google::protobuf::util::DefaultFieldComparator field_comparator; - ConfigureDifferencer(comp, &field_comparator, &differencer, - actual.GetDescriptor()); - - std::string diff; - differencer.ReportDifferencesToString(&diff); - - // We must put 'expected' as the first argument here, as Compare() - // reports the diff in terms of how the protobuf changes from the - // first argument to the second argument. - differencer.Compare(expected, actual); - - // Removes the trailing '\n' in the diff to make the output look nicer. - if (diff.length() > 0 && *(diff.end() - 1) == '\n') { - diff.erase(diff.end() - 1); - } - - return "with the difference:\n" + diff; -} - -bool ProtoMatcherBase::MatchAndExplain( - const ::google::protobuf::Message& arg, - bool is_matcher_for_pointer, // true iff this matcher is used to match - // a protobuf pointer. - ::testing::MatchResultListener* listener) const { - if (must_be_initialized_ && !arg.IsInitialized()) { - *listener << "which isn't fully initialized"; - return false; - } - - const google::protobuf::Message* const expected = - CreateExpectedProto(arg, listener); - if (expected == nullptr) return false; - - // Protobufs of different types cannot be compared. - const bool comparable = ProtoComparable(arg, *expected); - const bool match = comparable && ProtoCompare(comp(), arg, *expected); - - // Explaining the match result is expensive. We don't want to waste - // time calculating an explanation if the listener isn't interested. - if (listener->IsInterested()) { - const char* sep = ""; - if (is_matcher_for_pointer) { - *listener << PrintProtoPointee(&arg); - sep = ",\n"; - } - - if (!comparable) { - *listener << sep << DescribeTypes(*expected, arg); - } else if (!match) { - *listener << sep << DescribeDiff(comp(), arg, *expected); - } - } - - DeleteExpectedProto(expected); - return match; -} - -} // namespace internal -} // namespace testing diff --git a/ortools/base/protocol-buffer-matchers.h b/ortools/base/protocol-buffer-matchers.h deleted file mode 100644 index 535027ab6c..0000000000 --- a/ortools/base/protocol-buffer-matchers.h +++ /dev/null @@ -1,473 +0,0 @@ -// Copyright 2010-2025 Google LLC -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// emulates g3/testing/base/public/gmock_utils/protocol-buffer-matchers.h -#ifndef ORTOOLS_BASE_PROTOCOL_BUFFER_MATCHERS_H_ -#define ORTOOLS_BASE_PROTOCOL_BUFFER_MATCHERS_H_ - -// gMock matchers used to validate protocol buffer arguments. - -// WHAT THIS IS -// ============ -// -// This library defines the following matchers in the ::protobuf_matchers -// namespace: -// -// EqualsProto(pb) The argument equals pb. -// EquivToProto(pb) The argument is equivalent to pb. -// -// where: -// -// - pb can be either a protobuf value or a human-readable string -// representation of it. -// - When pb is a string, the matcher can optionally accept a -// template argument for the type of the protobuf, -// e.g. EqualsProto("foo: 1"). -// - "equals" is defined as the argument's Equals(pb) method returns true. -// - "equivalent to" is defined as the argument's Equivalent(pb) method -// returns true. -// -// These matchers can match either a protobuf value or a pointer to -// it. They make a copy of pb, and thus can out-live pb. When the -// match fails, the matchers print a detailed message (the value of -// the actual protobuf, the value of the expected protobuf, and which -// fields are different). -// -// EXAMPLES -// ======== -// -// using ::protobuf_matchers::EqualsProto; -// using ::protobuf_matchers::EquivToProto; -// -// // my_pb.Equals(expected_pb). -// EXPECT_THAT(my_pb, EqualsProto(expected_pb)); -// -// // my_pb is equivalent to a protobuf whose foo field is 1 and -// // whose bar field is "x". -// EXPECT_THAT(my_pb, EquivToProto("foo: 1 " -// "bar: 'x'")); - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/log/check.h" -#include "absl/log/log.h" -#include "absl/strings/string_view.h" -#include "gmock/gmock-matchers.h" -#include "google/protobuf/descriptor.h" -#include "google/protobuf/message.h" -#include "google/protobuf/util/message_differencer.h" -#include "gtest/gtest.h" - -namespace testing { -using DifferencerConfigFunction = - std::function; -namespace internal { -// Utilities. -// How to compare two fields (equal vs. equivalent). -typedef ::google::protobuf::util::MessageDifferencer::MessageFieldComparison - ProtoFieldComparison; - -// How to compare two floating-points (exact vs. approximate). -typedef ::google::protobuf::util::DefaultFieldComparator::FloatComparison - ProtoFloatComparison; - -// How to compare repeated fields (whether the order of elements matters). -typedef ::google::protobuf::util::MessageDifferencer::RepeatedFieldComparison - RepeatedFieldComparison; - -// Whether to compare all fields (full) or only fields present in the -// expected protobuf (partial). -typedef ::google::protobuf::util::MessageDifferencer::Scope - ProtoComparisonScope; - -const ProtoFieldComparison kProtoEqual = - ::google::protobuf::util::MessageDifferencer::EQUAL; -const ProtoFieldComparison kProtoEquiv = - ::google::protobuf::util::MessageDifferencer::EQUIVALENT; -const ProtoFloatComparison kProtoExact = - ::google::protobuf::util::DefaultFieldComparator::EXACT; -const ProtoFloatComparison kProtoApproximate = - ::google::protobuf::util::DefaultFieldComparator::APPROXIMATE; -const RepeatedFieldComparison kProtoCompareRepeatedFieldsRespectOrdering = - ::google::protobuf::util::MessageDifferencer::AS_LIST; -const RepeatedFieldComparison kProtoCompareRepeatedFieldsIgnoringOrdering = - ::google::protobuf::util::MessageDifferencer::AS_SET; -const ProtoComparisonScope kProtoFull = - ::google::protobuf::util::MessageDifferencer::FULL; -const ProtoComparisonScope kProtoPartial = - ::google::protobuf::util::MessageDifferencer::PARTIAL; - -// Options for comparing two protobufs. -struct ProtoComparison { - ProtoComparison() - : field_comp(kProtoEqual), - float_comp(kProtoExact), - treating_nan_as_equal(false), - has_custom_margin(false), - has_custom_fraction(false), - repeated_field_comp(kProtoCompareRepeatedFieldsRespectOrdering), - scope(kProtoFull), - float_margin(0.0), - float_fraction(0.0) {} - - ProtoFieldComparison field_comp; - ProtoFloatComparison float_comp; - bool treating_nan_as_equal; - bool has_custom_margin; // only used when float_comp = APPROXIMATE - bool has_custom_fraction; // only used when float_comp = APPROXIMATE - RepeatedFieldComparison repeated_field_comp; - ProtoComparisonScope scope; - double float_margin; // only used when has_custom_margin is set. - double float_fraction; // only used when has_custom_fraction is set. - std::vector ignore_fields; - std::vector ignore_field_paths; - DifferencerConfigFunction differencer_config_function; -}; - -// Whether the protobuf must be initialized. -const bool kMustBeInitialized = true; -const bool kMayBeUninitialized = false; - -// Parses the TextFormat representation of a protobuf, allowing required fields -// to be missing. Returns true if successful. -bool ParsePartialFromAscii(const std::string& pb_ascii, - ::google::protobuf::Message* proto, - std::string* error_text); - -// Returns true iff p and q can be compared (i.e. have the same descriptor). -bool ProtoComparable(const ::google::protobuf::Message& p, - const ::google::protobuf::Message& q); - -// Returns true iff actual and expected are comparable and match. The -// comp argument specifies how the two are compared. -bool ProtoCompare(const ProtoComparison& comp, - const ::google::protobuf::Message& actual, - const ::google::protobuf::Message& expected); - -// Describes the types of the expected and the actual protocol buffer. -std::string DescribeTypes(const ::google::protobuf::Message& expected, - const ::google::protobuf::Message& actual); -// Prints the protocol buffer pointed to by proto. -std::string PrintProtoPointee(const ::google::protobuf::Message* proto); - -// Describes the differences between the two protocol buffers. -std::string DescribeDiff(const ProtoComparison& comp, - const ::google::protobuf::Message& actual, - const ::google::protobuf::Message& expected); -// Common code for implementing EqualsProto and EquivToProto. -class ProtoMatcherBase { - public: - ProtoMatcherBase( - bool must_be_initialized, // Must the argument be fully initialized? - const ProtoComparison& comp) // How to compare the two protobufs. - : must_be_initialized_(must_be_initialized), comp_(new auto(comp)) {} - - ProtoMatcherBase(const ProtoMatcherBase& other) - : must_be_initialized_(other.must_be_initialized_), - comp_(new auto(*other.comp_)) {} - - ProtoMatcherBase(ProtoMatcherBase&& other) = default; - - virtual ~ProtoMatcherBase() {} - - // Prints the expected protocol buffer. - virtual void PrintExpectedTo(::std::ostream* os) const = 0; - - // Returns the expected value as a protobuf object; if the object - // cannot be created (e.g. in ProtoStringMatcher), explains why to - // 'listener' and returns NULL. The caller must call - // DeleteExpectedProto() on the returned value later. - virtual const ::google::protobuf::Message* CreateExpectedProto( - const ::google::protobuf::Message& arg, // For determining the type of - // the expected protobuf. - ::testing::MatchResultListener* listener) const = 0; - - // Deletes the given expected protobuf, which must be obtained from - // a call to CreateExpectedProto() earlier. - virtual void DeleteExpectedProto( - const ::google::protobuf::Message* expected) const = 0; - - bool MatchAndExplain(const ::google::protobuf::Message& arg, - ::testing::MatchResultListener* listener) const { - return MatchAndExplain(arg, false, listener); - } - - bool MatchAndExplain(const ::google::protobuf::Message* arg, - ::testing::MatchResultListener* listener) const { - return (arg != NULL) && MatchAndExplain(*arg, true, listener); - } - - // Describes the expected relation between the actual protobuf and - // the expected one. - void DescribeRelationToExpectedProto(::std::ostream* os) const { - if (comp_->repeated_field_comp == - kProtoCompareRepeatedFieldsIgnoringOrdering) { - *os << "(ignoring repeated field ordering) "; - } - if (!comp_->ignore_fields.empty()) { - *os << "(ignoring fields: "; - const char* sep = ""; - for (size_t i = 0; i < comp_->ignore_fields.size(); ++i, sep = ", ") - *os << sep << comp_->ignore_fields[i]; - *os << ") "; - } - if (comp_->float_comp == kProtoApproximate) { - *os << "approximately "; - if (comp_->has_custom_margin || comp_->has_custom_fraction) { - *os << "("; - if (comp_->has_custom_margin) { - std::stringstream ss; - ss << std::setprecision(std::numeric_limits::digits10 + 2) - << comp_->float_margin; - *os << "absolute error of float or double fields <= " << ss.str(); - } - if (comp_->has_custom_margin && comp_->has_custom_fraction) { - *os << " or "; - } - if (comp_->has_custom_fraction) { - std::stringstream ss; - ss << std::setprecision(std::numeric_limits::digits10 + 2) - << comp_->float_fraction; - *os << "relative error of float or double fields <= " << ss.str(); - } - *os << ") "; - } - } - - if (comp_->differencer_config_function) { - *os << "(with custom differencer config) "; - } - - *os << (comp_->scope == kProtoPartial ? "partially " : "") - << (comp_->field_comp == kProtoEqual ? "equal" : "equivalent") - << (comp_->treating_nan_as_equal ? " (treating NaNs as equal)" : "") - << " to "; - PrintExpectedTo(os); - } - - void DescribeTo(::std::ostream* os) const { - *os << "is " << (must_be_initialized_ ? "fully initialized and " : ""); - DescribeRelationToExpectedProto(os); - } - - void DescribeNegationTo(::std::ostream* os) const { - *os << "is " << (must_be_initialized_ ? "not fully initialized or " : "") - << "not "; - DescribeRelationToExpectedProto(os); - } - - bool must_be_initialized() const { return must_be_initialized_; } - - const ProtoComparison& comp() const { return *comp_; } - - private: - bool MatchAndExplain( - const ::google::protobuf::Message& arg, - bool is_matcher_for_pointer, // true iff this matcher is used to match a - // protobuf pointer. - ::testing::MatchResultListener* listener) const; - - const bool must_be_initialized_; - std::unique_ptr comp_; -}; - -// Returns a copy of the given proto2 message. -inline ::google::protobuf::Message* CloneProto2( - const ::google::protobuf::Message& src) { - ::google::protobuf::Message* clone = src.New(); - clone->CopyFrom(src); - return clone; -} - -// Implements EqualsProto and EquivToProto where the matcher parameter is a -// protobuf. -class ProtoMatcher : public ProtoMatcherBase { - public: - using MessageType = ::google::protobuf::Message; - - ProtoMatcher( - const MessageType& expected, // The expected protobuf. - bool must_be_initialized, // Must the argument be fully initialized? - const ProtoComparison& comp) // How to compare the two protobufs. - : ProtoMatcherBase(must_be_initialized, comp), - expected_(CloneProto2(expected)) { - if (must_be_initialized) { - CHECK(expected.IsInitialized()) - << "The protocol buffer given to *InitializedProto() " - << "must itself be initialized, but the following required fields " - << "are missing: " << expected.InitializationErrorString() << "."; - } - } - - virtual void PrintExpectedTo(::std::ostream* os) const { - *os << expected_->GetDescriptor()->full_name() << " "; - ::testing::internal::UniversalPrint(*expected_, os); - } - - virtual const ::google::protobuf::Message* CreateExpectedProto( - const ::google::protobuf::Message& /* arg */, - ::testing::MatchResultListener* /* listener */) const { - return expected_.get(); - } - - virtual void DeleteExpectedProto( - const ::google::protobuf::Message* /* expected */) const {} - - private: - const std::shared_ptr expected_; -}; - -using PolymorphicProtoMatcher = ::testing::PolymorphicMatcher; - -// Implements EqualsProto and EquivToProto where the matcher parameter is a -// string. -class ProtoStringMatcher : public ProtoMatcherBase { - public: - using MessageType = ::google::protobuf::Message; - - ProtoStringMatcher( - absl::string_view - expected, // The text representing the expected protobuf. - bool must_be_initialized, // Must the argument be fully initialized? - const ProtoComparison comp) // How to compare the two protobufs. - : ProtoMatcherBase(must_be_initialized, comp), - expected_(std::string(expected)) {} - - // Parses the expected string as a protobuf of the same type as arg, - // and returns the parsed protobuf (or NULL when the parse fails). - // The caller must call DeleteExpectedProto() on the return value - // later. - virtual const MessageType* CreateExpectedProto( - const MessageType& arg, ::testing::MatchResultListener* listener) const { - ::google::protobuf::Message* expected_proto = arg.New(); - // We don't insist that the expected string parses as an - // *initialized* protobuf. Otherwise EqualsProto("...") may - // wrongfully fail when the actual protobuf is not fully - // initialized. - std::string error_text; - if (ParsePartialFromAscii(expected_, expected_proto, &error_text)) { - return expected_proto; - } else { - delete expected_proto; - if (listener->IsInterested()) { - *listener << "where "; - PrintExpectedTo(listener->stream()); - *listener << " doesn't parse as a " << arg.GetDescriptor()->full_name() - << ":\n" - << error_text; - } - return NULL; - } - } - - virtual void DeleteExpectedProto( - const ::google::protobuf::Message* expected) const { - delete expected; - } - - virtual void PrintExpectedTo(::std::ostream* os) const { - *os << "<" << expected_ << ">"; - } - - private: - const std::string expected_; -}; - -} // namespace internal - -// Constructs a matcher that matches the argument if -// argument.Equals(m) or argument->Equals(m) returns true. -inline internal::PolymorphicProtoMatcher EqualsProto( - const ::google::protobuf::Message& m) { - internal::ProtoComparison comp; - comp.field_comp = internal::kProtoEqual; - return ::testing::MakePolymorphicMatcher( - internal::ProtoMatcher(m, internal::kMayBeUninitialized, comp)); -} - -inline PolymorphicMatcher EqualsProto( - absl::string_view m) { - internal::ProtoComparison comp; - comp.field_comp = internal::kProtoEqual; - return MakePolymorphicMatcher( - internal::ProtoStringMatcher(m, internal::kMayBeUninitialized, comp)); -} - -// for Pointwise -MATCHER(EqualsProto, "") { - const auto& a = ::testing::get<0>(arg); - const auto& b = ::testing::get<1>(arg); - return ::testing::ExplainMatchResult(EqualsProto(b), a, result_listener); -} - -// Constructs a matcher that matches the argument if -// argument.Equivalent(m) or argument->Equivalent(m) returns true. -inline internal::PolymorphicProtoMatcher EquivToProto( - const ::google::protobuf::Message& m) { - internal::ProtoComparison comp; - comp.field_comp = internal::kProtoEquiv; - return MakePolymorphicMatcher( - internal::ProtoMatcher(m, internal::kMayBeUninitialized, comp)); -} - -inline PolymorphicMatcher EquivToProto( - absl::string_view m) { - internal::ProtoComparison comp; - comp.field_comp = internal::kProtoEquiv; - return MakePolymorphicMatcher( - internal::ProtoStringMatcher(m, internal::kMayBeUninitialized, comp)); -} - -// Returns a matcher that is the same as a given inner matcher, but applies a -// given function to the message differencer before using it for the -// comparison between the expected and actual protobufs. -// -// Prefer more specific transformer functions if possible; they result in -// better error messages and more readable test code. -// -// By default, the differencer is configured to use the field comparator which -// is also passed to the config function. It's possible to modify that -// comparator, although it's preferable to customize it through other -// transformers, e.g. Approximately. -// -// It's also possible to replace the comparator entirely, by passing it to -// set_field_comparator() method of the provided differencer. The user retains -// the ownership over the comparator and must guarantee that its lifetime -// exceeds the lifetime of the matcher. -// -// The config function will be applied after any configuration settings -// specified by other transformers. Overwriting these settings may result in -// misleading test failure messages; in particular, a config function that -// provides its own field comparator should not be used with transformers that -// rely on the default comparator, i.e. Approximately and TreatingNaNsAsEqual. -template -inline InnerProtoMatcher WithDifferencerConfig( - DifferencerConfigFunction differencer_config_function, - InnerProtoMatcher inner_proto_matcher) { - inner_proto_matcher.mutable_impl().SetDifferencerConfigFunction( - differencer_config_function); - return inner_proto_matcher; -} - -} // namespace testing - -#endif // ORTOOLS_BASE_PROTOCOL_BUFFER_MATCHERS_H_ diff --git a/ortools/graph/BUILD.bazel b/ortools/graph/BUILD.bazel index 9e7cd30795..ef25840c45 100644 --- a/ortools/graph/BUILD.bazel +++ b/ortools/graph/BUILD.bazel @@ -180,9 +180,8 @@ cc_library( cc_test( name = "bidirectional_dijkstra_test", - size = "small", + size = "medium", srcs = ["bidirectional_dijkstra_test.cc"], - tags = ["manual"], deps = [ ":bidirectional_dijkstra", ":bounded_dijkstra", @@ -776,7 +775,6 @@ cc_test( name = "dag_shortest_path_test", size = "small", srcs = ["dag_shortest_path_test.cc"], - tags = ["manual"], deps = [ ":dag_shortest_path", "//ortools/base:dump_vars", @@ -852,6 +850,7 @@ cc_library( name = "iterators", hdrs = ["iterators.h"], visibility = ["//visibility:public"], + deps = ["@abseil-cpp//absl/log:check"], ) cc_test( diff --git a/ortools/graph/CMakeLists.txt b/ortools/graph/CMakeLists.txt index 976654252c..009a9e6d12 100644 --- a/ortools/graph/CMakeLists.txt +++ b/ortools/graph/CMakeLists.txt @@ -76,4 +76,10 @@ if(BUILD_TESTING) COMPILE_DEFINITIONS -DROOT_DIR="${CMAKE_SOURCE_DIR}" ) + + # These tests are too long so we disable them. + set_tests_properties( + cxx_graph_min_cost_flow_test + cxx_graph_bidirectional_dijkstra_test + PROPERTIES DISABLED TRUE) endif() diff --git a/ortools/graph/samples/assignment_linear_sum_assignment.cc b/ortools/graph/samples/assignment_linear_sum_assignment.cc index ba8a566010..75ccee5aae 100644 --- a/ortools/graph/samples/assignment_linear_sum_assignment.cc +++ b/ortools/graph/samples/assignment_linear_sum_assignment.cc @@ -15,7 +15,7 @@ // [START import] #include "ortools/graph/assignment.h" -#include +#include #include #include #include diff --git a/ortools/math_opt/cpp/BUILD.bazel b/ortools/math_opt/cpp/BUILD.bazel index 88ac229c0c..47d9e66cf8 100644 --- a/ortools/math_opt/cpp/BUILD.bazel +++ b/ortools/math_opt/cpp/BUILD.bazel @@ -137,8 +137,8 @@ cc_library( cc_test( name = "model_test", + size = "medium", srcs = ["model_test.cc"], - tags = ["manual"], deps = [ ":key_types", ":linear_constraint", @@ -215,6 +215,7 @@ cc_library( cc_test( name = "variable_and_expressions_test", + size = "medium", srcs = ["variable_and_expressions_test.cc"], deps = [ ":matchers", diff --git a/ortools/math_opt/cpp/CMakeLists.txt b/ortools/math_opt/cpp/CMakeLists.txt index c24c50168e..77e11a4ab6 100644 --- a/ortools/math_opt/cpp/CMakeLists.txt +++ b/ortools/math_opt/cpp/CMakeLists.txt @@ -69,4 +69,11 @@ if(BUILD_TESTING) GTest::gtest_main ) endforeach() + + # These tests are too long so we disable them. + set_tests_properties( + cxx_math_opt_cpp_model_test + cxx_math_opt_cpp_variable_and_expressions_test + PROPERTIES DISABLED TRUE) + endif() diff --git a/ortools/math_opt/elemental/CMakeLists.txt b/ortools/math_opt/elemental/CMakeLists.txt index c1fe79df97..dc1c3b1a13 100644 --- a/ortools/math_opt/elemental/CMakeLists.txt +++ b/ortools/math_opt/elemental/CMakeLists.txt @@ -29,23 +29,24 @@ target_link_libraries(${NAME} PRIVATE absl::strings ) -ortools_cxx_library( - NAME - math_opt_elemental_matcher - SOURCES - "elemental_matcher.cc" - "elemental_matcher.h" - TYPE - STATIC - LINK_LIBRARIES - absl::log - absl::status - absl::strings - GTest::gmock - TESTING -) - if(BUILD_TESTING) + ortools_cxx_library( + NAME + math_opt_elemental_matcher + SOURCES + "elemental_matcher.cc" + "elemental_matcher.h" + TYPE + STATIC + LINK_LIBRARIES + absl::log + absl::status + absl::strings + GTest::gmock + ortools::base_gmock + TESTING + ) + file(GLOB _TEST_SRCS "*_test.cc") list(FILTER _TEST_SRCS EXCLUDE REGEX "elemental_export_model_update_test.cc$") diff --git a/ortools/math_opt/samples/python/BUILD.bazel b/ortools/math_opt/samples/python/BUILD.bazel index b85d3cfcc1..25959a984e 100644 --- a/ortools/math_opt/samples/python/BUILD.bazel +++ b/ortools/math_opt/samples/python/BUILD.bazel @@ -99,6 +99,15 @@ py_binary( ], ) +py_binary( + name = "spillover", + srcs = ["spillover.py"], + deps = [ + requirement("absl-py"), + "//ortools/math_opt/python:mathopt", + ], +) + py_binary( name = "time_indexed_scheduling", srcs = ["time_indexed_scheduling.py"], diff --git a/ortools/math_opt/samples/python/spillover.py b/ortools/math_opt/samples/python/spillover.py new file mode 100644 index 0000000000..3fd1cabfd2 --- /dev/null +++ b/ortools/math_opt/samples/python/spillover.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# Copyright 2010-2025 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Solves the problem of buying physical machines to meet VM demand. + +The Spillover problem is defined as follows: + +You have M types of physical machines and V types of Virtual Machines (VMs). You +can use a physical machine of type m to get n_mv copies of VM v. Each physical +machine m has a cost of c_m. Each VM has a demand of d_v. VMs are assigned to +physical machines by the following rule. The demand for each VM type arrives +equally spaced out over the interval [0, 1]. For each VM type, there is a +priority order over the physical machine types that you must follow. When a +demand arrives, if there are any machines of the highest priority type +available, you use them first, then you move on to the second priority machine +type, and so on. Each VM type has a list of compatible physical machine types, +and when the list is exhausted, the remaining demand is not met. Your goal is +to pick quantities of the physical machines to buy (minimizing cost) so that at +least 95% of the total demand of all VM is met. + +The number of machines bought of each type and the number of VMs demanded of +each type is large enough that you can solve an approximate problem instead, +where the number of machines purchased and the assignment of machines to VMs is +fractional, if it is helpful to do so. + +Below, we give a linear programming solution to a continuous approximation of +the spillover problem. Even for the continuous approximation, modeling it with +LP (as opposed to MIP) is nontrivial. + +If for each VM type, the physical machines that are most cost effective are the +highest priority, AND the target service level is 100%, then the problem has a +trivial optimal solution: + 1. Rank the VMs by lowest cost to meet a unit of demand with the #1 preferred + machine type. + 2. For each VM type in the order above, buy machines from #1 preferred machine + type, until either you have met all demand for the VM type. + +The problem is not particularly interesting in isolation, it is more interesting +to embed this LP inside a larger optimization problem (e.g. consider a two stage +problem where in stage one, you buy machines, then in stage two, you realize VM +demand). + +MOE:begin_strip +This example is motivated by the Cloudy problem, see go/fluid-model. +MOE:end_strip +""" + +from collections.abc import Sequence +import dataclasses +import random + +from absl import app +from absl import flags + +from ortools.math_opt.python import mathopt + +_MACHINE_TYPES = flags.DEFINE_integer( + "machine_types", + 100, + "How many types of machines we can fulfill demand with.", +) + +_VM_TYPES = flags.DEFINE_integer( + "vm_types", 500, "How many types of VMs we need to supply." +) + +_FUNGIBILITY = flags.DEFINE_integer( + "fungibility", + 10, + "Each VM type can be satisfied with this many machine types, selected" + " uniformly at random.", +) + +_MAX_DEMAND = flags.DEFINE_integer( + "max_demand", + 100, + "Demand for each VM type is in [max_demand//2, max_demand], uniformly at" + " random.", +) + +_TEST_DATA = flags.DEFINE_bool( + "test_data", False, "Use small test instance instead of random data." +) + +_SEED = flags.DEFINE_integer("seed", 13, "RNG seed for instance creation.") + + +@dataclasses.dataclass(frozen=True) +class MachineUse: + machine_type: int + vms_per_machine: int + + +@dataclasses.dataclass(frozen=True) +class VmDemand: + compatible_machines: tuple[MachineUse, ...] + vm_quantity: int + + +@dataclasses.dataclass(frozen=True) +class SpilloverProblem: + machine_cost: tuple[float, ...] + vm_demands: tuple[VmDemand, ...] + service_level: float + + +def _random_spillover_problem( + num_machines: int, + num_vms: int, + fungibility: int, + max_vm_demand: int, +) -> SpilloverProblem: + """Generates a random SpilloverProblem.""" + machine_costs = tuple(random.random() for _ in range(num_machines)) + vm_demands = [] + all_machines = list(range(num_machines)) + min_vm_demand = max_vm_demand // 2 + for _ in range(num_vms): + machine_use = [] + for machine in random.sample(all_machines, fungibility): + vms_per_machine = random.randint(1, 10) + machine_use.append( + MachineUse(machine_type=machine, vms_per_machine=vms_per_machine) + ) + vm_demands.append( + VmDemand( + compatible_machines=tuple(machine_use), + vm_quantity=random.randint(min_vm_demand, max_vm_demand), + ) + ) + return SpilloverProblem( + machine_cost=machine_costs, + vm_demands=tuple(vm_demands), + service_level=0.95, + ) + + +def _test_problem() -> SpilloverProblem: + """Creates a small SpilloverProblem with optimal objective of 360.""" + # To avoid machine type 2, ensure we buy enough of 1 to not stock out, cost 20 + vm_a = VmDemand( + vm_quantity=10, + compatible_machines=( + MachineUse(machine_type=1, vms_per_machine=1), + MachineUse(machine_type=2, vms_per_machine=1), + ), + ) + # machine type 0 is cheaper, but we don't want to stock out of machine type 1, + # so use all machine type 1, cost 40. + vm_b = VmDemand( + vm_quantity=20, + compatible_machines=( + MachineUse(machine_type=1, vms_per_machine=1), + MachineUse(machine_type=0, vms_per_machine=1), + ), + ) + # Will use 3 copies of machine type 2, cost 300 + vm_c = VmDemand( + vm_quantity=30, + compatible_machines=(MachineUse(machine_type=2, vms_per_machine=10),), + ) + return SpilloverProblem( + machine_cost=(1.0, 2.0, 100.0), + vm_demands=(vm_a, vm_b, vm_c), + service_level=1.0, + ) + + +# Indices: +# * i in I, the VM demands +# * j in J, the machines supplied +# +# Data: +# * c_j: cost of a machine of type j +# * n_ij: how many VMs of type i you get from a machine of type j +# * d_i: the total demand for VMs of type i +# * service_level: the target fraction of demand that is met. +# * P_i subset J: the compatible machine types for VM demand i. +# * UP_i(j) subset P_i, for j in P_i: for VM demand type i, the machines of +# priority higher than j +# +# Decision variables: +# * s_j: the supply of machine type j +# * w_j: the time we run out of machine j, or 1 if we never run out +# * v_ij: when we start using supply j to meet demand i, or w_j if we never use +# this machine type for this demand. +# * o_i: the time we start failing to meet vm demand i +# * m_i: the total demand met for vm type i. +# +# Model the problem: +# min sum_{j in J} c_j s_j +# s.t. +# 1: sum_i m_i >= service_level * sum_{i in I} d_i +# 2: m_i = o_i * d_i for all i in I +# 3: v_ij >= w_r for all i in I, j in C_i, r in UP_i(j) +# 4: v_ij <= w_j for all i in I, j in C_i +# 5: o_i = sum_{j in P_i} (w_j - v_ij) for all i in I +# 6: sum_{i in I: j in P_i} d_i / n_ij(w_j - v_ij) <= s_j for all j in J +# o_i, w_j, v_ij in [0, 1] +# m_i, s_j >= 0 +# +# The constraints say: +# 1. The amount of demand served must be at least service_level fraction of +# total demand. +# 2. The demand served for VM type i is linear in the time we fail to keep +# serving demand. +# 3. Don't start using machine type j for demand i until all higher priority +# machine types r are used up. +# 4. The time we run out of machine type j must be after we start using it for +# VM demand type i. +# 5. The time we are unable to serve further VM demand i is the sum of the +# time spent serving the demand with each eligible machine type. +# 6. The total use of machine type j to serve demand does not exceed the +# supply. +def _solve_spillover_problem(problem: SpilloverProblem) -> None: + """Solves the spillover problem and prints the optimal objective.""" + model = mathopt.Model() + num_machines = len(problem.machine_cost) + num_vms = len(problem.vm_demands) + s = [model.add_variable(lb=0) for _ in range(num_machines)] + w = [model.add_variable(lb=0, ub=1) for _ in range(num_machines)] + o = [model.add_variable(lb=0, ub=1) for _ in range(num_vms)] + m = [model.add_variable(lb=0) for _ in range(num_vms)] + v = [ + { + compat.machine_type: model.add_variable(lb=0, ub=1) + for compat in vm_demand.compatible_machines + } + for vm_demand in problem.vm_demands + ] + + obj = mathopt.LinearExpression() + for j in range(num_machines): + obj += s[j] * problem.machine_cost[j] + model.minimize(obj) + + # Constraint 1: demand served is at least service_level fraction of total. + total_vm_demand = sum(vm_demand.vm_quantity for vm_demand in problem.vm_demands) + model.add_linear_constraint( + mathopt.fast_sum(m) >= problem.service_level * total_vm_demand + ) + + # Constraint 2: demand served is linear in time we stop serving. + for i in range(num_vms): + model.add_linear_constraint(m[i] == o[i] * problem.vm_demands[i].vm_quantity) + + # Constraint 3: use machine type j for demand i after all higher priority + # machine types r are used up. + for i in range(num_vms): + for k, meet_demand in enumerate(problem.vm_demands[i].compatible_machines): + j = meet_demand.machine_type + for l in range(k): + r = problem.vm_demands[i].compatible_machines[l].machine_type + model.add_linear_constraint(v[i][j] >= w[r]) + + # Constraint 4: outage time of machine j is after start time for using j to + # meet VM demand i. + for i in range(num_vms): + for meet_demand in problem.vm_demands[i].compatible_machines: + j = meet_demand.machine_type + model.add_linear_constraint(v[i][j] <= w[j]) + + # Constraint 5: For VM demand i, time service ends is the sum of the time + # spent serving with each eligible machine type. + for i in range(num_vms): + sum_serving = mathopt.LinearExpression() + for meet_demand in problem.vm_demands[i].compatible_machines: + j = meet_demand.machine_type + sum_serving += w[j] - v[i][j] + model.add_linear_constraint(o[i] == sum_serving) + + # Constraint 6: Total use of machine type j is at most the supply. + # + # We build the constraints in bulk because our data is transposed. + total_machine_use = [mathopt.LinearExpression() for _ in range(num_machines)] + for i in range(num_vms): + di = problem.vm_demands[i].vm_quantity + for meet_demand in problem.vm_demands[i].compatible_machines: + j = meet_demand.machine_type + nij = meet_demand.vms_per_machine + total_machine_use[j] += (di / float(nij)) * (w[j] - v[i][j]) + for j in range(num_machines): + model.add_linear_constraint(total_machine_use[j] <= s[j]) + + result = mathopt.solve( + model, + mathopt.SolverType.GLOP, + params=mathopt.SolveParameters(enable_output=True), + ) + if result.termination.reason != mathopt.TerminationReason.OPTIMAL: + raise RuntimeError(f"expected optimal, found: {result.termination}") + print(f"objective: {result.termination.objective_bounds.primal_bound}") + + +def main(argv: Sequence[str]) -> None: + del argv # Unused. + random.seed(_SEED.value) + if _TEST_DATA.value: + problem = _test_problem() + else: + problem = _random_spillover_problem( + _MACHINE_TYPES.value, + _VM_TYPES.value, + _FUNGIBILITY.value, + _MAX_DEMAND.value, + ) + + _solve_spillover_problem(problem) + + +if __name__ == "__main__": + app.run(main) diff --git a/ortools/math_opt/solvers/CMakeLists.txt b/ortools/math_opt/solvers/CMakeLists.txt index 143207c13c..7bb72c8bbd 100644 --- a/ortools/math_opt/solvers/CMakeLists.txt +++ b/ortools/math_opt/solvers/CMakeLists.txt @@ -88,11 +88,9 @@ if(USE_SCIP) "$" "$" ) - if(MSVC) - # Test fail on windows, to investigate. - set_tests_properties(cxx_math_opt_solvers_gscip_solver_test - PROPERTIES DISABLED TRUE) - endif() + # This test fail on windows and takes too long so we disable it. + set_tests_properties(cxx_math_opt_solvers_gscip_solver_test + PROPERTIES DISABLED TRUE) endif() if(USE_GLOP) @@ -149,7 +147,7 @@ ortools_cxx_test( "$" "$" ) -# This test is too long so we currently disable it. +# This test takes too long so we disable it. set_tests_properties(cxx_math_opt_solvers_cp_sat_solver_test PROPERTIES DISABLED TRUE) @@ -249,11 +247,9 @@ if(USE_HIGHS) "$" "$" ) - if(MSVC) - # Test fail on windows, to investigate. - set_tests_properties(cxx_math_opt_solvers_highs_solver_test - PROPERTIES DISABLED TRUE) - endif() + # This test fail on windows and takes too long so we disable it. + set_tests_properties(cxx_math_opt_solvers_highs_solver_test + PROPERTIES DISABLED TRUE) endif() if(USE_XPRESS) diff --git a/ortools/pdlp/BUILD.bazel b/ortools/pdlp/BUILD.bazel index 10283bda2a..187db82bda 100644 --- a/ortools/pdlp/BUILD.bazel +++ b/ortools/pdlp/BUILD.bazel @@ -34,8 +34,8 @@ cc_library( cc_test( name = "scheduler_test", + size = "medium", srcs = ["scheduler_test.cc"], - tags = ["manual"], deps = [ ":scheduler", ":solvers_cc_proto", @@ -95,8 +95,8 @@ cc_library( cc_test( name = "iteration_stats_test", + size = "medium", srcs = ["iteration_stats_test.cc"], - tags = ["manual"], deps = [ ":iteration_stats", ":quadratic_program", @@ -152,7 +152,6 @@ cc_test( name = "primal_dual_hybrid_gradient_test", size = "medium", srcs = ["primal_dual_hybrid_gradient_test.cc"], - tags = ["manual"], deps = [ ":iteration_stats", ":primal_dual_hybrid_gradient", @@ -254,9 +253,8 @@ cc_library( cc_test( name = "sharded_optimization_utils_test", - size = "small", + size = "medium", srcs = ["sharded_optimization_utils_test.cc"], - tags = ["manual"], deps = [ ":quadratic_program", ":sharded_optimization_utils", @@ -288,8 +286,8 @@ cc_library( cc_test( name = "sharded_quadratic_program_test", + size = "medium", srcs = ["sharded_quadratic_program_test.cc"], - tags = ["manual"], deps = [ ":quadratic_program", ":sharded_quadratic_program", @@ -319,9 +317,8 @@ cc_library( cc_test( name = "sharder_test", - size = "small", + size = "medium", srcs = ["sharder_test.cc"], - tags = ["manual"], deps = [ ":scheduler", ":sharder", @@ -430,8 +427,8 @@ cc_library( cc_test( name = "trust_region_test", + size = "medium", srcs = ["trust_region_test.cc"], - tags = ["manual"], deps = [ ":quadratic_program", ":sharded_optimization_utils", diff --git a/ortools/sat/BUILD.bazel b/ortools/sat/BUILD.bazel index bd646ee91f..be9903df67 100644 --- a/ortools/sat/BUILD.bazel +++ b/ortools/sat/BUILD.bazel @@ -630,9 +630,8 @@ cc_library( cc_test( name = "cp_model_search_test", - size = "small", + size = "medium", srcs = ["cp_model_search_test.cc"], - tags = ["manual"], deps = [ ":cp_model_cc_proto", ":cp_model_solver", @@ -1241,12 +1240,9 @@ cc_library( cc_test( name = "cp_model_presolve_test", - size = "small", + size = "medium", srcs = ["cp_model_presolve_test.cc"], - tags = [ - "manual", - "noautofuzz", - ], + tags = ["noautofuzz"], deps = [ ":cp_model_cc_proto", ":cp_model_checker", @@ -3614,6 +3610,7 @@ cc_library( cc_test( name = "2d_rectangle_presolve_test", + size = "medium", srcs = ["2d_rectangle_presolve_test.cc"], deps = [ ":2d_orthogonal_packing_testing", diff --git a/ortools/sat/CMakeLists.txt b/ortools/sat/CMakeLists.txt index e49cc257ce..c5d5638094 100644 --- a/ortools/sat/CMakeLists.txt +++ b/ortools/sat/CMakeLists.txt @@ -57,6 +57,15 @@ if(BUILD_TESTING) GTest::gtest_main ) endforeach() + + # These tests are too long so we disable them. + set_tests_properties( + cxx_sat_2d_rectangle_presolve_test + cxx_sat_cp_model_presolve_random_test + cxx_sat_cp_model_presolve_test + cxx_sat_cp_model_search_test + cxx_sat_integer_expr_test + PROPERTIES DISABLED TRUE) endif() # Sat Runner diff --git a/ortools/set_cover/CMakeLists.txt b/ortools/set_cover/CMakeLists.txt index 3f83ed41e9..d52ea30a2f 100644 --- a/ortools/set_cover/CMakeLists.txt +++ b/ortools/set_cover/CMakeLists.txt @@ -14,6 +14,7 @@ file(GLOB _SRCS "*.h" "*.cc") list(FILTER _SRCS EXCLUDE REGEX ".*/.*_test.cc") list(FILTER _SRCS EXCLUDE REGEX ".*/set_cover_solve.cc") +list(FILTER _SRCS EXCLUDE REGEX ".*/formatting_helper.*") set(NAME ${PROJECT_NAME}_set_cover) diff --git a/ortools/set_cover/python/set_cover_test.py b/ortools/set_cover/python/set_cover_test.py index ee30263878..ee1ed4b9f5 100644 --- a/ortools/set_cover/python/set_cover_test.py +++ b/ortools/set_cover/python/set_cover_test.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from absl import app from absl.testing import absltest from ortools.set_cover.python import set_cover @@ -228,9 +227,5 @@ class SetCoverTest(absltest.TestCase): # KnightsCoverMip -def main(_): - absltest.main() - - if __name__ == "__main__": - app.run(main) + absltest.main() diff --git a/patches/protobuf-matchers-v0.1.1.patch b/patches/protobuf-matchers-v0.1.1.patch new file mode 100644 index 0000000000..f13179a764 --- /dev/null +++ b/patches/protobuf-matchers-v0.1.1.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 63f8581..9b7b207 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -5,7 +5,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}) + list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR}) + + find_package(Protobuf REQUIRED) +-find_package(GTest REQUIRED) ++find_package(absl REQUIRED) + + add_library(protobuf-matchers protobuf-matchers/protocol-buffer-matchers.cc) + target_include_directories(protobuf-matchers PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})