always-cautious/ortools-clone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[MPSolver] Add better code to interrupt solve; rewrite thread management code when using the CP-SAT solver backend

Browse Source
This commit is contained in:
Laurent Perron
2021-09-23 14:30:01 +02:00
parent 604260959e
commit 778b1b3854
11 changed files with 158 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
ortools/linear_solver/gurobi_interface.cc
View File

@@ -83,7 +83,7 @@ class GurobiInterface : public MPSolverInterface {
// Solves the problem using the parameter values specified.
MPSolver::ResultStatus Solve(const MPSolverParameters& param) override;
absl::optional<MPSolutionResponse> DirectlySolveProto(
const MPModelRequest& request) override;
const MPModelRequest& request, std::atomic<bool>* interrupt) override;
// Writes the model.
void Write(const std::string& filename) override;
@@ -160,7 +160,7 @@ class GurobiInterface : public MPSolverInterface {
return 0.0;
}
// TODO(user,user): Not yet working.
// TODO(user): Not yet working.
LOG(DFATAL) << "ComputeExactConditionNumber not implemented for"
<< " GUROBI_LINEAR_PROGRAMMING";
return 0.0;
@@ -717,7 +717,7 @@ bool GurobiInterface::AddIndicatorConstraint(MPConstraint* const ct) {
return !IsContinuous();
}
void GurobiInterface::AddVariable(MPVariable* const ct) {
void GurobiInterface::AddVariable(MPVariable* const var) {
sync_status_ = MUST_RELOAD;
}
@@ -1245,7 +1245,7 @@ MPSolver::ResultStatus GurobiInterface::Solve(const MPSolverParameters& param) {
result_status_ = MPSolver::UNBOUNDED;
break;
case GRB_INF_OR_UNBD:
// TODO(user,user): We could introduce our own "infeasible or
// TODO(user): We could introduce our own "infeasible or
// unbounded" status.
result_status_ = MPSolver::INFEASIBLE;
break;
@@ -1318,7 +1318,10 @@ MPSolver::ResultStatus GurobiInterface::Solve(const MPSolverParameters& param) {
}
absl::optional<MPSolutionResponse> GurobiInterface::DirectlySolveProto(
const MPModelRequest& request) {
const MPModelRequest& request, std::atomic<bool>* interrupt) {
// Interruption via atomic<bool> is not directly supported by Gurobi.
if (interrupt != nullptr) return absl::nullopt;
// Here we reuse the Gurobi environment to support single-use license that
// forbids creating a second environment if one already exists.
const auto status_or = GurobiSolveProto(request, env_);
@@ -1362,7 +1365,7 @@ bool GurobiInterface::NextSolution() {
var->set_solution_value(
grb_variable_values.at(mp_var_to_gurobi_var_.at(i)));
}
// TODO(user,user): This reset may not be necessary, investigate.
// TODO(user): This reset may not be necessary, investigate.
GRBresetparams(GRBgetenv(model_));
return true;
}

48
ortools/linear_solver/linear_solver.cc
View File

@@ -859,7 +859,7 @@ void AppendStatusStr(const std::string& msg, MPSolutionResponse* response) {
// static
void MPSolver::SolveWithProto(const MPModelRequest& model_request,
MPSolutionResponse* response,
const std::atomic<bool>* interrupt) {
std::atomic<bool>* interrupt) {
CHECK(response != nullptr);
if (interrupt != nullptr &&
@@ -880,13 +880,11 @@ void MPSolver::SolveWithProto(const MPModelRequest& model_request,
// If interruption support is not required, we don't need access to the
// underlying solver and can solve it directly if the interface supports it.
if (interrupt == nullptr) {
auto optional_response =
solver.interface_->DirectlySolveProto(model_request);
if (optional_response) {
*response = std::move(optional_response).value();
return;
}
auto optional_response =
solver.interface_->DirectlySolveProto(model_request, interrupt);
if (optional_response) {
*response = std::move(optional_response).value();
return;
}
const absl::optional<LazyMutableCopy<MPModelProto>> optional_model =
@@ -950,7 +948,7 @@ void MPSolver::SolveWithProto(const MPModelRequest& model_request,
constexpr absl::Duration kPollDelay = absl::Microseconds(100);
constexpr absl::Duration kMaxInterruptionDelay = absl::Seconds(10);
while (!interrupt) {
while (!interrupt->load()) {
if (solve_finished.HasBeenNotified()) return;
absl::SleepFor(kPollDelay);
}
@@ -1004,12 +1002,12 @@ void MPSolver::SolveWithProto(const MPModelRequest& model_request,
thread_pool.StartWorkers();
thread_pool.Schedule(polling_func);
// Make sure the interruption notification didn't arrived while waiting to
// Make sure the interruption notification didn't arrive while waiting to
// be scheduled.
if (!interrupt) {
if (!interrupt->load()) {
solver.Solve();
solver.FillSolutionResponseProto(response);
} else {
} else { // *interrupt == true
response->set_status(MPSOLVER_CANCELLED_BY_USER);
response->set_status_str(
"Solve not started, because the user set the atomic<bool> in "
@@ -1220,15 +1218,18 @@ absl::Status MPSolver::LoadSolutionFromProto(const MPSolutionResponse& response,
}
void MPSolver::Clear() {
{
absl::MutexLock lock(&global_count_mutex_);
global_num_variables_ += variables_.size();
global_num_constraints_ += constraints_.size();
}
MutableObjective()->Clear();
gtl::STLDeleteElements(&variables_);
gtl::STLDeleteElements(&constraints_);
variables_.clear();
if (variable_name_to_index_) {
variable_name_to_index_->clear();
}
variable_is_extracted_.clear();
constraints_.clear();
if (constraint_name_to_index_) {
constraint_name_to_index_->clear();
}
@@ -1742,6 +1743,25 @@ bool MPSolver::SupportsCallbacks() const {
return interface_->SupportsCallbacks();
}
// Global counters.
absl::Mutex MPSolver::global_count_mutex_(absl::kConstInit);
int64_t MPSolver::global_num_variables_ = 0;
int64_t MPSolver::global_num_constraints_ = 0;
// static
int64_t MPSolver::global_num_variables() {
// Why not ReaderMutexLock? See go/totw/197#when-are-shared-locks-useful.
absl::MutexLock lock(&global_count_mutex_);
return global_num_variables_;
}
// static
int64_t MPSolver::global_num_constraints() {
// Why not ReaderMutexLock? See go/totw/197#when-are-shared-locks-useful.
absl::MutexLock lock(&global_count_mutex_);
return global_num_constraints_;
}
bool MPSolverResponseStatusIsRpcError(MPSolverResponseStatus status) {
switch (status) {
// Cases that don't yield an RPC error when they happen on the server.

31
ortools/linear_solver/linear_solver.h
View File

@@ -552,13 +552,16 @@ class MPSolver {
*/
static void SolveWithProto(const MPModelRequest& model_request,
MPSolutionResponse* response,
const std::atomic<bool>* interrupt = nullptr);
// `interrupt` is non-const because the internal
// solver may set it to true itself, in some cases.
std::atomic<bool>* interrupt = nullptr);
static bool SolverTypeSupportsInterruption(
const MPModelRequest::SolverType solver) {
// Interruption requires that MPSolver::InterruptSolve is supported for the
// underlying solver. Interrupting requests using SCIP is also not supported
// as of 2021/08/23, since InterruptSolve is not thread-safe for SCIP.
// as of 2021/08/23, since InterruptSolve is not go/thread-safe
// for SCIP (see e.g. cl/350545631 for details).
return solver == MPModelRequest::GLOP_LINEAR_PROGRAMMING ||
solver == MPModelRequest::GUROBI_LINEAR_PROGRAMMING ||
solver == MPModelRequest::GUROBI_MIXED_INTEGER_PROGRAMMING ||
@@ -800,6 +803,12 @@ class MPSolver {
void SetCallback(MPCallback* mp_callback);
bool SupportsCallbacks() const;
// Global counters of variables and constraints ever created across all
// MPSolver instances. Those are only updated after the destruction
// (or Clear()) of each MPSolver instance.
static int64_t global_num_variables();
static int64_t global_num_constraints();
// DEPRECATED: Use TimeLimit() and SetTimeLimit(absl::Duration) instead.
// NOTE: These deprecated functions used the convention time_limit = 0 to mean
// "no limit", which now corresponds to time_limit_ = InfiniteDuration().
@@ -905,6 +914,10 @@ class MPSolver {
// Permanent storage for SetSolverSpecificParametersAsString().
std::string solver_specific_parameter_string_;
static absl::Mutex global_count_mutex_;
static int64_t global_num_variables_ ABSL_GUARDED_BY(global_count_mutex_);
static int64_t global_num_constraints_ ABSL_GUARDED_BY(global_count_mutex_);
MPSolverResponseStatus LoadModelFromProtoInternal(
const MPModelProto& input_model, bool clear_names,
bool check_model_validity, std::string* error_message);
@@ -1564,11 +1577,17 @@ class MPSolverInterface {
// solution is optimal.
virtual MPSolver::ResultStatus Solve(const MPSolverParameters& param) = 0;
// Directly solves a MPModelRequest, bypassing the MPSolver data structures
// entirely. Returns {} (eg. absl::nullopt) if the feature is not supported by
// the underlying solver.
// Attempts to directly solve a MPModelRequest, bypassing the MPSolver data
// structures entirely. Like MPSolver::SolveWithProto(), optionally takes in
// an 'interrupt' boolean.
// Returns {} (eg. absl::nullopt) if direct-solve is not supported by the
// underlying solver (possibly because interrupt != nullptr), in which case
// the user should fall back to using MPSolver.
virtual absl::optional<MPSolutionResponse> DirectlySolveProto(
const MPModelRequest& request) {
const MPModelRequest& request,
// `interrupt` is non-const because the internal
// solver may set it to true itself, in some cases.
std::atomic<bool>* interrupt) {
return absl::nullopt;
}

7
ortools/linear_solver/python/linear_solver.i
View File

@@ -88,6 +88,13 @@ from ortools.linear_solver.linear_solver_natural_api import VariableExpr
return error_message;
}
// Ditto for LoadModelFromProtoWithUniqueNamesOrDie()
std::string LoadModelFromProtoWithUniqueNamesOrDie(const operations_research::MPModelProto& input_model) {
std::string error_message;
$self->LoadModelFromProtoWithUniqueNamesOrDie(input_model, &error_message);
return error_message;
}
// Change the API of LoadSolutionFromProto() to simply return a boolean.
bool LoadSolutionFromProto(
const operations_research::MPSolutionResponse& response,

1
ortools/linear_solver/samples/integer_programming_example.cc
View File

@@ -71,7 +71,6 @@ void IntegerProgrammingExample() {
// [END objective]
// [START solve]
solver->SetNumThreads(1);
const MPSolver::ResultStatus result_status = solver->Solve();
// Check that the problem has an optimal solution.
if (result_status != MPSolver::OPTIMAL) {

1
ortools/linear_solver/samples/integer_programming_example.py
View File

@@ -64,7 +64,6 @@ def IntegerProgrammingExample():
# Solve the problem and print the solution.
# [START print_solution]
solver.SetNumThreads(1)
solver.Solve()
# Print the objective value of the solution.
print('Maximum objective function value = %d' % solver.Objective().Value())

27
ortools/linear_solver/sat_interface.cc
View File

@@ -46,6 +46,8 @@ class SatInterface : public MPSolverInterface {
// ----- Solve -----
MPSolver::ResultStatus Solve(const MPSolverParameters& param) override;
absl::optional<MPSolutionResponse> DirectlySolveProto(
const MPModelRequest& request, std::atomic<bool>* interrupt) override;
bool InterruptSolve() override;
// ----- Model modifications and extraction -----
@@ -101,7 +103,7 @@ class SatInterface : public MPSolverInterface {
std::atomic<bool> interrupt_solve_;
sat::SatParameters parameters_;
int num_threads_ = 8;
int num_threads_ = 0;
};
SatInterface::SatInterface(MPSolver* const solver)
@@ -182,6 +184,26 @@ MPSolver::ResultStatus SatInterface::Solve(const MPSolverParameters& param) {
return result_status_;
}
absl::optional<MPSolutionResponse> SatInterface::DirectlySolveProto(
const MPModelRequest& request, std::atomic<bool>* interrupt) {
absl::StatusOr<MPSolutionResponse> status_or =
SatSolveProto(request, interrupt);
if (status_or.ok()) return std::move(status_or).value();
if (request.enable_internal_solver_output()) {
LOG(INFO) << "Failed SAT solve: " << status_or.status();
}
MPSolutionResponse response;
// As of 2021-08, the sole non-OK status returned by SatSolveProto is an
// INVALID_ARGUMENT error caused by invalid solver parameters.
// TODO(user): Move that conversion to SatSolveProto, which should always
// return a MPSolutionResponse, even for errors.
response.set_status(absl::IsInvalidArgument(status_or.status())
? MPSOLVER_MODEL_INVALID_SOLVER_PARAMETERS
: MPSOLVER_ABNORMAL);
response.set_status_str(status_or.status().ToString());
return response;
}
bool SatInterface::InterruptSolve() {
interrupt_solve_ = true;
return true;
@@ -263,10 +285,9 @@ void SatInterface::ExtractNewConstraints() { NonIncrementalChange(); }
void SatInterface::ExtractObjective() { NonIncrementalChange(); }
void SatInterface::SetParameters(const MPSolverParameters& param) {
// By default, we use 8 threads as it allows to try a good set of orthogonal
// parameters. This can be overridden by the user.
parameters_.set_num_search_workers(num_threads_);
parameters_.set_log_search_progress(!quiet_);
parameters_.set_linearization_level(2);
SetCommonParameters(param);
}

78
ortools/linear_solver/sat_proto_solver.cc
View File

@@ -67,11 +67,9 @@ MPSolverResponseStatus ToMPSolverResponseStatus(sat::CpSolverStatus status,
absl::StatusOr<MPSolutionResponse> SatSolveProto(
MPModelRequest request, std::atomic<bool>* interrupt_solve,
std::function<void(const std::string&)> logging_callback) {
// By default, we use 8 threads as it allows to try a good set of orthogonal
// parameters. This can be overridden by the user.
std::function<void(const std::string&)> logging_callback,
std::function<void(const MPSolution&)> solution_callback) {
sat::SatParameters params;
params.set_num_search_workers(8);
params.set_log_search_progress(request.enable_internal_solver_output());
if (request.has_solver_specific_parameters()) {
// See EncodeSatParametersAsString() documentation.
@@ -91,9 +89,9 @@ absl::StatusOr<MPSolutionResponse> SatSolveProto(
}
}
if (request.has_solver_time_limit_seconds()) {
params.set_max_time_in_seconds(
static_cast<double>(request.solver_time_limit_seconds()) / 1000.0);
params.set_max_time_in_seconds(request.solver_time_limit_seconds());
}
params.set_linearization_level(2);
// TODO(user): We do not support all the parameters here. In particular the
// logs before the solver is called will not be appended to the response. Fix
@@ -125,18 +123,20 @@ absl::StatusOr<MPSolutionResponse> SatSolveProto(
const glop::GlopParameters glop_params;
MPModelProto* const mp_model = request.mutable_model();
std::vector<std::unique_ptr<glop::Preprocessor>> for_postsolve;
const auto status =
ApplyMipPresolveSteps(glop_params, mp_model, &for_postsolve, &logger);
if (status == MPSolverResponseStatus::MPSOLVER_INFEASIBLE) {
if (params.log_search_progress()) {
// This is needed for our benchmark scripts.
sat::CpSolverResponse cp_response;
cp_response.set_status(sat::CpSolverStatus::INFEASIBLE);
LOG(INFO) << CpSolverResponseStats(cp_response);
if (!params.enumerate_all_solutions()) {
const auto status =
ApplyMipPresolveSteps(glop_params, mp_model, &for_postsolve, &logger);
if (status == MPSolverResponseStatus::MPSOLVER_INFEASIBLE) {
if (params.log_search_progress()) {
// This is needed for our benchmark scripts.
sat::CpSolverResponse cp_response;
cp_response.set_status(sat::CpSolverStatus::INFEASIBLE);
LOG(INFO) << CpSolverResponseStats(cp_response);
}
response.set_status(MPSolverResponseStatus::MPSOLVER_INFEASIBLE);
response.set_status_str("Problem proven infeasible during MIP presolve");
return response;
}
response.set_status(MPSolverResponseStatus::MPSOLVER_INFEASIBLE);
response.set_status_str("Problem proven infeasible during MIP presolve");
return response;
}
// We need to do that before the automatic detection of integers.
@@ -210,6 +210,33 @@ absl::StatusOr<MPSolutionResponse> SatSolveProto(
interrupt_solve);
}
auto post_solve = [&](const sat::CpSolverResponse& cp_response) {
MPSolution mp_solution;
mp_solution.set_objective_value(cp_response.objective_value());
// Postsolve the bound shift and scaling.
glop::ProblemSolution glop_solution((glop::RowIndex(old_num_constraints)),
(glop::ColIndex(old_num_variables)));
for (int v = 0; v < glop_solution.primal_values.size(); ++v) {
glop_solution.primal_values[glop::ColIndex(v)] =
static_cast<double>(cp_response.solution(v)) / var_scaling[v];
}
for (int i = for_postsolve.size(); --i >= 0;) {
for_postsolve[i]->RecoverSolution(&glop_solution);
}
for (int v = 0; v < glop_solution.primal_values.size(); ++v) {
mp_solution.add_variable_value(
glop_solution.primal_values[glop::ColIndex(v)]);
}
return mp_solution;
};
if (solution_callback != nullptr) {
sat_model.Add(sat::NewFeasibleSolutionObserver(
[&](const sat::CpSolverResponse& cp_response) {
solution_callback(post_solve(cp_response));
}));
}
// Solve.
const sat::CpSolverResponse cp_response =
sat::SolveCpModel(cp_model, &sat_model);
@@ -223,20 +250,9 @@ absl::StatusOr<MPSolutionResponse> SatSolveProto(
response.status() == MPSOLVER_OPTIMAL) {
response.set_objective_value(cp_response.objective_value());
response.set_best_objective_bound(cp_response.best_objective_bound());
// Postsolve the bound shift and scaling.
glop::ProblemSolution solution((glop::RowIndex(old_num_constraints)),
(glop::ColIndex(old_num_variables)));
for (int v = 0; v < solution.primal_values.size(); ++v) {
solution.primal_values[glop::ColIndex(v)] =
static_cast<double>(cp_response.solution(v)) / var_scaling[v];
}
for (int i = for_postsolve.size(); --i >= 0;) {
for_postsolve[i]->RecoverSolution(&solution);
}
for (int v = 0; v < solution.primal_values.size(); ++v) {
response.add_variable_value(solution.primal_values[glop::ColIndex(v)]);
}
MPSolution post_solved_solution = post_solve(cp_response);
*response.mutable_variable_value() =
std::move(*post_solved_solution.mutable_variable_value());
}
return response;

8
ortools/linear_solver/sat_proto_solver.h
View File

@@ -43,9 +43,15 @@ namespace operations_research {
// log_to_stdout is true so even with a callback, the logs will appear on stdout
// too unless log_to_stdout is set to false. The enable_internal_solver_output
// in the request will act as the SAT parameter log_search_progress.
//
// The optional solution_callback will be called on each intermediate solution
// found by the solver. The solver may call solution_callback from multiple
// threads, but it will ensure that at most one thread executes
// solution_callback at a time.
absl::StatusOr<MPSolutionResponse> SatSolveProto(
MPModelRequest request, std::atomic<bool>* interrupt_solve = nullptr,
std::function<void(const std::string&)> logging_callback = nullptr);
std::function<void(const std::string&)> logging_callback = nullptr,
std::function<void(const MPSolution&)> solution_callback = nullptr);
// Returns a string that should be used in MPModelRequest's
// solver_specific_parameters field to encode the SAT parameters.

7
ortools/linear_solver/scip_interface.cc
View File

@@ -66,7 +66,7 @@ class SCIPInterface : public MPSolverInterface {
void SetOptimizationDirection(bool maximize) override;
MPSolver::ResultStatus Solve(const MPSolverParameters& param) override;
absl::optional<MPSolutionResponse> DirectlySolveProto(
const MPModelRequest& request) override;
const MPModelRequest& request, std::atomic<bool>* interrupt) override;
void Reset() override;
void SetVariableBounds(int var_index, double lb, double ub) override;
@@ -860,10 +860,13 @@ void SCIPInterface::SetSolution(SCIP_SOL* solution) {
}
absl::optional<MPSolutionResponse> SCIPInterface::DirectlySolveProto(
const MPModelRequest& request) {
const MPModelRequest& request, std::atomic<bool>* interrupt) {
// ScipSolveProto doesn't solve concurrently.
if (solver_->GetNumThreads() > 1) return absl::nullopt;
// Interruption via atomic<bool> is not directly supported by SCIP.
if (interrupt != nullptr) return absl::nullopt;
const auto status_or = ScipSolveProto(request);
if (status_or.ok()) return status_or.value();
// Special case: if something is not implemented yet, fall back to solving

1
ortools/linear_solver/scip_proto_solver.cc
View File

@@ -882,7 +882,6 @@ absl::StatusOr<MPSolutionResponse> ScipSolveProto(
// NOTE(user): As of SCIP 7.0.1, getting the pointer to all
// solutions is as fast as getting the pointer to the best solution.
// See google3/scip/scip_sol.c?l=2264&rcl=322332899.
SCIP_SOL** const scip_solutions = SCIPgetSols(scip);
response.set_objective_value(SCIPgetSolOrigObj(scip, scip_solutions[0]));
response.set_best_objective_bound(SCIPgetDualbound(scip));