Compiled Clarabel code sometimes producing wrong results compared to cvxpy only
The-SS opened this issue · comments
Hi,
I noticed that the cvxpygen C code, with the python wrapper, may produce some wrong results even when solving with cvxpy works fine.
I say may because I've noticed that this behavior is not consistent and varies based on the problem.
I also think that this is related to #45 and #46 (and the associated commits).
Here are my observations. Note that in all cases, using cvxpy works fine.
Observations
MPC.ipynb benchmark
- with
cvxpygenv0.3.2:- ECOS produces a wrong result on the second run.
- CLARABEL, OSQP, SCS work (even on the second run)
This was resolved through #45 and now withcvxpygenv0.3.3 or v0.3.4 all four solvers produce correct results even on the second run.
mpc_2d_robot benchmark (see code below)
- with
cvxpygen0.3.2:- ECOS fails (again, this was addressed in #45) (control value jumps between max/min)
- CLARABEL, OSQP, SCS work fine.
- with
cvxpygen0.3.3 or 0.3.4- CLARABEL fails (control value constant and robot fails to reach the goal)
- OSQP, SCS, ECOS work fine.
Notes
I'm on MacOS Sonoma 14.4.1 and using AppleClang. Python environment:
- Python 3.10 (with Anaconda)
- cvxpy 1.4.3
- osqp 0.6.5
- scs 3.2.4.post1
- ecos 2.0.13
- clarabel 0.7.1
The main issue is that with cvxpygen v0.3.3 and v0.3.4, CLARABEL compiled code sometimes doesn't work as expected.
Going back to v0.3.2, ECOS produced wrong code on the second run, but CLARABEL worked fine.
Code
mpc_2d_robot benchmark
import time
import cvxpy as cp
import numpy as np
from cvxpygen import cpg
from matplotlib import pyplot as plt
def mpc_2d_robot(use_cpg=False, gen_cpg=False, plot_res=False, solver=cp.CLARABEL):
"""
Certainty equivalence MPC. Linear dynamics with additive noise.
:return:
"""
# Dynamics
A, B = np.eye(2), np.eye(2)
def dyn(x, u):
return A @ x + B @ u # dynamics
T = 7 # horizon
x0_val = np.array([-2, -0.8]) # initial state
Q, R = np.diag([1, 1]), np.diag([1, 1]) # object cost matrices
# params and vars
x0 = cp.Parameter(2, 'x0')
ctrl = cp.Variable((2, T), 'ctrl')
state = cp.Variable((2, T + 1), 'state')
def find_xQx(t_future, u, x_now):
xt = find_xt(t_future, x_now, u)
return cp.QuadForm(xt, Q)
def find_xt(t_future, x_now, u):
At = np.linalg.matrix_power(A, t_future)
xt = At @ x_now
for i in range(t_future):
Ai = np.linalg.matrix_power(A, i)
Bu = B @ u[:, t_future - 1 - i]
AiBu = Ai @ Bu
xt = xt + AiBu
return xt
# define optimization problem
obj = 0
for t in range(T):
obj += find_xQx(t + 1, ctrl, x0)
obj += cp.quad_form(ctrl[:, t], R)
constr = [state[:, 0] == x0]
for t in range(T):
constr += [state[:, t + 1] == dyn(state[:, t], ctrl[:, t])]
constr += [ctrl <= np.array([[0.2, 0.2]]).T,
ctrl >= np.array([[-0.2, -0.2]]).T]
prob = cp.Problem(cp.Minimize(obj), constr)
if use_cpg:
if gen_cpg:
cpg.generate_code(prob, code_dir='mpc_2d', solver=solver)
from mpc_2d.cpg_solver import cpg_solve
prob.register_solve('cpg', cpg_solve)
# sim settings
steps = 20
current_state = x0_val
# plotting results
x_hist = [current_state]
u_hist = []
t_hist = []
for t in range(steps):
x0.value = current_state
if use_cpg:
if solver in [cp.ECOS, cp.OSQP]:
ts = time.time()
prob.solve(method='cpg', updated_params=['x0'])
te = time.time()
else:
ts = time.time()
prob.solve(method='cpg', updated_params=['x0'], verbose=False)
te = time.time()
else:
ts = time.time()
prob.solve(solver)
te = time.time()
t_hist.append(te - ts)
print(prob.status)
u_now = ctrl.value[:, 0]
next_state = dyn(current_state, u_now)
x_hist.append(next_state)
current_state = next_state
print(current_state)
u_hist.append(ctrl.value[:, 0])
x_hist = np.array(x_hist)
u_hist = np.array(u_hist)
if plot_res:
plt.plot(x_hist[:, 0], x_hist[:, 1])
plt.scatter(0, 0)
plt.show()
fig, axs = plt.subplots(2)
axs[0].plot(range(steps), u_hist[:, 0])
axs[1].plot(range(steps), u_hist[:, 1])
plt.show()
return t_hist
if __name__ == "__main__":
# solver_list = [cp.OSQP, cp.SCS, cp.CLARABEL, cp.ECOS]
solver = cp.CLARABEL
mpc_2d_robot(use_cpg=False, gen_cpg=False, plot_res=True, solver=solver)
mpc_2d_robot(use_cpg=True, gen_cpg=True, plot_res=True, solver=solver)
# in all cases, cvxpy solves fine
# results for codegen:
# cvxpygen 0.3.2:
# - works: OSQP, SCS, CLARABEL
# - fails: ECOS
# cvxpygen 0.3.3:
# - works: OSQP, SCS, ECOS
# - fails: CLARABEL
# cvxpygen 0.3.4:
# - works: OSQP, SCS, ECOS
# - fails: CLARABEL
MPC.ipynb code (slightly modified version of MPC.ipynb)
import cvxpy as cp
# cvxpygen v0.3.2:
# - works: CLARABEL, OSQP, SCS,
# - fails: ECOS
# cvxpygen v0.3.3:
# - works: CLARABEL, OSQP, SCS, ECOS
# - fails: /
# cvxpygen v0.3.4:
# - works: CLARABEL, OSQP, SCS, ECOS
# - fails: /
# solver_list = [cp.OSQP, cp.SCS, cp.CLARABEL, cp.ECOS]
solver = cp.ECOS
# define dimensions
H, n, m = 10, 6, 3
# define variables
U = cp.Variable((m, H), name='U')
X = cp.Variable((n, H+1), name='X')
# define parameters
Psqrt = cp.Parameter((n, n), name='Psqrt')
Qsqrt = cp.Parameter((n, n), name='Qsqrt')
Rsqrt = cp.Parameter((m, m), name='Rsqrt')
A = cp.Parameter((n, n), name='A')
B = cp.Parameter((n, m), name='B')
x_init = cp.Parameter(n, name='x_init')
# define objective
objective = cp.Minimize(cp.sum_squares(Psqrt@X[:,H]) + cp.sum_squares(Qsqrt@X[:,:H]) + cp.sum_squares(Rsqrt@U))
# define constraints
constraints = [X[:,1:] == A@X[:,:H]+B@U,
cp.abs(U) <= 1,
X[:,0] == x_init]
# define problem
problem = cp.Problem(objective, constraints)
import numpy as np
# continuous-time dynmaics
A_cont = np.concatenate((np.array([[0, 0, 0, 1, 0, 0],
[0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 1]]),
np.zeros((3,6))), axis=0)
mass = 1
B_cont = np.concatenate((np.zeros((3,3)),
(1/mass)*np.diag(np.ones(3))), axis=0)
# discrete-time dynamics
td = 0.1
A.value = np.eye(n)+td*A_cont
B.value = td*B_cont
# cost
Psqrt.value = np.eye(n)
Qsqrt.value = np.eye(n)
Rsqrt.value = np.sqrt(0.1)*np.eye(m)
# measurement
x_init.value = np.array([2, 2, 2, -1, -1, 1])
val = problem.solve(solver=solver)
from cvxpygen import cpg
cpg.generate_code(problem, code_dir='MPC_code', solver=solver)
from MPC_code.cpg_solver import cpg_solve
import numpy as np
import pickle
import time
# load the serialized problem formulation
with open('MPC_code/problem.pickle', 'rb') as f:
prob = pickle.load(f)
# assign parameter values
prob.param_dict['A'].value = np.eye(n)+td*A_cont
prob.param_dict['B'].value = td*B_cont
prob.param_dict['Psqrt'].value = np.eye(n)
prob.param_dict['Qsqrt'].value = np.eye(n)
prob.param_dict['Rsqrt'].value = np.sqrt(0.1)*np.eye(m)
prob.param_dict['x_init'].value = np.array([2, 2, 2, -1, -1, 1])
# solve problem conventionally
t0 = time.time()
# CVXPY chooses eps_abs=eps_rel=1e-5, max_iter=10000, polish=True by default,
# however, we choose the OSQP default values here, as they are used for code generation as well
val = prob.solve(solver=solver)
t1 = time.time()
print('\nCVXPY\nSolve time: %.3f ms' % (1000 * (t1 - t0)))
print('Objective function value: %.6f\n' % val)
cvxpy_only_val = prob.value
# solve problem with C code via python wrapper
prob.register_solve('CPG', cpg_solve)
t0 = time.time()
val = prob.solve(method='CPG')
t1 = time.time()
print('\nCVXPYgen\nSolve time: %.3f ms' % (1000 * (t1 - t0)))
print('Objective function value: %.6f\n' % val)
prob.register_solve('CPG', cpg_solve)
t0 = time.time()
val = prob.solve(method='CPG')
t1 = time.time()
print('\nCVXPYgen\nSolve time: %.3f ms' % (1000 * (t1 - t0)))
print('Objective function value: %.6f\n' % val)
cvxpygen_val = prob.value
print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
print("Summary: ")
print("cvxpy only:\n\t", cvxpy_only_val)
print("cvxpygen (2nd solve):\n\t", cvxpygen_val)
Thanks for pointing this out, it is fixed in version 0.3.5.
Thanks!