Incompatible shapes: [1,63,4] vs. [1,64,4] while training
Tejeshwarabm opened this issue · comments
INFO:tensorflow:global step 2: loss = 1.7981 (25.561 sec/step)
INFO:tensorflow:Error reported to Coordinator: <class 'tensorflow.python.framework.errors_impl.InvalidArgumentError'>, Incompatible shapes: [1,63,4] vs. [1,64,4]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradients/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
I ave the same error;
INFO:tensorflow:Error reported to Coordinator: <class 'tensorflow.python.framewo
rk.errors_impl.InvalidArgumentError'>, Incompatible shapes: [1,63,4] vs. [1,64,4
]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradien
tArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task
:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradient
s/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
Caused by op 'gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradientAr
gs', defined at:
File "object_detection/train.py", line 198, in
tf.app.run()
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\platform\app.py", line 126, in run
_sys.exit(main(argv))
File "object_detection/train.py", line 194, in main
worker_job_name, is_chief, FLAGS.train_dir)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 226, in train
clones, training_optimizer, regularization_losses=None)
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 297, in o
ptimize_clones
optimizer, clone, num_clones, regularization_losses, **kwargs)
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 261, in _
optimize_clone
clone_grad = optimizer.compute_gradients(sum_loss, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\training\optimizer.py", line 526, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 494, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 636, in _GradientsHelper
lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 385, in _MaybeCompile
return grad_fn() # Exit early
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 636, in
lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\math_grad.py", line 857, in _SubGrad
rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gen_array_ops.py", line 812, in broadcast_gradient_args
"BroadcastGradientArgs", s0=s0, s1=s1, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-
access
...which was originally created as op 'Loss/BoxClassifierLoss/Loss/sub', defined
at:
File "object_detection/train.py", line 198, in
tf.app.run()
[elided 1 identical lines from previous traceback]
File "object_detection/train.py", line 194, in main
worker_job_name, is_chief, FLAGS.train_dir)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 192, in train
clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue])
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 193, in c
reate_clones
outputs = model_fn(*args, **kwargs)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 133, in crea
te_losses
losses_dict = detection_model.loss(prediction_dict)
File "C:\Users\sukhovva\Conv1\object_detection\meta_architectures\faster_rcnn
meta_arch.py", line 1265, in loss
groundtruth_classes_with_background_list))
File "C:\Users\sukhovva\Conv1\object_detection\meta_architectures\faster_rcnn_
meta_arch.py", line 1421, in loss_box_classifier
batch_reg_targets, weights=batch_reg_weights) / normalizer
File "C:\Users\sukhovva\Conv1\object_detection\core\losses.py", line 71, in __
call_
return self._compute_loss(prediction_tensor, target_tensor, **params)
File "C:\Users\sukhovva\Conv1\object_detection\core\losses.py", line 157, in _
compute_loss
diff = prediction_tensor - target_tensor
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\math_ops.py", line 979, in binary_op_wrapper
return func(x, y, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gen_math_ops.py", line 8582, in sub
"Sub", x=x, y=y, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-
access
InvalidArgumentError (see above for traceback): Incompatible shapes: [1,63,4] vs
. [1,64,4]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradien
tArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task
:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradient
s/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1322, in _do_call
return fn(*args)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shape
s: [1,63,4] vs. [1,64,4]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradien
tArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task
:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradient
s/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "object_detection/train.py", line 198, in
tf.app.run()
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\platform\app.py", line 126, in run
_sys.exit(main(argv))
File "object_detection/train.py", line 194, in main
worker_job_name, is_chief, FLAGS.train_dir)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 296, in train
saver=saver)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\co
ntrib\slim\python\slim\learning.py", line 769, in train
sess, train_op, global_step, train_step_kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\co
ntrib\slim\python\slim\learning.py", line 487, in train_step
run_metadata=run_metadata)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 900, in run
run_metadata_ptr)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1316, in _do_run
run_metadata)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shape
s: [1,63,4] vs. [1,64,4]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradien
tArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task
:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradient
s/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
Caused by op 'gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradientAr
gs', defined at:
File "object_detection/train.py", line 198, in
tf.app.run()
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\platform\app.py", line 126, in run
_sys.exit(main(argv))
File "object_detection/train.py", line 194, in main
worker_job_name, is_chief, FLAGS.train_dir)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 226, in train
clones, training_optimizer, regularization_losses=None)
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 297, in o
ptimize_clones
optimizer, clone, num_clones, regularization_losses, **kwargs)
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 261, in _
optimize_clone
clone_grad = optimizer.compute_gradients(sum_loss, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\training\optimizer.py", line 526, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 494, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 636, in _GradientsHelper
lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 385, in _MaybeCompile
return grad_fn() # Exit early
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gradients_impl.py", line 636, in
lambda: grad_fn(op, *out_grads))
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\math_grad.py", line 857, in _SubGrad
rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gen_array_ops.py", line 812, in broadcast_gradient_args
"BroadcastGradientArgs", s0=s0, s1=s1, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-
access
...which was originally created as op 'Loss/BoxClassifierLoss/Loss/sub', defined
at:
File "object_detection/train.py", line 198, in
tf.app.run()
[elided 1 identical lines from previous traceback]
File "object_detection/train.py", line 194, in main
worker_job_name, is_chief, FLAGS.train_dir)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 192, in train
clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue])
File "C:\Users\sukhovva\Conv1\slim\deployment\model_deploy.py", line 193, in c
reate_clones
outputs = model_fn(*args, **kwargs)
File "C:\Users\sukhovva\Conv1\object_detection\trainer.py", line 133, in crea
te_losses
losses_dict = detection_model.loss(prediction_dict)
File "C:\Users\sukhovva\Conv1\object_detection\meta_architectures\faster_rcnn
meta_arch.py", line 1265, in loss
groundtruth_classes_with_background_list))
File "C:\Users\sukhovva\Conv1\object_detection\meta_architectures\faster_rcnn_
meta_arch.py", line 1421, in loss_box_classifier
batch_reg_targets, weights=batch_reg_weights) / normalizer
File "C:\Users\sukhovva\Conv1\object_detection\core\losses.py", line 71, in __
call_
return self._compute_loss(prediction_tensor, target_tensor, **params)
File "C:\Users\sukhovva\Conv1\object_detection\core\losses.py", line 157, in _
compute_loss
diff = prediction_tensor - target_tensor
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\math_ops.py", line 979, in binary_op_wrapper
return func(x, y, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\ops\gen_math_ops.py", line 8582, in sub
"Sub", x=x, y=y, name=name)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\py
thon\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-
access
InvalidArgumentError (see above for traceback): Incompatible shapes: [1,63,4] vs
. [1,64,4]
[[Node: gradients/Loss/BoxClassifierLoss/Loss/sub_grad/BroadcastGradien
tArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task
:0/device:CPU:0"](gradients/Loss/BoxClassifierLoss/Loss/sub_grad/Shape, gradient
s/Loss/BoxClassifierLoss/Loss/sub_1_grad/Shape)]]
How it can be fixed?
@Tejeshwarabm
it can be solved by adjusting number of objects that classified in config file for CNN:
For example if you put 3 items in label_map.pbtxt than you should point the same num_classes in faster_rcnn_resnet101.config file:
model {
faster_rcnn {
num_classes: 3
image_resizer {
keep_aspect_ratio_resizer {
min_dimension: 600
max_dimension: 1024
}
}