ViLD running zeroshot evaluation on custom dataset
Jawing opened this issue · comments
INFO:tensorflow:Done running local_init_op.
I0719 11:49:25.955098 139723603752768 session_manager.py:530] Done running local_init_op.
2022-07-19 11:54:59.144640: W tensorflow/core/grappler/optimizers/loop_optimizer.cc:907] Skipping loop optimization for Merge node with control input: generate_detections/nms_detections_3_255/PartitionedCall/cond/branch_executed/_138007
Traceback (most recent call last):
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1377, in _do_call
return fn(*args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1360, in _run_fn
return self._call_tf_sessionrun(options, feed_dict, fetch_list,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1453, in _call_tf_sessionrun
return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
[[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
[[stack_1/_80625]]
(1) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
[[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
tf.app.run(main)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
executor.evaluate(eval_input_fn, eval_times, ckpt)
File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
outputs = six.next(predictor)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 642, in predict
preds_evaluated = mon_sess.run(predictions)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 782, in run
return self._sess.run(
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1311, in run
return self._sess.run(
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1416, in run
raise six.reraise(*original_exc_info)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/six.py", line 719, in reraise
raise value
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1401, in run
return self._sess.run(*args, **kwargs)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1469, in run
outputs = _WrappedSession.run(
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/training/monitored_session.py", line 1232, in run
return self._sess.run(*args, **kwargs)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 967, in run
result = self._run(None, fetches, feed_dict, options_ptr,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1190, in _run
results = self._do_run(handle, final_targets, final_fetches,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1370, in _do_run
return self._do_call(_run_fn, feeds, fetches, targets, options,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/client/session.py", line 1396, in _do_call
raise type(e)(node_def, op, message) # pylint: disable=no-value-for-parameter
tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:
Detected at node 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2' defined at (most recent call last):
File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
tf.app.run(main)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
executor.evaluate(eval_input_fn, eval_times, ckpt)
File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
outputs = six.next(predictor)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
return self._model.predict(features)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
outputs = self._build_outputs(images, labels, mode)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
Node: 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2'
Detected at node 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2' defined at (most recent call last):
File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
tf.app.run(main)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
executor.evaluate(eval_input_fn, eval_times, ckpt)
File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
outputs = six.next(predictor)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
return self._model.predict(features)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
outputs = self._build_outputs(images, labels, mode)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
Node: 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2'
2 root error(s) found.
(0) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
[[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
[[stack_1/_80625]]
(1) INVALID_ARGUMENT: Incompatible shapes: [1,1,1203] vs. [256,1000,6]
[[{{node frcnn_layer_0/fast_rcnn_head/BroadcastTo_2}}]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'frcnn_layer_0/fast_rcnn_head/BroadcastTo_2':
File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
tf.app.run(main)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "/home/wanjiz/tpu/models/official/detection/main.py", line 160, in main
executor.evaluate(eval_input_fn, eval_times, ckpt)
File "/home/wanjiz/tpu/models/official/detection/executor/tpu_executor.py", line 204, in evaluate
outputs = six.next(predictor)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 623, in predict
estimator_spec = self._call_model_fn(features, None, ModeKeys.PREDICT,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1174, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 51, in __call__
return self._model.predict(features)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 389, in predict
outputs = self.build_outputs(images, labels, mode=mode_keys.PREDICT)
File "/home/wanjiz/tpu/models/official/detection/modeling/base_model.py", line 206, in build_outputs
outputs = self._build_outputs(images, labels, mode)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 174, in _build_outputs
distill_class_outputs) = self._frcnn_head_fn(roi_features, is_training)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 325, in __call__
class_outputs = _divide_no_nan(class_outputs, classifier_norm[None,
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_head.py", line 38, in _divide_no_nan
tf.greater_equal(tf.broadcast_to(y, x.get_shape()), -epsilon),
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/ops/gen_array_ops.py", line 876, in broadcast_to
_, _, _op, _outputs = _op_def_library._apply_op_helper(
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/op_def_library.py", line 797, in _apply_op_helper
op = g._create_op_internal(op_type_name, inputs, dtypes=None,
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/ops.py", line 3754, in _create_op_internal
ret = Operation(
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/framework/ops.py", line 2133, in __init__
self._traceback = tf_stack.extract_stack_for_node(self._c_op)
Have been getting this error when running the inference script with custom class prompt and dataset with the vild_resnet.yaml configs altered as shown below.
anchor:
anchor_size: 8
aspect_ratios: [1.0, 2.0, 0.5]
num_scales: 1
architecture:
backbone: resnet
feat_distill_weight: 0.5
filter_distill_boxes_size: 0
include_mask: true
mask_target_size: 28
max_level: 6
max_num_rois: 300
min_level: 2
multilevel_features: fpn
normalize_feat_during_training: true
num_classes: 7
parser: vild_parser
pre_parser: null
space_to_depth_block_size: 1
use_bfloat16: false
visual_feature_dim: 512
visual_feature_distill: vanilla
batch_norm_activation:
activation: relu
batch_norm_epsilon: 0.0001
batch_norm_momentum: 0.997
batch_norm_trainable: true
use_sync_bn: true
dropblock:
dropblock_keep_prob: null
dropblock_size: null
enable_summary: false
eval:
eval_batch_size: 8
eval_dataset_type: tfrecord
eval_samples: 2956
eval_timeout: null
min_eval_interval: 5
num_steps_per_eval: 1000
per_category_metrics: true
skip_eval_loss: false
suffix: ''
type: lvis_box
use_json_file: true
fpn:
fpn_feat_dims: 256
use_batch_norm: true
use_separable_conv: false
frcnn_box_loss:
huber_loss_delta: 1.0
frcnn_class_loss:
mask_rare: false
frcnn_head:
class_agnostic_bbox_pred: true
clip_dim: 512
fc_dims: 1024
normalize_classifier: true
normalize_visual: true
num_convs: 4
num_fcs: 2
num_filters: 256
temperature: 100.0
use_batch_norm: true
use_separable_conv: false
mask_sampling:
num_mask_samples_per_image: 128
mrcnn_head:
class_agnostic_mask_pred: true
num_convs: 4
num_filters: 256
use_batch_norm: true
use_separable_conv: false
postprocess:
apply_nms: true
apply_sigmoid: false
discard_background: false
max_total_size: 300
nms_iou_threshold: 0.5
nms_version: v1
pre_nms_num_boxes: 1000
score_threshold: 0.0
use_batched_nms: false
predict:
predict_batch_size: 8
resnet:
init_drop_connect_rate: null
resnet_depth: 50
roi_proposal:
rpn_min_size_threshold: 0.0
rpn_nms_threshold: 0.7
rpn_post_nms_top_k: 1000
rpn_pre_nms_top_k: 2000
rpn_score_threshold: 0.0
test_rpn_min_size_threshold: 0.0
test_rpn_nms_threshold: 0.7
test_rpn_post_nms_top_k: 1000
test_rpn_pre_nms_top_k: 1000
test_rpn_score_threshold: 0.0
use_batched_nms: false
roi_sampling:
bg_iou_thresh_hi: 0.5
bg_iou_thresh_lo: 0.0
cascade_iou_thresholds: null
fg_fraction: 0.25
fg_iou_thresh: 0.5
mix_gt_boxes: true
num_samples_per_image: 512
rpn_box_loss:
huber_loss_delta: 0.1111111111111111
rpn_head:
anchors_per_location: null
cast_to_float32: true
num_convs: 2
num_filters: 256
use_batch_norm: true
use_separable_conv: false
rpn_score_loss:
rpn_batch_size_per_im: 256
train:
checkpoint:
path: ''
prefix: ''
skip_variables_regex: ''
frozen_variable_prefix: frcnn_layer_0/fast_rcnn_head/class-predict
gradient_clip_norm: 0.0
input_partition_dims: null
iterations_per_loop: 100
l2_weight_decay: 4.0e-05
learning_rate:
init_learning_rate: 0.32
learning_rate_levels: [0.032, 0.0032]
learning_rate_steps: [162000, 171000, 175500]
type: step
warmup_learning_rate: 0.0032
warmup_steps: 1000
losses: all
num_cores_per_replica: null
optimizer:
momentum: 0.9
type: momentum
pre_parser_dataset:
dataset_type: tfrecord
file_pattern: ''
regularization_variable_regex: .*(kernel|weight):0$
space_to_depth_block_size: 1
total_steps: 180000
train_batch_size: 256
train_dataset_type: tfrecord
transpose_input: true
type: vild
use_tpu: false
vild_parser:
aug_rand_hflip: true
aug_scale_max: 2.0
aug_scale_min: 0.1
copy_paste: false
mask_crop_size: 112
max_num_instances: 300
output_size: [1024, 1024]
regenerate_source_id: false
rpn_batch_size_per_im: 256
rpn_fg_fraction: 0.5
rpn_match_threshold: 0.7
rpn_unmatched_threshold: 0.3
skip_crowd_during_training: true
Also I've set frcnn_class_loss.mask_rare = false
since I kept on getting this error. every directory is structure the same way as it is mentioned here
Traceback (most recent call last):
File "/home/wanjiz/tpu/models/official/detection/main.py", line 184, in <module>
tf.app.run(main)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/platform/app.py", line 36, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 312, in run
_run_main(main, args)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/absl/app.py", line 258, in _run_main
sys.exit(main(argv))
File "/home/wanjiz/tpu/models/official/detection/main.py", line 116, in main
model_fn = model_builder.ModelFn(params)
File "/home/wanjiz/tpu/models/official/detection/modeling/model_builder.py", line 26, in __init__
self._model = factory.model_generator(params)
File "/home/wanjiz/tpu/models/official/detection/modeling/factory.py", line 41, in model_generator
model_fn = vild_model.ViLDModel(params)
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_model.py", line 73, in __init__
self._frcnn_class_loss_fn = vild_losses.FastrcnnClassLoss(
File "/home/wanjiz/tpu/models/official/detection/projects/vild/modeling/vild_losses.py", line 33, in __init__
self._rare_mask = np.array(np.load(f), dtype=np.float32)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/numpy/lib/npyio.py", line 414, in load
magic = fid.read(N)
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/lib/io/file_io.py", line 114, in read
self._preread_check()
File "/home/wanjiz/vild_finetune/venv/lib/python3.10/site-packages/tensorflow/python/lib/io/file_io.py", line 76, in _preread_check
self._read_buf = _pywrap_file_io.BufferedInputStream(
tensorflow.python.framework.errors_impl.NotFoundError: ; No such file or directory
Please share some insight on how to evaluate ViLD on custom dataset. Thanks!