The im2col_encoding method is not used

[津津有味]

I want to implement two layers of convolution,You provide an im2col_encoding method that encodes the data to the appropriate operation vector. But, after the output of the first convolution, what can I do to make the data fit the input of the second convolution.

The code is as follows. Thank you.
class EncConvNet:
def init(self, torch_nn):
self.conv1_weight = torch_nn.conv1.weight.data.view(
torch_nn.conv1.out_channels, torch_nn.conv1.kernel_size[0],
torch_nn.conv1.kernel_size[1]
).tolist()
self.conv1_bias = torch_nn.conv1.bias.data.tolist()

    self.conv2_weight = torch_nn.conv2.weight.data.view(
        torch_nn.conv2.out_channels, torch_nn.conv2.kernel_size[0],
        torch_nn.conv2.kernel_size[1]
    ).tolist()
    self.conv2_bias = torch_nn.conv2.bias.data.tolist()
    self.fc1_weight = torch_nn.fc1.weight.T.data.tolist()
    self.fc1_bias = torch_nn.fc1.bias.data.tolist()

    self.fc2_weight = torch_nn.fc2.weight.T.data.tolist()
    self.fc2_bias = torch_nn.fc2.bias.data.tolist()

def forward(self, enc_x, windows_nb):
    # conv layer
    enc_channels1 = []
    for kernel, bias in zip(self.conv1_weight, self.conv1_bias):
        y = enc_x.conv2d_im2col(kernel, windows_nb) + bias

        # y = max_pool2d(y)  # add pooling layer
        enc_channels1.append(y)
    enc_x = ts.CKKSVector(context, enc_channels1)
     # conv layer 2
    enc_channels2 = []

    for kernel, bias in zip(self.conv2_weight, self.conv2_bias):
        y = enc_x.conv2d_im2col(kernel, windows_nb) + bias
        enc_channels2.append(y)
    enc_x = ts.CKKSVector.pack_vectors(enc_channels2)
    # square activation
    enc_x.square_()
    # fc1 layer
    enc_x = enc_x.mm(self.fc1_weight) + self.fc1_bias
    # square activation
    enc_x.square_()
    # fc2 layer
    enc_x = enc_x.mm(self.fc2_weight) + self.fc2_bias
    return enc_x
def __call__(self, *args, **kwargs):
    return self.forward(*args, **kwargs)

[Fan Xuwei]

Did you solve it?

[Ayoub Benaissa]

This is unfortunately not possible using im2col encoding, as the ordering has to happen in plaintext (read https://arxiv.org/pdf/2104.03152.pdf, 3.2 is about convolution)