Recovery failure on deserializing snapshot

[Alireza]

Version Information
Version of Akka.NET? 1.5.13
Which Akka.NET Modules? Akka.Persistence.Redis, Akka.Serialization.Hyperion

Describe the bug
Rarely, an actor couldn’t get started due to snapshot recovery failure, and this exception occurs:

---> System.Runtime.Serialization.SerializationException: Failed to deserialize object of type [oms.models.ActorSnapshots.UserActorState] from the stream. Cause: Failed to deserialize object of type [System.Collections.Generic.List`1[domain.Entities.OrderBaseEntity]] from the stream. Cause: Failed to deserialize object of type [domain.Entities.OrderSellEntity] from the stream. Cause: Unable to cast object of type 'domain.Entities.TradeItem' to type 'domain.Enums.OrderValidityEnum'.

To Reproduce
it's not reproducible.

Expected behavior
To Deserialize Snapshot state successfully. This is the State of the actor which is persisted :

public class UserActorState
{
    public List<OrderBaseEntity> Orders { get; set; } = new();
}

public abstract class OrderBaseEntity : BaseEntity
{
    public sealed override string Id { get; set; }
    public string ParentId { get; set; }
    public string ReferenceId { get; set; } = string.Empty;// algo parentId 
    public long Hon { get; set; }
    public string SymbolIsin { get; set; } 
    public string SymbolName { get; set; }
    public OrderValidityEnum Validity { get; set; }  
    public DateTime? ValidityDate { get; set; } 
    public decimal Price { get; set; } 
    public long Quantity { get; set; } 
    public virtual OrderSide Side { get; protected set; } 
    public string CustomerIsin { get; set; } 
    public OrderState OrderState { get; set; }
    public OrderFrom OrderFrom { get; set; }
    public DateTime? ModifiedAt { get; set; }
    public DateTime? ParentCreationDate { get; set; }
    public long ParentBlock { get; set; }
    public bool IsModifyToUpperCost { get; set; } = false;
    public long ParentRemainedQuantity { get; set; }
    public int OrderErrorCode { get; set; }
    public string OrderErrorText { get; set; }
    public OrderActionEnum OrderAction { get; set; } = OrderActionEnum.Add;
    public int? HidePrice { get; set; }
    public byte? SettlementDelay { get; set; }
    public int CSize { get; protected set; } = 1;//option
    public List<TradeItem> Trades { get; set; } = new();
}

public abstract class BaseEntity
{
    protected BaseEntity()
    {
    }

    protected BaseEntity(string identifier)
    {
        Id = identifier;
    }

    public virtual DateTime CreateDateTime { get; set; } = DateTime.Now;

    [Key, MaxLength(36)]
    public virtual string Id { get; set; } = Ulid.NewUlid().ToString();


    private List<IDomainEvent> _domainEvents;

    [JsonIgnore]
    public IReadOnlyCollection<IDomainEvent> DomainEvents => _domainEvents?.AsReadOnly();
}

public class TradeItem
{
    public TradeItem(DateTime dateTime,
        int tradeNumber,
        string orderId,
        string symbolIsin,
        OrderSide side,
        int? hidePrice,
        long quantity,
        int price,
        bool isCancel)
    {
        DateTime   =dateTime;
        TradeNumber=tradeNumber;
        OrderId    =orderId;
        SymbolIsin =symbolIsin;
        Side       =side;
        HidePrice  =hidePrice;
        Quantity   =quantity;
        Price      =price;
        IsCancel   =isCancel;
    }
    public DateTime DateTime { get; set; }
    public int TradeNumber { get; set; }
    public string OrderId { get; set; }
    public string SymbolIsin { get; set; }
    public OrderSide Side { get; set; }
    public int? HidePrice { get; set; }
    public long Quantity { get; set; }
    public int Price { get; set; }
    public bool IsCancel { get; set; } = false;
}

public class OrderBuyEntity : OrderBaseEntity
{
   public override OrderSide Side { get; protected set; } = OrderSide.Buy;
}

public class OrderSellEntity : OrderBaseEntity
{
   public override OrderSide Side { get; protected set; } = OrderSide.Sell;
}

public enum OrderValidityEnum
{
    DAY = 0,
    GOOD_TILL_DATE = 1
}

Actual behavior
Sometimes Failed to recover snapshot state

Environment
docker, .net8

Additional context
I've configured Hyperion serializer like this:

akka { actor { provider = cluster serializers { hyperion = "Akka.Serialization.HyperionSerializer, Akka.Serialization.Hyperion" } serialization-bindings { "System.Object" = hyperion } }

[Gregorius Soedharmo]

Have you ever modified your model in the past and does it follow the extend only design principle?
It might be best to extend the OnRecoveryFailure() method in your persistence actor and do a dump of the failed message, you can debug the possibly corrupt message from there.