parser does not support ZIP64 archives

Question

parser does not support ZIP64 archives

silvialeung opened this issue 2 years ago · comments

I try to read a programmatically generated xlsx file (which can be open in MS Excel) but it throws a Nodejs memory overflow error despite the file being very small (5KB).
The file can pass the parsing when resaving it in MS Excel though.

code:

const workbook = XLSX.readFile(
        "test_20220812111206.xlsx"
   );

error: (file path is partially hidden)

[app] 2022-08-12 11:12 error: Error: Cannot create a string longer than 0x1fffffe8 characters
    at Object.slice (node:buffer:621:37)
    at Buffer.toString (node:buffer:812:14)
    at cc2str (/node_modules/xlsx/xlsx.js:3272:14)
    at getdatastr (/node_modules/xlsx/xlsx.js:3348:39)
    at getzipstr (/node_modules/xlsx/xlsx.js:3396:19)
    at parse_zip (/node_modules/xlsx/xlsx.js:23118:22)
    at read_zip (/node_modules/xlsx/xlsx.js:23644:9)
    at readSync (/node_modules/xlsx/xlsx.js:23715:69)
    at Object.readFileSync (/node_modules/xlsx/xlsx.js:23738:9)
    at/src/controllers/Import.ts:57:29
    at Generator.next (<anonymous>)
    at fulfilled (/src/controllers/Import.ts:5:58)
    at processTicksAndRejections (node:internal/process/task_queues:96:5) {
  code: 'ERR_STRING_TOO_LONG'
}

Further tracing reveals that the code fails when getting [Content_Types].xml string. I compared the generated excel's and the "resaved in MS Excel application"'s [Content_Types].xml. The differences are as below:

There is an extra line in generated excel.

<Override PartName="/xl/theme/theme1.xml" ContentType="application/vnd.openxmlformats-officedocument.theme+xml" />

There is an standalone attribute in <?xml> line in generated excel

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

The file that causes the issue:
test_20220812111206.xlsx

Eric P Sheets · Answer 1 · Fri Aug 12 2022 11:59:50 GMT+0800 (China Standard Time)

Thanks for sharing! This is a ZIP issue. The writer used the ZIP64 extension, which is not currently supported in the cfb library

silvialeung · Answer 2 · Fri Aug 12 2022 12:07:34 GMT+0800 (China Standard Time)

Thank you for the quick reply. Is there any workaround for this issue currently?

Tony Qu · Answer 3 · Sun Aug 14 2022 08:53:29 GMT+0800 (China Standard Time)

@silvialeung which version of NPOI is used to generate this xlsx file?

Eric P Sheets · Answer 4 · Mon Aug 15 2022 12:46:18 GMT+0800 (China Standard Time)

To wrap up this issue on the SheetJS side:

Reading the ZIP64 extended length info requires a change to bits/18_cfb.js:

--- a/bits/18_cfb.js
+++ b/bits/18_cfb.js
@@ -211,8 +211,15 @@ function parse_extra_field(blob/*:CFBlob*/)/*:any*/ {
                                        if(flags & 4) p.ctime = blob.read_shift(4);
                                }
                                if(p.mtime) p.mt = new Date(p.mtime*1000);
-                       }
-                       break;
+                       } break;
+                       /* ZIP64 Extended Information Field */
+                       case 0x0001: {
+                               var sz1 = blob.read_shift(4), sz2 = blob.read_shift(4);
+                               p.usz = (sz2 * Math.pow(2,32) + sz1);
+                               sz1 = blob.read_shift(4); sz2 = blob.read_shift(4);
+                               p.csz = (sz2 * Math.pow(2,32) + sz1);
+                               // NOTE: volume fields are skipped
+                       } break;
                }
                blob.l = tgt;
                o[type] = p;
@@ -1401,6 +1408,11 @@ function parse_zip(file/*:RawBytes*/, options/*:CFBReadOpts*/)/*:CFBContainer*/
 
                var L = blob.l;
                blob.l = offset + 4;
+               /* ZIP64 lengths */
+               if(EF && EF[0x0001]) {
+                       if((EF[0x0001]||{}).usz) usz = EF[0x0001].usz;
+                       if((EF[0x0001]||{}).csz) csz = EF[0x0001].csz;
+               }
                parse_local_file(blob, csz, usz, o, EF);
                blob.l = L;
        }
@@ -1430,7 +1442,13 @@ function parse_local_file(blob/*:CFBlob*/, csz/*:number*/, usz/*:number*/, o/*:C
        if(efsz) {
                var ef = parse_extra_field(/*::(*/blob.slice(blob.l, blob.l + efsz)/*:: :any)*/);
                if((ef[0x5455]||{}).mt) date = ef[0x5455].mt;
-               if(((EF||{})[0x5455]||{}).mt) date = EF[0x5455].mt;
+               if((ef[0x0001]||{}).usz) _usz = ef[0x0001].usz;
+               if((ef[0x0001]||{}).csz) _csz = ef[0x0001].csz;
+               if(EF) {
+                       if((EF[0x5455]||{}).mt) date = EF[0x5455].mt;
+                       if((EF[0x0001]||{}).usz) _usz = ef[0x0001].usz;
+                       if((EF[0x0001]||{}).csz) _csz = ef[0x0001].csz;
+               }
        }
        blob.l += efsz;

The test file uses a custom namespace in the custom properties, requiring a change to bits/35_custprops.js:

--- a/bits/35_custprops.js
+++ b/bits/35_custprops.js
@@ -5,7 +5,7 @@ function parse_cust_props(data/*:string*/, opts) {
        var m = data.match(custregex);
        if(m) for(var i = 0; i != m.length; ++i) {
                var x = m[i], y = parsexmltag(x);
-               switch(y[0]) {
+               switch(strip_ns(y[0])) {
                        case '<?xml': break;
                        case '<Properties': break;
                        case '<property': name = unescapexml(y.name); break;

(we'll accept a PR, as well as a similar PR for https://github.com/SheetJS/js-cfb )

silvialeung · Answer 5 · Mon Aug 15 2022 18:36:59 GMT+0800 (China Standard Time)

NPOI

@tonyqus Sorry but the version is unknown. The excel was downloaded from a third party private software.

Santosh-745 · Answer 6 · Fri Sep 30 2022 19:38:54 GMT+0800 (China Standard Time)

To wrap up this issue on the SheetJS side:

Reading the ZIP64 extended length info requires a change to bits/18_cfb.js:

--- a/bits/18_cfb.js
+++ b/bits/18_cfb.js
@@ -211,8 +211,15 @@ function parse_extra_field(blob/*:CFBlob*/)/*:any*/ {
                                        if(flags & 4) p.ctime = blob.read_shift(4);
                                }
                                if(p.mtime) p.mt = new Date(p.mtime*1000);
-                       }
-                       break;
+                       } break;
+                       /* ZIP64 Extended Information Field */
+                       case 0x0001: {
+                               var sz1 = blob.read_shift(4), sz2 = blob.read_shift(4);
+                               p.usz = (sz2 * Math.pow(2,32) + sz1);
+                               sz1 = blob.read_shift(4); sz2 = blob.read_shift(4);
+                               p.csz = (sz2 * Math.pow(2,32) + sz1);
+                               // NOTE: volume fields are skipped
+                       } break;
                }
                blob.l = tgt;
                o[type] = p;
@@ -1401,6 +1408,11 @@ function parse_zip(file/*:RawBytes*/, options/*:CFBReadOpts*/)/*:CFBContainer*/
 
                var L = blob.l;
                blob.l = offset + 4;
+               /* ZIP64 lengths */
+               if(EF && EF[0x0001]) {
+                       if((EF[0x0001]||{}).usz) usz = EF[0x0001].usz;
+                       if((EF[0x0001]||{}).csz) csz = EF[0x0001].csz;
+               }
                parse_local_file(blob, csz, usz, o, EF);
                blob.l = L;
        }
@@ -1430,7 +1442,13 @@ function parse_local_file(blob/*:CFBlob*/, csz/*:number*/, usz/*:number*/, o/*:C
        if(efsz) {
                var ef = parse_extra_field(/*::(*/blob.slice(blob.l, blob.l + efsz)/*:: :any)*/);
                if((ef[0x5455]||{}).mt) date = ef[0x5455].mt;
-               if(((EF||{})[0x5455]||{}).mt) date = EF[0x5455].mt;
+               if((ef[0x0001]||{}).usz) _usz = ef[0x0001].usz;
+               if((ef[0x0001]||{}).csz) _csz = ef[0x0001].csz;
+               if(EF) {
+                       if((EF[0x5455]||{}).mt) date = EF[0x5455].mt;
+                       if((EF[0x0001]||{}).usz) _usz = ef[0x0001].usz;
+                       if((EF[0x0001]||{}).csz) _csz = ef[0x0001].csz;
+               }
        }
        blob.l += efsz;

The test file uses a custom namespace in the custom properties, requiring a change to bits/35_custprops.js:

--- a/bits/35_custprops.js
+++ b/bits/35_custprops.js
@@ -5,7 +5,7 @@ function parse_cust_props(data/*:string*/, opts) {
        var m = data.match(custregex);
        if(m) for(var i = 0; i != m.length; ++i) {
                var x = m[i], y = parsexmltag(x);
-               switch(y[0]) {
+               switch(strip_ns(y[0])) {
                        case '<?xml': break;
                        case '<Properties': break;
                        case '<property': name = unescapexml(y.name); break;

(we'll accept a PR, as well as a similar PR for https://github.com/SheetJS/js-cfb )

@SheetJSDev
Is it worked??

Eric P Sheets · Answer 7 · Fri Sep 30 2022 22:01:41 GMT+0800 (China Standard Time)

This commit was part of 0.18.11 and it should work with the current release (0.18.12). See https://docs.sheetjs.com/docs/getting-started/#installation for installation instructions for your deployment

Santosh-745 · Answer 8 · Mon Oct 03 2022 13:50:32 GMT+0800 (China Standard Time)

@SheetJSDev Thanks for the quick reply, I installed version 0.18.12 but it still returns me the same error.

…

On Fri, Sep 30, 2022 at 7:31 PM SheetJSDev ***@***.***> wrote: This commit was part of 0.18.11 and it should work with the current release (0.18.12). See https://docs.sheetjs.com/docs/getting-started/#installation for installation instructions for your deployment — Reply to this email directly, view it on GitHub <#2766 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ATFQKE4G43CXSVT3AUZR7PDWA3XFDANCNFSM56KH7MZA> . You are receiving this because you commented.Message ID: ***@***.***>

Eric P Sheets · Answer 9 · Wed Oct 19 2022 15:41:43 GMT+0800 (China Standard Time)

@Santosh1652 can you share the file?