I Need Jszip And Gzip For My Web Page, And Jszip Has All The Ingredients, But Hides Them In Ways I Can't Crack

Support for gzip in JavaScript is surprisingly weak. All browsers implement it for supporting the Content-encoding: gzip header, but there is no standard access to the gzip / gunzi

Solution 1:

As Evert was saying, I should have checked first for the build instructions in the documentation https://stuk.github.io/jszip/documentation/contributing.html.

From that it is clear, first one needs git and makes a local clone. Then one needs to set up the grunt command line, which requires, npm, which comes with nodejs. Once grunt runs, there are other dependencies that need to be npm install-ed. It's the usual little things off and not working, but enough Googling and brute force retrying to get it done.

Now jszip/lib/index.js contains the resource that is finally exported. It is that JSZip object. So just to play with the internal stuff, I could add these to the JSZip object, for example, it already contains:

JSZip.external = require("./external");module.exports = JSZip;

and so we can easily add other resources we want to play with:

JSZip.flate = require("./flate");JSZip.DataWorker = require('./stream/DataWorker');JSZip.DataLengthProbe = require('./stream/DataLengthProbe');JSZip.Crc32Probe = require('./stream/Crc32Probe');JSZip.StreamHelper = require('./stream/StreamHelper');JSZip.pako = require("pako");

Now with that, I can create a proof of concept in the Chrome debugger:

(newJSZip.StreamHelper(
   (newJSZip.DataWorker(Promise.resolve("Hello World! Hello World! Hello World! Hello World! Hello World! Hello World!")))
      .pipe(newJSZip.DataLengthProbe("uncompressedSize"))
      .pipe(newJSZip.Crc32Probe())
      .pipe(JSZip.flate.compressWorker({}))
      .pipe(newJSZip.DataLengthProbe("compressedSize"))
      .on("end", function(event) { console.log("onEnd: ", this.streamInfo) }), 
   "uint8array", "")
).accumulate(function(data) { console.log("acc: ", data); })
 .then(function(data) { console.log("then: ", data); })

and this works. I have been making myself a GZipFileStream with gzip header and trailer, creating everything correctly. I put a jszip/lib/generate/GZipFileWorker.js in as follows:

'use strict';

var external = require('../external');
var utils = require('../utils');
var flate = require('../flate');
varGenericWorker = require('../stream/GenericWorker');
varDataWorker = require('../stream/DataWorker');
varStreamHelper = require('../stream/StreamHelper');
varDataLengthProbe = require('../stream/DataLengthProbe');
varCrc32Probe = require('../stream/Crc32Probe');

functionGZipFileWorker() {
    GenericWorker.call(this, "GZipFileWorker");
    this.virgin = true;
}
utils.inherits(GZipFileWorker, GenericWorker);

GZipFileWorker.prototype.processChunk = function(chunk) {
    if(this.virgin) {
        this.virgin = false;
        var headerBuffer = newArrayBuffer(10);
        var headerView = newDataView(headerBuffer);
        headerView.setUint16(0, 0x8b1f, true); // GZip magic
        headerView.setUint8(2, 0x08); // compression algorithm DEFLATE
        headerView.setUint8(3, 0x00); // flags// bit 0   FTEXT// bit 1   FHCRC// bit 2   FEXTRA// bit 3   FNAME// bit 4   FCOMMENT
        headerView.setUint32(4, (newDate()).getTime()/1000>>>0, true);
        headerView.setUint8(8, 0x00); // no extension headers
        headerView.setUint8(9, 0x03); // OS type UNIXthis.push({data: newUint8Array(headerBuffer)});
    }
    this.push(chunk);
};

GZipFileWorker.prototype.flush = function() {
    var trailerBuffer = newArrayBuffer(8);
    var trailerView = newDataView(trailerBuffer);
    trailerView.setUint32(0, this.streamInfo["crc32"]>>>0, true);
    trailerView.setUint32(4, this.streamInfo["originalSize"]>>>0 & 0xffffffff, true);
    this.push({data: newUint8Array(trailerBuffer)});
};

exports.gzip = function(data, inputFormat, outputFormat, compressionOptions, onUpdate) {
    var mimeType = data.contentType || data.mimeType || "";
    if(! (data instanceofGenericWorker)) {
        inputFormat = (inputFormat || "").toLowerCase();
        data = newDataWorker(
            utils.prepareContent(data.name || "gzip source",
                                 data,
                                 inputFormat !== "string",
                                 inputFormat === "binarystring",
                                 inputFormat === "base64"));
    }
    returnnewStreamHelper(
        data
            .pipe(newDataLengthProbe("originalSize"))
            .pipe(newCrc32Probe())
            .pipe(flate.compressWorker( compressionOptions || {} ))
            .pipe(newGZipFileWorker()),
        outputFormat.toLowerCase(), mimeType).accumulate(onUpdate);
};

and in jszip/lib/index.js I need just this:

var gzip = require("./generate/GZipFileWorker");
JSZip.gzip = gzip.gzip;

and this works like that:

JSZip.gzip("Hello World! Hello World! Hello World! Hello World! Hello World! Hello World!", "string", "base64", {level: 3}).then(function(result) { console.log(result); })

I can paste the result into a UNIX pipe like this:

$ echo -n "H4sIAOyR/VsAA/NIzcnJVwjPL8pJUVTwoJADAPCORolNAAAA" |base64 -d |zcat

and it correctly returns

Hello World! Hello World! Hello World! Hello World! Hello World! Hello World!

It can also be used with files:

JSZip.gzip(file, "", "Blob").then(function(blob) { 
     xhr.setRequestProperty("Content-encoding", "gzip");
     xhr.send(blob); 
  })

and I can send the blob to my web server. I have checked that indeed the large file is processed in chunks.

The only thing I don't like about this is that the final blob is still assembled as one big Blob, so I am assuming it holds all compressed data in memory. It would be better if that Blow was an end-point of that Worker pipeline so that when the xhr.send grabs the data chunk-wise from the Blob, it would consume chunks from the Worker pipeline only then. However, the impact is lessened a lot given that it only holds compressed content, and likely (for me at least) large files would be multi-media files that won't need to be gzip compressed anyway.

I did not write a gunzip function, because frankly, I don't need one and I don't want to make one that fails to properly parse extension headers in the gzip headers. As soon as I have uploaded compressed content to the server (S3 in my case), when I'm fetching it again I assume the browser would do the decompressing for me. I haven't checked that though. If it's becoming a problem I'll come back end edit this answer more.

Here is my fork on github: https://github.com/gschadow/jszip, pull request already entered.